From 6f80a820da24aa10481bfc0cbd99b37aa2bbc9f2 Mon Sep 17 00:00:00 2001 From: retoor Date: Fri, 16 Oct 2026 07:20:44 +0200 Subject: [PATCH] Zo doen wijdat in oost --- CLAUDE.md | 7 + Dockerfile | 2 +- Makefile | 8 + README.md | 17 +- devplacepy/cli/community.py | 158 +++++ devplacepy/cli/main.py | 4 + devplacepy/cli/nodes.py | 132 ++++ devplacepy/config.py | 4 + devplacepy/database/CLAUDE.md | 1 + devplacepy/database/moderation.py | 2 + devplacepy/database/nodes.py | 326 +++++++++- devplacepy/database/schema.py | 139 +++++ devplacepy/database/soft_delete.py | 2 + devplacepy/database/sync.py | 131 ++++ devplacepy/docs_api/groups/__init__.py | 4 + devplacepy/docs_api/groups/admin.py | 17 + devplacepy/docs_api/groups/community.py | 156 +++++ devplacepy/docs_api/groups/nodes.py | 260 ++++++++ devplacepy/docs_api/groups/sync.py | 119 ++++ devplacepy/main.py | 4 + devplacepy/models.py | 94 +++ devplacepy/routers/CLAUDE.md | 3 +- devplacepy/routers/admin/__init__.py | 4 + devplacepy/routers/admin/communities.py | 263 ++++++++ devplacepy/routers/admin/nodes.py | 376 ++++++++++++ devplacepy/routers/admin/settings.py | 2 +- devplacepy/routers/docs/pages.py | 6 + devplacepy/routers/nodes/__init__.py | 5 +- devplacepy/routers/nodes/clients.py | 95 +++ devplacepy/routers/nodes/me.py | 143 +++++ devplacepy/routers/nodes/social.py | 98 +++ devplacepy/routers/nodes/ws.py | 3 + devplacepy/routers/profile/index.py | 14 +- devplacepy/routers/sync/__init__.py | 11 + devplacepy/routers/sync/_shared.py | 66 ++ devplacepy/routers/sync/comments.py | 21 + devplacepy/routers/sync/posts.py | 21 + devplacepy/routers/sync/spec.py | 18 + devplacepy/routers/sync/users.py | 21 + devplacepy/schemas/__init__.py | 9 + devplacepy/schemas/admin.py | 125 ++++ devplacepy/schemas/nodes.py | 71 +++ devplacepy/schemas/profile.py | 1 + devplacepy/schemas/sync.py | 24 + devplacepy/services/audit/categories.py | 1 + devplacepy/services/community/CLAUDE.md | 66 ++ devplacepy/services/community/__init__.py | 1 + devplacepy/services/community/ai.py | 183 ++++++ devplacepy/services/community/discoverer.py | 261 ++++++++ devplacepy/services/community/extract.py | 469 ++++++++++++++ devplacepy/services/community/feeds.py | 262 ++++++++ devplacepy/services/community/fetcher.py | 153 +++++ devplacepy/services/community/files.py | 40 ++ devplacepy/services/community/generator.py | 117 ++++ devplacepy/services/community/learner.py | 581 ++++++++++++++++++ devplacepy/services/community/logbook.py | 39 ++ devplacepy/services/community/pusher.py | 73 +++ devplacepy/services/community/runner.py | 232 +++++++ devplacepy/services/community/runtime.py | 487 +++++++++++++++ devplacepy/services/community/search.py | 99 +++ devplacepy/services/community/service.py | 113 ++++ devplacepy/services/community/spec.py | 91 +++ devplacepy/services/community/store.py | 200 ++++++ .../devii/actions/catalog/__init__.py | 6 + .../services/devii/actions/catalog/admin.py | 2 + .../devii/actions/catalog/community.py | 113 ++++ .../services/devii/actions/catalog/nodes.py | 149 +++++ .../services/devii/actions/catalog/sync.py | 44 ++ .../services/devii/actions/dispatcher.py | 29 + devplacepy/services/devii/agent.py | 9 + devplacepy/services/devii/registry.py | 3 + devplacepy/services/jobs/deepsearch/crawl.py | 46 ++ .../services/jobs/deepsearch/service.py | 11 + devplacepy/services/jobs/deepsearch/worker.py | 6 +- devplacepy/services/moderation/deletion.py | 1 + devplacepy/services/nodes/CLAUDE.md | 67 +- .../services/nodes/adapters/__init__.py | 1 + devplacepy/services/nodes/adapters/base.py | 101 +++ devplacepy/services/nodes/adapters/direct.py | 197 ++++++ devplacepy/services/nodes/adapters/reddit.py | 43 ++ .../services/nodes/adapters/registry.py | 12 + devplacepy/services/nodes/broker.py | 127 ++++ devplacepy/services/nodes/cache.py | 90 +++ devplacepy/services/nodes/circuit.py | 71 +++ devplacepy/services/nodes/dispatcher.py | 273 ++++++++ devplacepy/services/nodes/fabric.py | 204 ++++++ devplacepy/services/nodes/ledger.py | 25 + devplacepy/services/nodes/policy.py | 91 +++ devplacepy/services/nodes/registry.py | 6 + devplacepy/services/nodes/scheduler.py | 163 +++++ devplacepy/services/nodes/service.py | 59 ++ devplacepy/services/nodes/session.py | 74 ++- devplacepy/services/nodes/social.py | 96 +++ devplacepy/services/nodes/spec.py | 7 + devplacepy/services/nodes/tunnel.py | 220 +++++++ devplacepy/services/sync/CLAUDE.md | 87 +++ devplacepy/services/sync/__init__.py | 1 + devplacepy/services/sync/ai.py | 26 + devplacepy/services/sync/aliaser.py | 213 +++++++ devplacepy/services/sync/batch.py | 34 + devplacepy/services/sync/gate.py | 23 + devplacepy/services/sync/identity.py | 194 ++++++ devplacepy/services/sync/normalize.py | 112 ++++ devplacepy/services/sync/options.py | 71 +++ devplacepy/services/sync/processors.py | 516 ++++++++++++++++ devplacepy/services/sync/records.py | 215 +++++++ devplacepy/services/sync/rewriter.py | 65 ++ devplacepy/services/sync/selection.py | 70 +++ devplacepy/services/sync/spec.py | 139 +++++ devplacepy/services/sync/style.py | 70 +++ devplacepy/services/sync/translator.py | 32 + devplacepy/static/css/admin.css | 92 +++ devplacepy/static/css/communities.css | 100 +++ devplacepy/static/css/nodes.css | 304 +++++++++ devplacepy/static/js/AdminNodes.js | 108 ++++ devplacepy/static/js/Application.js | 4 + devplacepy/static/js/NodeDonation.js | 486 +++++++++++++++ devplacepy/templates/_node_donation.html | 196 ++++++ devplacepy/templates/admin_base.html | 6 + devplacepy/templates/admin_communities.html | 74 +++ devplacepy/templates/admin_community.html | 153 +++++ devplacepy/templates/admin_nodes.html | 168 +++++ devplacepy/templates/admin_settings.html | 15 + devplacepy/templates/docs/nodes-protocol.html | 2 +- devplacepy/templates/docs/nodes.html | 104 ++++ devplacepy/templates/docs/privacy.html | 34 +- devplacepy/templates/profile.html | 8 + devplacepy/utils/accounts.py | 91 ++- devplacepy/utils/badges.py | 5 + events.md | 34 +- nodes/client/README.md | 62 ++ nodes/client/build.sh | 54 ++ nodes/client/config.go | 177 ++++++ nodes/client/config_test.go | 116 ++++ nodes/client/fetcher.go | 156 +++++ nodes/client/fetcher_test.go | 198 ++++++ nodes/client/go.mod | 3 + nodes/client/limits.go | 238 +++++++ nodes/client/limits_test.go | 162 +++++ nodes/client/logger.go | 26 + nodes/client/main.go | 211 +++++++ nodes/client/packaging/compose.yaml | 15 + nodes/client/packaging/devplace-node.service | 16 + nodes/client/packaging/nl.devplace.node.plist | 32 + nodes/client/packaging/windows-install.ps1 | 34 + nodes/client/policy.go | 317 ++++++++++ nodes/client/policy_test.go | 233 +++++++ nodes/client/protocol.go | 146 +++++ nodes/client/session.go | 445 ++++++++++++++ nodes/client/session_test.go | 362 +++++++++++ nodes/client/stats.go | 90 +++ nodes/client/tunnel.go | 201 ++++++ nodes/client/ws.go | 358 +++++++++++ nodes/client/ws_test.go | 220 +++++++ nodes/extension/README.md | 70 +++ nodes/extension/build.sh | 41 ++ nodes/extension/manifest.json | 33 + nodes/extension/package.json | 5 + nodes/extension/src/Config.js | 126 ++++ nodes/extension/src/Fetcher.js | 193 ++++++ nodes/extension/src/Limits.js | 167 +++++ nodes/extension/src/Policy.js | 128 ++++ nodes/extension/src/State.js | 37 ++ nodes/extension/src/Usage.js | 81 +++ nodes/extension/src/WsClient.js | 375 +++++++++++ nodes/extension/src/background.js | 135 ++++ nodes/extension/src/spec.js | 104 ++++ nodes/extension/test/fetcher.test.mjs | 197 ++++++ nodes/extension/test/limits.test.mjs | 91 +++ nodes/extension/test/policy.test.mjs | 114 ++++ nodes/extension/test/spec.test.mjs | 38 ++ nodes/extension/test/wsclient.test.mjs | 323 ++++++++++ nodes/extension/ui/disclosure.html | 55 ++ nodes/extension/ui/disclosure.js | 31 + nodes/extension/ui/popup.css | 226 +++++++ nodes/extension/ui/popup.html | 77 +++ nodes/extension/ui/popup.js | 155 +++++ pyproject.toml | 2 +- tests/CLAUDE.md | 2 +- tests/api/admin/communities/__init__.py | 1 + tests/api/admin/communities/index.py | 247 ++++++++ tests/api/admin/nodes/__init__.py | 1 + tests/api/admin/nodes/conftest.py | 36 ++ tests/api/admin/nodes/flags.py | 34 + tests/api/admin/nodes/index.py | 42 ++ tests/api/admin/nodes/killswitch.py | 24 + tests/api/admin/nodes/settings.py | 36 ++ tests/api/admin/nodes/suspend.py | 47 ++ tests/api/nodes/clients.py | 47 ++ tests/api/nodes/jobs.py | 187 ++++++ tests/api/nodes/me.py | 84 +++ tests/api/nodes/social.py | 65 ++ tests/api/sync/__init__.py | 1 + tests/api/sync/comments.py | 70 +++ tests/api/sync/conftest.py | 98 +++ tests/api/sync/posts.py | 118 ++++ tests/api/sync/settings.py | 39 ++ tests/api/sync/spec.py | 15 + tests/api/sync/users.py | 173 ++++++ tests/conftest.py | 1 + tests/unit/cli/nodes.py | 44 ++ tests/unit/database/nodes.py | 167 +++++ tests/unit/services/community/__init__.py | 1 + tests/unit/services/community/ai.py | 67 ++ tests/unit/services/community/alias.py | 133 ++++ tests/unit/services/community/conftest.py | 13 + tests/unit/services/community/discoverer.py | 133 ++++ tests/unit/services/community/extract.py | 60 ++ tests/unit/services/community/fakes.py | 228 +++++++ tests/unit/services/community/generator.py | 168 +++++ tests/unit/services/community/search.py | 39 ++ tests/unit/services/community/service.py | 81 +++ tests/unit/services/devii/actions/catalog.py | 14 + tests/unit/services/nodes/adapters.py | 91 +++ tests/unit/services/nodes/cache.py | 109 ++++ tests/unit/services/nodes/circuit.py | 97 +++ tests/unit/services/nodes/dispatcher.py | 415 +++++++++++++ tests/unit/services/nodes/fabric.py | 231 +++++++ tests/unit/services/nodes/ledger.py | 55 ++ tests/unit/services/nodes/policy.py | 143 +++++ tests/unit/services/nodes/scheduler.py | 283 +++++++++ tests/unit/services/nodes/service.py | 30 + tests/unit/services/nodes/session.py | 13 +- tests/unit/services/nodes/social.py | 88 +++ tests/unit/services/nodes/tunnel.py | 112 ++++ tests/unit/services/sync/__init__.py | 1 + tests/unit/services/sync/alias.py | 273 ++++++++ tests/unit/services/sync/normalize.py | 97 +++ tests/unit/services/sync/records.py | 85 +++ 229 files changed, 23894 insertions(+), 74 deletions(-) create mode 100644 devplacepy/cli/community.py create mode 100644 devplacepy/cli/nodes.py create mode 100644 devplacepy/database/sync.py create mode 100644 devplacepy/docs_api/groups/community.py create mode 100644 devplacepy/docs_api/groups/sync.py create mode 100644 devplacepy/routers/admin/communities.py create mode 100644 devplacepy/routers/admin/nodes.py create mode 100644 devplacepy/routers/nodes/clients.py create mode 100644 devplacepy/routers/nodes/me.py create mode 100644 devplacepy/routers/nodes/social.py create mode 100644 devplacepy/routers/sync/__init__.py create mode 100644 devplacepy/routers/sync/_shared.py create mode 100644 devplacepy/routers/sync/comments.py create mode 100644 devplacepy/routers/sync/posts.py create mode 100644 devplacepy/routers/sync/spec.py create mode 100644 devplacepy/routers/sync/users.py create mode 100644 devplacepy/schemas/sync.py create mode 100644 devplacepy/services/community/CLAUDE.md create mode 100644 devplacepy/services/community/__init__.py create mode 100644 devplacepy/services/community/ai.py create mode 100644 devplacepy/services/community/discoverer.py create mode 100644 devplacepy/services/community/extract.py create mode 100644 devplacepy/services/community/feeds.py create mode 100644 devplacepy/services/community/fetcher.py create mode 100644 devplacepy/services/community/files.py create mode 100644 devplacepy/services/community/generator.py create mode 100644 devplacepy/services/community/learner.py create mode 100644 devplacepy/services/community/logbook.py create mode 100644 devplacepy/services/community/pusher.py create mode 100644 devplacepy/services/community/runner.py create mode 100644 devplacepy/services/community/runtime.py create mode 100644 devplacepy/services/community/search.py create mode 100644 devplacepy/services/community/service.py create mode 100644 devplacepy/services/community/spec.py create mode 100644 devplacepy/services/community/store.py create mode 100644 devplacepy/services/devii/actions/catalog/community.py create mode 100644 devplacepy/services/devii/actions/catalog/nodes.py create mode 100644 devplacepy/services/devii/actions/catalog/sync.py create mode 100644 devplacepy/services/nodes/adapters/__init__.py create mode 100644 devplacepy/services/nodes/adapters/base.py create mode 100644 devplacepy/services/nodes/adapters/direct.py create mode 100644 devplacepy/services/nodes/adapters/reddit.py create mode 100644 devplacepy/services/nodes/adapters/registry.py create mode 100644 devplacepy/services/nodes/broker.py create mode 100644 devplacepy/services/nodes/cache.py create mode 100644 devplacepy/services/nodes/circuit.py create mode 100644 devplacepy/services/nodes/dispatcher.py create mode 100644 devplacepy/services/nodes/fabric.py create mode 100644 devplacepy/services/nodes/policy.py create mode 100644 devplacepy/services/nodes/scheduler.py create mode 100644 devplacepy/services/nodes/social.py create mode 100644 devplacepy/services/nodes/tunnel.py create mode 100644 devplacepy/services/sync/CLAUDE.md create mode 100644 devplacepy/services/sync/__init__.py create mode 100644 devplacepy/services/sync/ai.py create mode 100644 devplacepy/services/sync/aliaser.py create mode 100644 devplacepy/services/sync/batch.py create mode 100644 devplacepy/services/sync/gate.py create mode 100644 devplacepy/services/sync/identity.py create mode 100644 devplacepy/services/sync/normalize.py create mode 100644 devplacepy/services/sync/options.py create mode 100644 devplacepy/services/sync/processors.py create mode 100644 devplacepy/services/sync/records.py create mode 100644 devplacepy/services/sync/rewriter.py create mode 100644 devplacepy/services/sync/selection.py create mode 100644 devplacepy/services/sync/spec.py create mode 100644 devplacepy/services/sync/style.py create mode 100644 devplacepy/services/sync/translator.py create mode 100644 devplacepy/static/css/communities.css create mode 100644 devplacepy/static/css/nodes.css create mode 100644 devplacepy/static/js/AdminNodes.js create mode 100644 devplacepy/static/js/NodeDonation.js create mode 100644 devplacepy/templates/_node_donation.html create mode 100644 devplacepy/templates/admin_communities.html create mode 100644 devplacepy/templates/admin_community.html create mode 100644 devplacepy/templates/admin_nodes.html create mode 100644 devplacepy/templates/docs/nodes.html create mode 100644 nodes/client/README.md create mode 100755 nodes/client/build.sh create mode 100644 nodes/client/config.go create mode 100644 nodes/client/config_test.go create mode 100644 nodes/client/fetcher.go create mode 100644 nodes/client/fetcher_test.go create mode 100644 nodes/client/go.mod create mode 100644 nodes/client/limits.go create mode 100644 nodes/client/limits_test.go create mode 100644 nodes/client/logger.go create mode 100644 nodes/client/main.go create mode 100644 nodes/client/packaging/compose.yaml create mode 100644 nodes/client/packaging/devplace-node.service create mode 100644 nodes/client/packaging/nl.devplace.node.plist create mode 100644 nodes/client/packaging/windows-install.ps1 create mode 100644 nodes/client/policy.go create mode 100644 nodes/client/policy_test.go create mode 100644 nodes/client/protocol.go create mode 100644 nodes/client/session.go create mode 100644 nodes/client/session_test.go create mode 100644 nodes/client/stats.go create mode 100644 nodes/client/tunnel.go create mode 100644 nodes/client/ws.go create mode 100644 nodes/client/ws_test.go create mode 100644 nodes/extension/README.md create mode 100755 nodes/extension/build.sh create mode 100644 nodes/extension/manifest.json create mode 100644 nodes/extension/package.json create mode 100644 nodes/extension/src/Config.js create mode 100644 nodes/extension/src/Fetcher.js create mode 100644 nodes/extension/src/Limits.js create mode 100644 nodes/extension/src/Policy.js create mode 100644 nodes/extension/src/State.js create mode 100644 nodes/extension/src/Usage.js create mode 100644 nodes/extension/src/WsClient.js create mode 100644 nodes/extension/src/background.js create mode 100644 nodes/extension/src/spec.js create mode 100644 nodes/extension/test/fetcher.test.mjs create mode 100644 nodes/extension/test/limits.test.mjs create mode 100644 nodes/extension/test/policy.test.mjs create mode 100644 nodes/extension/test/spec.test.mjs create mode 100644 nodes/extension/test/wsclient.test.mjs create mode 100644 nodes/extension/ui/disclosure.html create mode 100644 nodes/extension/ui/disclosure.js create mode 100644 nodes/extension/ui/popup.css create mode 100644 nodes/extension/ui/popup.html create mode 100644 nodes/extension/ui/popup.js create mode 100644 tests/api/admin/communities/__init__.py create mode 100644 tests/api/admin/communities/index.py create mode 100644 tests/api/admin/nodes/__init__.py create mode 100644 tests/api/admin/nodes/conftest.py create mode 100644 tests/api/admin/nodes/flags.py create mode 100644 tests/api/admin/nodes/index.py create mode 100644 tests/api/admin/nodes/killswitch.py create mode 100644 tests/api/admin/nodes/settings.py create mode 100644 tests/api/admin/nodes/suspend.py create mode 100644 tests/api/nodes/clients.py create mode 100644 tests/api/nodes/jobs.py create mode 100644 tests/api/nodes/me.py create mode 100644 tests/api/nodes/social.py create mode 100644 tests/api/sync/__init__.py create mode 100644 tests/api/sync/comments.py create mode 100644 tests/api/sync/conftest.py create mode 100644 tests/api/sync/posts.py create mode 100644 tests/api/sync/settings.py create mode 100644 tests/api/sync/spec.py create mode 100644 tests/api/sync/users.py create mode 100644 tests/unit/cli/nodes.py create mode 100644 tests/unit/services/community/__init__.py create mode 100644 tests/unit/services/community/ai.py create mode 100644 tests/unit/services/community/alias.py create mode 100644 tests/unit/services/community/conftest.py create mode 100644 tests/unit/services/community/discoverer.py create mode 100644 tests/unit/services/community/extract.py create mode 100644 tests/unit/services/community/fakes.py create mode 100644 tests/unit/services/community/generator.py create mode 100644 tests/unit/services/community/search.py create mode 100644 tests/unit/services/community/service.py create mode 100644 tests/unit/services/nodes/adapters.py create mode 100644 tests/unit/services/nodes/cache.py create mode 100644 tests/unit/services/nodes/circuit.py create mode 100644 tests/unit/services/nodes/dispatcher.py create mode 100644 tests/unit/services/nodes/fabric.py create mode 100644 tests/unit/services/nodes/ledger.py create mode 100644 tests/unit/services/nodes/policy.py create mode 100644 tests/unit/services/nodes/scheduler.py create mode 100644 tests/unit/services/nodes/social.py create mode 100644 tests/unit/services/nodes/tunnel.py create mode 100644 tests/unit/services/sync/__init__.py create mode 100644 tests/unit/services/sync/alias.py create mode 100644 tests/unit/services/sync/normalize.py create mode 100644 tests/unit/services/sync/records.py diff --git a/CLAUDE.md b/CLAUDE.md index 99c1d1a2..c0d604df 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,6 +73,10 @@ devplace zips prune # delete expired zip archives + job rows devplace zips clear # delete every zip archive + job row devplace forks prune # delete expired completed fork job rows (forked projects persist) devplace forks clear # delete every fork job row (forked projects persist) +devplace community discover [--run] [--relearn] [--dry-run] # discover, learn and generate collectors for an external community +devplace community run [--dry-run] # run a community's collectors through the sync API +devplace community login [--login-user U] [--login-email E] [--login-password P | --clear] # store or remove the account used to log in +devplace community list # list communities devplace messaging prune-tickets # delete expired WebSocket auth tickets (ws_tickets) devplace seo prune # delete expired SEO audit reports + job rows devplace seo clear # delete every SEO audit report + job row @@ -163,6 +167,8 @@ Nested `CLAUDE.md` files (loaded automatically by Claude Code only when a file i | `devplacepy/services/quiz/CLAUDE.md` | Quizzes (the terminal publish lock, attempt atomicity, answer-key withholding, AI free-text grading, the best-attempt scoreboard) | | `devplacepy/services/skills/CLAUDE.md` | Agent Skills: strict `SKILL.md` parser, SSRF-guarded fetcher, the bounded prompt index, per-user privacy, soft delete and account-deletion cascade, why a skill is data and never code | | `devplacepy/services/nodes/CLAUDE.md` | Community node network: the protocol specification (single source of truth), the no-open-proxy rule, the two transports and their trust levels, the decisions taken, and the gaps per phase | +| `devplacepy/services/community/CLAUDE.md` | Community discovery: rsearch + feed detection, one-time structure learning, generated collectors, dry run, the failure floor, admin-only visibility | +| `devplacepy/services/sync/CLAUDE.md` | Community sync: the origin link table, idempotent batch upserts, normalization and rejection rules, what is deliberately not imported, the race that was measured | | `devplacepy/services/CLAUDE.md` | Background task queue, AI correction/modifier, presence, live view relay, `BaseService`/`ServiceManager` | | `devplacepy/database/CLAUDE.md` | Dataset rules, indexing conventions, soft delete, tables, site settings | | `devplacepy/utils/CLAUDE.md` | Notifications and gamification (XP/levels/badges/leaderboard) | @@ -206,6 +212,7 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror | `/xmlrpc` | xmlrpc.py - see `services/xmlrpc/CLAUDE.md` | | `/api` | devrant/ package - see `routers/devrant/CLAUDE.md` | | `/dbapi` | dbapi/ package, **primary-administrator-only, strictly READ-ONLY** - see `services/dbapi/CLAUDE.md` | +| `/sync` | sync/ package, **administrator only**, off by default - see `services/sync/CLAUDE.md` | | `/game` | game/ package - see `services/game/CLAUDE.md` | | `/reports`, `/admin/moderation`, `/workspaces` | reports.py, admin/moderation.py, workspaces.py - see `services/moderation/CLAUDE.md` | | `/quizzes` | quizzes/ package - see `services/quiz/CLAUDE.md` | diff --git a/Dockerfile b/Dockerfile index af2c25ea..630bf73e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM docker.io/library/python:3.13-slim WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ - curl ca-certificates rclone \ + curl ca-certificates rclone git \ libglib2.0-0 libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz0b libfontconfig1 \ fonts-dejavu-core \ && rm -rf /var/lib/apt/lists/* diff --git a/Makefile b/Makefile index 8a4a4c76..6ca0a353 100644 --- a/Makefile +++ b/Makefile @@ -148,6 +148,14 @@ clean: test-cache-clean: rm -rf .pytest_cache +# Builds the donor clients (Go binaries for five platforms and the browser +# extension zip) into the directory the /nodes/download route serves. +NODE_CLIENT_OUT ?= data/node_clients +.PHONY: node-client +node-client: + sh nodes/client/build.sh $(NODE_CLIENT_OUT) + sh nodes/extension/build.sh $(NODE_CLIENT_OUT) + # Safe but aggressive disk-space cleanup, acting on the REAL data/devplace.db # and data/ tree (never the test database) - see CLAUDE.md "devplace system # prune". prune-dry-run reports what would be removed without deleting. diff --git a/README.md b/README.md index a6517456..54b265b4 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,8 @@ devplacepy/ | `/feed` | Post feed with topic/tab filtering and free-text `search` (title, content, and author username) in the left panel (public). Each page interleaves authors so no two consecutive posts share an author. | | `/topics` | Crawlable per-topic category pages (public): `/topics` hub links every topic with a live post count, `/topics/{topic}` lists that topic's posts with its own canonical URL, title, and breadcrumbs | | `/skills` | Per-user Agent Skills: list, detail, install from a URL, enable/disable, delete. Private to the owner, `noindex` | -| `/nodes` | Community node network: `GET /nodes/spec.json`, `GET/POST /nodes/devices` (+ `/rotate`, `/delete`), `POST /nodes/ticket`, `WS /nodes/ws`. Server side only, off by default, no client yet (see `plan.md`) | +| `/nodes` | Community node network: `GET /nodes/spec.json`, `GET/POST /nodes/devices` (+ `/rotate`, `/delete`), `POST /nodes/ticket`, `WS /nodes/ws`. Off by default; also `GET /nodes/me`, `POST /nodes/settings`, `GET /nodes/social/search`, `POST /nodes/internal/fetch`, `GET /nodes/stats`, `GET /nodes/leaderboard`, `GET /nodes/client-version`, `GET /nodes/download/{artifact}` (see `plan.md`) | +| `/sync` | Community sync: `GET /sync/spec.json`, `POST /sync/{users,posts,comments}`. Administrator only, off by default (`sync_enabled`), idempotent batch upserts keyed on the origin community's ids | | `/news` | Developer news listing, detail page with comments | | `/posts` | Post detail, creation | | `/gists` | Code gist listing, detail, creation, and editing; left panel offers language filtering and free-text `search` (title, description, and author username), public read. A plain Markdown gist has a **View rendered / View raw** toggle beside its Copy button, switching between the raw source and the same rendered view a `Markdown Rendered` gist always shows | @@ -252,9 +253,19 @@ DevPlace is open in the sense that it does not editorialise technical opinion. I Members can install [Agent Skills](https://agentskills.io) from a URL at `/skills` so Devii can use them. A skill is a `SKILL.md` (YAML frontmatter plus markdown instructions) with optional text files under `scripts/`, `references/` and `assets/`. Skills are private to their owner, count against a per-user cap, and only the name and a one-line description of enabled skills are added to Devii's system prompt; Devii loads the full text on demand. A skill is untrusted third-party text: it can never grant Devii a tool, bypass a confirmation or run a script. Deleting a skill is a soft delete and needs confirmation when Devii does it. Documented in the API docs under *Skills* and in `devplacepy/services/skills/CLAUDE.md`. -## Community nodes (server side built, no client yet) +## Community sync -Members will be able to opt in to donate a little of their connection to the DevPlace community, so search can reach sites that block datacenter addresses. **Today the server side exists:** the specification (`GET /nodes/spec.json` and the [Node network protocol](/docs/nodes-protocol.html) docs page, both generated from `devplacepy/services/nodes/spec.py`), device registration (`/nodes/devices`), the socket ticket (`POST /nodes/ticket`), the node socket (`WS /nodes/ws`), the consent gate and the administrator kill switch (`nodes_enabled`, off by default). There is no client and no search integration yet, so nothing donates and nothing is relayed. +Users, posts and comments can be copied in from another community. The sending side posts batches of up to 50 records to `POST /sync/users`, `/sync/posts` and `/sync/comments` (administrator API key, `sync_enabled` set to `1`). Each record carries the id it has in the origin community (`origin_uid`); the server keeps a `sync_links` row from `(origin, kind, origin_uid)` to the local row, so a repeated record is an update or a no-op and never a duplicate. Records are validated and normalized with Pydantic (`devplacepy/services/sync/records.py`) and every record gets its own result, so one bad record never fails the batch. Votes, stars, XP and levels cannot be verified across communities and are not imported; they are listed under `ignored` in the result. Synced content sends no notifications, awards no XP, runs no AI correction and passes the content filter. `sync_origins` optionally restricts which origins may send. Limits, statuses and the record schemas are published at `GET /sync/spec.json`. There is no export endpoint and no scheduled sync service yet; the batch API is the building block for it. Usage is written up in `prompt.md`. + +### Community discovery + +Administrators can add an external community by name or address at `/admin/communities` (or `devplace community discover`). The system searches (rsearch), detects RSS, Atom, JSON and HTML feeds, learns the structure once, records the community's description, rules and size for the admin detail page, generates standalone collector code under `data/communities//`, and then runs it on a schedule through the sync API above. Members never see these communities. Login-protected communities take a stored account; JavaScript pages are rendered through stealthii; the internal AI model helps with HTML structure it cannot read otherwise. Administrators can manage everything through the admin pages, `devplace community ...` and Devii. Dry run (`--dry-run`, the admin button, and the default of the `Community sync` service) validates and logs everything without writing. Design and limits: `add_community.md` and `devplacepy/services/community/CLAUDE.md`. Per community, administrators can optionally (all off by default) sample which posts are accepted, filter by topic and minimum comments, jitter the schedule, translate content, and store authors under permanent pseudonyms with every mention rewritten (`community_sync_additional.md`); this is administrator-only and invisible to members. + +## Community nodes (built, off by default) + +Members will be able to opt in to donate a little of their connection to the DevPlace community, so search can reach sites that block datacenter addresses. **Today the server side exists:** the specification (`GET /nodes/spec.json` and the [Node network protocol](/docs/nodes-protocol.html) docs page, both generated from `devplacepy/services/nodes/spec.py`), device registration (`/nodes/devices`), the socket ticket (`POST /nodes/ticket`), the node socket (`WS /nodes/ws`), the consent gate and the administrator kill switch (`nodes_enabled`, off by default). The server queues, schedules and meters jobs and relays them either as a `tunnel` (raw TCP, TLS stays end to end between DevPlace and the site, so the donor cannot read or forge the page) or as a `fetch` (extension donors, results verified and labelled). Donor clients are a Go binary (`nodes/client/`) and a browser extension (`nodes/extension/`); `make node-client` builds them into `data/node_clients/`, served at `/nodes/client-version` and `/nodes/download/{artifact}`. Social search (`GET /nodes/social/search`, Devii `social_search`, optional DeepSearch source) reads Hacker News, Stack Overflow, GitHub, dev.to and Mastodon directly and Reddit through donors when `nodes_social_via_nodes` is on. Whether Reddit works from residential donors is not measured yet. Administrators manage it at `/admin/nodes` (pool and queue summary, devices by status, kind, address class and country, open flags, timed device suspension, the kill switch and the caps), with the same operations available to Devii for administrators. + +Members manage their donation on the owner-only **Donate** tab of their profile (`/profile/{username}?tab=donate`, noindex): the explanation, the `bandwidth_sharing` consent, limits and active hours, devices with a one-time secret and install lines, and a live "Donated by you" card. The user guide is [Donate your connection](/docs/nodes.html). What is fixed already, because the specification enforces it: nobody is enrolled by default; a node is not an open proxy and only ever contacts a short published allowlist of websites, on ports 80 and 443, never a private address; the refusal is enforced on the donor's machine as well as on the server; every limit is a published hard cap. The implementation plan, the measured and unmeasured claims, and the legal gate are in `plan.md`. diff --git a/devplacepy/cli/community.py b/devplacepy/cli/community.py new file mode 100644 index 00000000..b1726e70 --- /dev/null +++ b/devplacepy/cli/community.py @@ -0,0 +1,158 @@ +# retoor + +import asyncio +import os +import sys + +from devplacepy.cli._shared import _audit_cli + + +def _prepare(): + from devplacepy.database import init_db + + init_db() + + +def _credentials(args) -> dict: + return { + "username": args.login_user or "", + "email": args.login_email or "", + "password": args.login_password or os.environ.get("COMMUNITY_PASSWORD", ""), + } + + +def _print_report(report): + mode = "DRY RUN" if report.dry_run else "LIVE" + print(f"\n[{mode}] status: {report.status}") + print(f"collected: {report.collected}") + for kind, counts in report.pushed.items(): + print(f" {kind}: {counts or 'nothing'}") + for error in report.errors[:10]: + print(f" problem: {error}") + + +def cmd_community_discover(args): + from devplacepy.services.community.logbook import LogBook + from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner + from devplacepy.services.community.spec import STATUS_READY + from devplacepy.services.community.store import store + + from devplacepy.services.community.discoverer import slug_for_query + + _prepare() + runner = CommunityRunner() + existing = store.get(slug_for_query(args.query)) + if existing and not args.relearn and not args.dry_run: + print(f"Community '{existing['slug']}' already exists ({existing['status']}); pass --relearn to learn it again") + if args.run and existing["status"] == STATUS_READY: + with LogBook(echo=True): + _print_report(asyncio.run(runner.run(existing, dry_run=False, trigger=TRIGGER_CLI))) + return + with LogBook(echo=True): + result, row, _ = asyncio.run( + runner.discover( + args.query, + "cli", + dry_run=args.dry_run, + relearn_slug=existing["slug"] if existing else None, + credentials=_credentials(args), + ) + ) + print(f"\n{result.slug or '-'}: {result.status}" + (f" ({result.error})" if result.error else "")) + if result.status == STATUS_READY: + for feed in result.schema["feeds"]: + print(f" feed {feed['id']}: {feed['type']} {feed['url']} comments={feed['comments']['mode']} pagination={feed['pagination']['style']}") + print(f" sample: { {kind: len(rows) for kind, rows in result.sample['records'].items()} }") + if args.dry_run: + print("dry run: nothing was saved") + else: + _audit_cli("cli.community.discover", f"CLI discovered community {result.slug}: {result.status}", target_label=result.slug, metadata={"status": result.status}) + if result.status != STATUS_READY and not args.dry_run and result.status != "login_required": + sys.exit(1) + if args.run and row and result.status == STATUS_READY and not args.dry_run: + with LogBook(echo=True): + _print_report(asyncio.run(runner.run(store.get(row["uid"]), dry_run=False, trigger=TRIGGER_CLI))) + + +def cmd_community_run(args): + from devplacepy.services.community.logbook import LogBook + from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner + from devplacepy.services.community.store import store + + _prepare() + row = store.get(args.slug) + if row is None: + print(f"Unknown community '{args.slug}'") + sys.exit(1) + with LogBook(echo=True): + report = asyncio.run(CommunityRunner().run(row, dry_run=args.dry_run, trigger=TRIGGER_CLI)) + _print_report(report) + if not args.dry_run: + _audit_cli("cli.community.run", f"CLI ran community {row['slug']}: {report.status}", target_label=row["slug"], metadata={"status": report.status, "collected": report.collected}) + if report.status == "failed": + sys.exit(1) + + +def cmd_community_login(args): + from devplacepy.services.community.store import store + + _prepare() + row = store.get(args.slug) + if row is None: + print(f"Unknown community '{args.slug}'") + sys.exit(1) + if args.clear: + store.update(row["uid"], login_username="", login_email="", login_password="") + print(f"Removed the stored account of {row['slug']}") + else: + secrets = _credentials(args) + if not secrets["password"] or not (secrets["username"] or secrets["email"]): + print("Give --login-user or --login-email, and --login-password (or COMMUNITY_PASSWORD)") + sys.exit(1) + store.update(row["uid"], login_username=secrets["username"], login_email=secrets["email"], login_password=secrets["password"]) + print(f"Stored the account for {row['slug']}; run 'community discover \"{row['query']}\" --relearn' to learn behind the login") + _audit_cli("cli.community.login", f"CLI changed the stored account of community {row['slug']}", target_label=row["slug"]) + + +def cmd_community_list(args): + from devplacepy.services.community.store import store + + _prepare() + rows = store.list() + if not rows: + print("No communities") + return + for row in rows: + print( + f"{row['slug']:<28} {row['status']:<15} runs={row.get('run_count') or 0:<4} " + f"last={row.get('last_run_at') or '-'} next={row.get('next_run_at') or '-'}" + ) + + +def add_login_arguments(parser): + parser.add_argument("--login-user", help="username of an account on the community") + parser.add_argument("--login-email", help="email of an account on the community") + parser.add_argument("--login-password", help="password (or set COMMUNITY_PASSWORD)") + + +def register_community(subparsers): + community = subparsers.add_parser("community", help="External community discovery and sync") + sub = community.add_subparsers(title="action", dest="action") + discover = sub.add_parser("discover", help="Discover, learn and generate collectors for a community") + discover.add_argument("query", help="community name, address or search phrase") + discover.add_argument("--run", action="store_true", help="also run the collectors once, live") + discover.add_argument("--relearn", action="store_true", help="learn again even if the community exists") + discover.add_argument("--dry-run", action="store_true", help="learn and report, save nothing") + add_login_arguments(discover) + discover.set_defaults(func=cmd_community_discover) + run = sub.add_parser("run", help="Run the generated collectors of a community") + run.add_argument("slug") + run.add_argument("--dry-run", action="store_true", help="collect and validate, write nothing, keep the watermark") + run.set_defaults(func=cmd_community_run) + login = sub.add_parser("login", help="Store or remove the account used to log in to a community") + login.add_argument("slug") + login.add_argument("--clear", action="store_true", help="remove the stored account") + add_login_arguments(login) + login.set_defaults(func=cmd_community_login) + listing = sub.add_parser("list", help="List communities") + listing.set_defaults(func=cmd_community_list) diff --git a/devplacepy/cli/main.py b/devplacepy/cli/main.py index d9e879cc..73f15086 100644 --- a/devplacepy/cli/main.py +++ b/devplacepy/cli/main.py @@ -18,6 +18,8 @@ from devplacepy.cli.quiz import register_quiz from devplacepy.cli.gateway import register_gateway from devplacepy.cli.messaging import register_messaging from devplacepy.cli.system import register_system +from devplacepy.cli.community import register_community +from devplacepy.cli.nodes import register_nodes def build_parser(): @@ -40,6 +42,8 @@ def build_parser(): register_messaging(sub) register_accounts(sub) register_system(sub) + register_community(sub) + register_nodes(sub) return parser diff --git a/devplacepy/cli/nodes.py b/devplacepy/cli/nodes.py new file mode 100644 index 00000000..35248613 --- /dev/null +++ b/devplacepy/cli/nodes.py @@ -0,0 +1,132 @@ +# retoor + +import sys + +from devplacepy.cli._shared import _audit_cli + +LIST_LIMIT = 200 +SUSPEND_DEFAULT_MINUTES = 60 +SUSPEND_MAX_MINUTES = 7 * 24 * 60 + + +def _prepare(): + from devplacepy.database import init_db + + init_db() + + +def cmd_nodes_list(args): + from devplacepy.database import get_table + from devplacepy.database.nodes import effective_status + + _prepare() + rows = list(get_table("node_devices").find(deleted_at=None, order_by=["-created_at"], _limit=LIST_LIMIT)) + if not rows: + print("No devices") + return + for row in rows: + print( + f"{row['uid']} {effective_status(row):<10} {row.get('kind') or '-':<9} " + f"{row.get('platform') or '-':<14} user={row['user_uid']} last={row.get('last_seen_at') or '-'}" + ) + + +def cmd_nodes_stats(args): + from devplacepy.database.nodes import list_flags, public_stats + + _prepare() + stats = public_stats() + print(f"donors: {stats['donors']} requests: {stats['requests']} bytes: {stats['bytes']}") + print(f"open flags: {len(list_flags())}") + + +def cmd_nodes_flags(args): + from devplacepy.database.nodes import list_flags + + _prepare() + flags = list_flags() + if not flags: + print("No open flags") + return + for flag in flags: + print(f"{flag['uid']} {flag.get('kind') or '-':<22} device={flag.get('device_uid') or '-'} {flag.get('detail') or ''}") + + +def cmd_nodes_suspend(args): + from devplacepy.database.nodes import suspend_device + + _prepare() + minutes = max(1, min(args.minutes, SUSPEND_MAX_MINUTES)) + if suspend_device(args.device_uid, args.reason, minutes): + print(f"Suspended {args.device_uid} for {minutes} minutes") + _audit_cli( + "cli.nodes.suspend", + f"CLI suspended device {args.device_uid}", + target_type="node_device", + target_uid=args.device_uid, + metadata={"minutes": minutes, "reason": args.reason}, + ) + return + print(f"No active device {args.device_uid}") + sys.exit(1) + + +def cmd_nodes_unsuspend(args): + from devplacepy.database.nodes import unsuspend_device + + _prepare() + if unsuspend_device(args.device_uid): + print(f"Lifted the suspension of {args.device_uid}") + _audit_cli( + "cli.nodes.unsuspend", + f"CLI lifted the suspension of device {args.device_uid}", + target_type="node_device", + target_uid=args.device_uid, + ) + return + print(f"{args.device_uid} is not suspended") + sys.exit(1) + + +def cmd_nodes_prune(args): + from devplacepy.services.nodes.ledger import UsageLedger + + _prepare() + counts = UsageLedger().prune(dry_run=args.dry_run) + label = "would remove" if args.dry_run else "removed" + for name, count in counts.items(): + print(f"{label} {count} {name}") + + +def cmd_nodes_clear(args): + from devplacepy.database.nodes import clear_ephemeral + + _prepare() + counts = clear_ephemeral(dry_run=not args.force) + label = "removed" if args.force else "would remove (pass --force)" + for name, count in counts.items(): + print(f"{label} {count} {name}") + if args.force: + _audit_cli("cli.nodes.clear","CLI cleared the node job queue and event log", metadata=counts) + + +def register_nodes(subparsers): + nodes = subparsers.add_parser("nodes", help="Community donor node network") + sub = nodes.add_subparsers(title="action", dest="action") + sub.add_parser("list", help="List devices").set_defaults(func=cmd_nodes_list) + sub.add_parser("stats", help="Aggregate donation totals and open flags").set_defaults(func=cmd_nodes_stats) + sub.add_parser("flags", help="List open flags that need a human").set_defaults(func=cmd_nodes_flags) + suspend = sub.add_parser("suspend", help="Suspend one device for a while") + suspend.add_argument("device_uid") + suspend.add_argument("--minutes", type=int, default=SUSPEND_DEFAULT_MINUTES) + suspend.add_argument("--reason", default="suspended from the command line") + suspend.set_defaults(func=cmd_nodes_suspend) + unsuspend = sub.add_parser("unsuspend", help="Lift the suspension of a device") + unsuspend.add_argument("device_uid") + unsuspend.set_defaults(func=cmd_nodes_unsuspend) + prune = sub.add_parser("prune", help="Prune usage buckets, events and finished requests past their retention") + prune.add_argument("--dry-run", action="store_true", help="report what would be removed") + prune.set_defaults(func=cmd_nodes_prune) + clear = sub.add_parser("clear", help="Empty the job queue and the event log (never usage or donor totals)") + clear.add_argument("--force", action="store_true", help="actually delete; without it only counts are shown") + clear.set_defaults(func=cmd_nodes_clear) diff --git a/devplacepy/config.py b/devplacepy/config.py index 8efec4ef..37a75674 100644 --- a/devplacepy/config.py +++ b/devplacepy/config.py @@ -25,6 +25,7 @@ ZIP_STAGING_DIR = DATA_DIR / "zip_staging" FORK_STAGING_DIR = DATA_DIR / "fork_staging" BACKUPS_DIR = DATA_DIR / "backups" BACKUP_STAGING_DIR = DATA_DIR / "backup_staging" +NODE_CLIENTS_DIR = DATA_DIR / "node_clients" RCLONE_BIN = environ.get("DEVPLACE_RCLONE_BIN", "rclone") RCLONE_CONFIG_FILE = environ.get( @@ -45,6 +46,7 @@ ISSLOP_RUNS_DIR = ISSLOP_DIR / "runs" ISSLOP_MEDIA_DIR = ISSLOP_DIR / "media" KEYS_DIR = DATA_DIR / "keys" BOT_DIR = DATA_DIR / "bot" +COMMUNITIES_DIR = DATA_DIR / "communities" LOCKS_DIR = DATA_DIR / "locks" DEVII_TASKS_DB = DATA_DIR / "devii_tasks.db" @@ -152,6 +154,7 @@ DATA_PATHS: dict[str, Path] = { "fork_staging": FORK_STAGING_DIR, "backups": BACKUPS_DIR, "backup_staging": BACKUP_STAGING_DIR, + "node_clients": NODE_CLIENTS_DIR, "seo_reports": SEO_REPORTS_DIR, "planning_reports": PLANNING_REPORTS_DIR, "dbapi": DBAPI_DIR, @@ -163,6 +166,7 @@ DATA_PATHS: dict[str, Path] = { "isslop_media": ISSLOP_MEDIA_DIR, "keys": KEYS_DIR, "bot": BOT_DIR, + "communities": COMMUNITIES_DIR, "locks": LOCKS_DIR, } diff --git a/devplacepy/database/CLAUDE.md b/devplacepy/database/CLAUDE.md index c7593654..f5afd0a8 100644 --- a/devplacepy/database/CLAUDE.md +++ b/devplacepy/database/CLAUDE.md @@ -223,6 +223,7 @@ Site settings are seeded on startup (`site_settings` table): | `terms_version` / `privacy_version` / `guidelines_version` | `"1"` | Bumping `terms_version` forces re-acceptance before the next write. **Every reader uses `get_setting(key, "1") or "1"`** - an empty stored value must read as the default or the gate 403s every write | | `ai_third_party_provider` | `""` | Named in the consent copy and the privacy policy | | `extra_head` | `""` | Raw HTML emitted verbatim into every page `` by `templating.extra_head_tag()`; site-wide trusted-admin input, not sanitized | +| `sync_enabled` / `sync_origins` | `"0"` / `""` | Community sync kill switch (`routers/sync/_shared.py`) and an optional comma separated allowlist of sending origins; empty allows any origin | Besides the site/news/upload keys above, the **Operational** group is admin-editable at `/admin/settings`: `site_url` (public origin for absolute links incl. container ingress; resolved by `seo.public_base_url()` = setting -> `DEVPLACE_SITE_URL` env -> request origin), `rate_limit_per_minute`, `rate_limit_window_seconds`, `news_service_interval`, `session_max_age_days`, `session_remember_days`, `registration_open`, `maintenance_mode`, `maintenance_message`, `docs_search_mode` (`agent`|`bm25`, default `agent` - picks the `/docs/search.html` surface: the in-page Devii chat or the classic BM25 list; a viewer over their daily AI limit, or a guest when Devii is disabled, auto-falls-back to BM25 via `routers/docs/views.py` `_agent_search_state`), `outbound_proxy_url` (empty by default - when set, every `stealth.stealth_async_client`/`stealth_sync_client` call across the whole app routes through it via `stealth.configured_proxy_url()`; validated as `http(s)://`/`socks5(h)://` with a host in `AdminSettingsForm`; falls back to `DEVPLACE_OUTBOUND_PROXY_URL` when unset - see the "Outbound HTTP" note in the root `CLAUDE.md`). The **Custom Code** key `extra_head` and the `ios_app_url` badge link are the keys in the settings handler's `CLEARABLE_SETTINGS` set, so saving an empty textarea/field removes them (the default loop skips empty values). diff --git a/devplacepy/database/moderation.py b/devplacepy/database/moderation.py index c68f27a6..bcdf0294 100644 --- a/devplacepy/database/moderation.py +++ b/devplacepy/database/moderation.py @@ -49,10 +49,12 @@ UNREPORTABLE_TABLES: dict[str, str] = { "bookmarks": "private to the owner", "notes": "private to the owner", "skills": "private to the owner, never shown to others", + "communities": "administrator-only configuration, never shown to members", "node_devices": "private to the owner, the label is never shown to others", "node_settings": "private to the owner", "node_usage_daily": "derived donation counters, carry no authored content", "node_donors": "derived donation totals, carry no authored content", + "node_caller_daily": "derived daily request counters, carry no authored content", "node_flags": "operational flags for administrators, carry no authored content", "follows": "relationship rows, carry no authored content", "poll_votes": "private ballots", diff --git a/devplacepy/database/nodes.py b/devplacepy/database/nodes.py index a0e63f77..00c5f199 100644 --- a/devplacepy/database/nodes.py +++ b/devplacepy/database/nodes.py @@ -621,9 +621,20 @@ def prune_usage_daily(*, dry_run: bool = False, moment: datetime | None = None) ) ) )["c"] - if not dry_run and count: - _execute("DELETE FROM node_usage_daily WHERE day < :cutoff", {"cutoff": cutoff}) - return count + callers = next( + iter( + db.query( + "SELECT COUNT(*) AS c FROM node_caller_daily WHERE day < :cutoff", + cutoff=cutoff, + ) + ) + )["c"] + if not dry_run: + if count: + _execute("DELETE FROM node_usage_daily WHERE day < :cutoff", {"cutoff": cutoff}) + if callers: + _execute("DELETE FROM node_caller_daily WHERE day < :cutoff", {"cutoff": cutoff}) + return count + callers def prune_events(*, dry_run: bool = False, moment: datetime | None = None) -> int: @@ -669,3 +680,312 @@ def prune_requests(*, dry_run: bool = False, moment: datetime | None = None) -> if not dry_run and count: _execute("DELETE FROM node_requests WHERE created_at < :cutoff", {"cutoff": cutoff}) return count + + +def clear_ephemeral(*, dry_run: bool = True) -> dict: + counts = {} + for table in ("node_requests", "node_events"): + counts[table] = next(iter(db.query(f"SELECT COUNT(*) AS c FROM {table}")))["c"] + if not dry_run and counts[table]: + _execute(f"DELETE FROM {table}", {}) + return counts + + +class QueueFull(NodeLimitError): + pass + + +REQUEST_OPEN = ("queued", "assigned") +REQUEST_COLUMNS_PUBLIC = ( + "uid", + "status", + "result_status", + "result_bytes", + "result_headers", + "truncated", + "error", +) + + +def enqueue_request( + caller_kind: str, + caller_id: str, + url: str, + method: str, + adapter: str, + mode: str, +) -> str: + uid = _uid() + created = _execute( + "INSERT INTO node_requests (uid, caller_kind, caller_id, url, method, adapter, " + "mode, status, device_uid, result_status, result_bytes, result_body, " + "result_headers, truncated, error, created_at, assigned_at, finished_at) " + "SELECT :uid, :kind, :caller, :url, :method, :adapter, :mode, 'queued', '', 0, 0, " + "'', '', 0, '', :now, '', '' " + "WHERE (SELECT COUNT(*) FROM node_requests WHERE status IN ('queued','assigned')) " + "< :max_depth", + { + "uid": uid, + "kind": caller_kind, + "caller": caller_id, + "url": url, + "method": method, + "adapter": adapter, + "mode": mode, + "now": _now_iso(), + "max_depth": LIMITS.max_queue_depth, + }, + ) + if created != 1: + raise QueueFull("The node queue is full") + return uid + + +def get_request(request_uid: str) -> dict | None: + return get_table("node_requests").find_one(uid=request_uid) + + +def list_queued(limit: int) -> list[dict]: + return [ + dict(row) + for row in db.query( + "SELECT * FROM node_requests WHERE status = 'queued' " + "ORDER BY created_at, id LIMIT :n", + n=int(limit), + ) + ] + + +def count_open_requests() -> int: + return next( + iter( + db.query( + "SELECT COUNT(*) AS c FROM node_requests WHERE status IN ('queued','assigned')" + ) + ) + )["c"] + + +def claim_request(request_uid: str, device_uid: str) -> bool: + return ( + _execute( + "UPDATE node_requests SET status = 'assigned', device_uid = :device, " + "assigned_at = :now WHERE uid = :uid AND status = 'queued'", + {"device": device_uid, "now": _now_iso(), "uid": request_uid}, + ) + == 1 + ) + + +def finish_request( + request_uid: str, + *, + result_status: int, + result_headers: str, + result_body: str, + result_bytes: int, + truncated: bool, +) -> bool: + return ( + _execute( + "UPDATE node_requests SET status = 'done', result_status = :status, " + "result_headers = :headers, result_body = :body, result_bytes = :bytes, " + "truncated = :truncated, finished_at = :now " + "WHERE uid = :uid AND status = 'assigned'", + { + "status": int(result_status), + "headers": result_headers, + "body": result_body, + "bytes": int(result_bytes), + "truncated": 1 if truncated else 0, + "now": _now_iso(), + "uid": request_uid, + }, + ) + == 1 + ) + + +def set_request_transport(request_uid: str, transport: str) -> bool: + return ( + _execute( + "UPDATE node_requests SET mode = :mode WHERE uid = :uid AND status = 'assigned'", + {"mode": transport, "uid": request_uid}, + ) + == 1 + ) + + +def fail_request(request_uid: str, error: str, *, expired: bool = False) -> bool: + return ( + _execute( + "UPDATE node_requests SET status = :status, error = :error, finished_at = :now " + "WHERE uid = :uid AND status IN ('queued','assigned')", + { + "status": "expired" if expired else "failed", + "error": (error or "")[:EVENT_DETAIL_MAX], + "now": _now_iso(), + "uid": request_uid, + }, + ) + == 1 + ) + + +def expire_requests(moment: datetime | None = None) -> dict[str, int]: + now = moment or datetime.now(timezone.utc) + queued_cutoff = (now - timedelta(seconds=LIMITS.queue_max_wait_seconds)).isoformat() + assigned_cutoff = ( + now - timedelta(seconds=LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds) + ).isoformat() + finished = now.isoformat() + queued = _execute( + "UPDATE node_requests SET status = 'expired', error = 'no_node', finished_at = :now " + "WHERE status = 'queued' AND created_at < :cutoff", + {"now": finished, "cutoff": queued_cutoff}, + ) + assigned = _execute( + "UPDATE node_requests SET status = 'failed', error = 'timeout', finished_at = :now " + "WHERE status = 'assigned' AND assigned_at < :cutoff", + {"now": finished, "cutoff": assigned_cutoff}, + ) + return {"queued": queued, "assigned": assigned} + + +def reserve_caller_request( + caller_kind: str, caller_id: str, limit: int, *, moment: datetime | None = None +) -> bool: + if limit <= 0 or not caller_id: + return False + now = (moment or datetime.now(timezone.utc)).isoformat() + day = _day(moment) + with db: + db.executable.execute( + text( + "INSERT OR IGNORE INTO node_caller_daily (uid, caller_kind, caller_id, day, " + "requests, created_at, updated_at) VALUES (:uid, :kind, :caller, :day, 0, :now, :now)" + ), + {"uid": _uid(), "kind": caller_kind, "caller": caller_id, "day": day, "now": now}, + ) + counted = db.executable.execute( + text( + "UPDATE node_caller_daily SET requests = COALESCE(requests, 0) + 1, " + "updated_at = :now WHERE caller_kind = :kind AND caller_id = :caller " + "AND day = :day AND deleted_at IS NULL AND COALESCE(requests, 0) < :limit" + ), + {"now": now, "kind": caller_kind, "caller": caller_id, "day": day, "limit": int(limit)}, + ).rowcount + return counted == 1 + + +def caller_requests_today(caller_kind: str, caller_id: str) -> int: + row = get_table("node_caller_daily").find_one( + caller_kind=caller_kind, caller_id=caller_id, day=_day(), deleted_at=None + ) + return int((row or {}).get("requests") or 0) + + +def get_settings_by_users(user_uids) -> dict[str, dict]: + found: dict[str, dict] = {} + for batch in _batches(user_uids): + placeholders, params = _in_clause(batch) + for row in db.query( + f"SELECT * FROM node_settings WHERE user_uid IN ({placeholders}) " + "AND deleted_at IS NULL", + **params, + ): + found[row["user_uid"]] = normalise_settings(dict(row)) + return found + + +def donation_days(user_uid: str) -> int: + rows = db.query( + "SELECT COUNT(DISTINCT day) AS days FROM node_usage_daily " + "WHERE user_uid = :user AND COALESCE(requests_ok, 0) > 0 AND deleted_at IS NULL", + user=user_uid, + ) + return int(next(iter(rows))["days"] or 0) + + +def public_stats() -> dict: + row = next( + iter( + db.query( + "SELECT COALESCE(SUM(requests_ok), 0) AS requests, " + "COALESCE(SUM(bytes_total), 0) AS bytes, COUNT(*) AS donors " + "FROM node_donors WHERE deleted_at IS NULL AND COALESCE(requests_ok, 0) > 0" + ) + ) + ) + return { + "requests": int(row["requests"] or 0), + "bytes": int(row["bytes"] or 0), + "donors": int(row["donors"] or 0), + } + + +def top_donors(limit: int = 10) -> list[dict]: + limit = max(1, min(int(limit), 50)) + rows = db.query( + "SELECT d.user_uid AS user_uid, d.requests_ok AS requests_ok, d.bytes_total AS bytes_total " + "FROM node_donors d JOIN node_settings s ON s.user_uid = d.user_uid " + "WHERE d.deleted_at IS NULL AND s.deleted_at IS NULL AND COALESCE(s.public_display, 0) = 1 " + "AND COALESCE(d.requests_ok, 0) > 0 ORDER BY d.bytes_total DESC LIMIT :limit", + limit=limit, + ) + return [dict(row) for row in rows] + + +ADMIN_LIST_LIMIT = 100 +ADMIN_BREAKDOWN_LIMIT = 50 +DEVICE_BREAKDOWN_COLUMNS = ("status", "kind", "asn_class", "country") +REQUEST_STATUSES = ("queued", "assigned", "done", "failed", "expired") + + +def device_breakdown(column: str) -> dict[str, int]: + if column not in DEVICE_BREAKDOWN_COLUMNS: + raise ValueError("Unknown device breakdown column") + rows = db.query( + f"SELECT COALESCE(NULLIF({column}, ''), 'unknown') AS bucket, COUNT(*) AS total " + "FROM node_devices WHERE deleted_at IS NULL " + "GROUP BY bucket ORDER BY total DESC, bucket LIMIT :n", + n=ADMIN_BREAKDOWN_LIMIT, + ) + return {row["bucket"]: int(row["total"]) for row in rows} + + +def request_status_counts() -> dict[str, int]: + counts = {status: 0 for status in REQUEST_STATUSES} + for row in db.query("SELECT status, COUNT(*) AS total FROM node_requests GROUP BY status"): + counts[row["status"]] = int(row["total"]) + return counts + + +def usage_day_totals(moment: datetime | None = None) -> dict[str, int]: + row = next( + iter( + db.query( + "SELECT COALESCE(SUM(requests_ok), 0) AS requests_ok, " + "COALESCE(SUM(requests_failed), 0) AS requests_failed, " + "COALESCE(SUM(bytes_up), 0) AS bytes_up, " + "COALESCE(SUM(bytes_down), 0) AS bytes_down, " + "COALESCE(SUM(cache_hits_served), 0) AS cache_hits_served, " + "COALESCE(SUM(integrity_failures), 0) AS integrity_failures " + "FROM node_usage_daily WHERE day = :day AND deleted_at IS NULL", + day=_day(moment), + ) + ) + ) + return {key: int(value or 0) for key, value in dict(row).items()} + + +def recent_devices(limit: int = ADMIN_LIST_LIMIT) -> list[dict]: + limit = max(1, min(int(limit), ADMIN_LIST_LIMIT)) + return [ + dict(row) + for row in db.query( + "SELECT * FROM node_devices WHERE deleted_at IS NULL " + "ORDER BY COALESCE(NULLIF(last_seen_at, ''), created_at) DESC, id DESC LIMIT :n", + n=limit, + ) + ] diff --git a/devplacepy/database/schema.py b/devplacepy/database/schema.py index f3b92987..225aca2d 100644 --- a/devplacepy/database/schema.py +++ b/devplacepy/database/schema.py @@ -1854,6 +1854,8 @@ def init_db(): _ensure_moderation_tables() _ensure_skills_table() _ensure_nodes_tables() + _ensure_sync_tables() + _ensure_community_tables() for table in db.tables: _uid_index(db, table) @@ -2014,6 +2016,8 @@ def init_db(): "nodes_daily_quota_guest": str(NODES_SPEC.limits.default_daily_requests_guest), "nodes_cache_ttl_seconds": str(NODES_SPEC.limits.cache_ttl_seconds), "nodes_social_via_nodes": "0", + "sync_enabled": "0", + "sync_origins": "", } for key, value in operational_defaults.items(): existing = db["site_settings"].find_one(key=key) @@ -2285,11 +2289,24 @@ NODES_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = { ("result_status", 0), ("result_bytes", 0), ("result_body", ""), + ("result_headers", ""), + ("truncated", 0), ("error", ""), ("created_at", ""), ("assigned_at", ""), ("finished_at", ""), ), + "node_caller_daily": ( + ("uid", ""), + ("caller_kind", ""), + ("caller_id", ""), + ("day", ""), + ("requests", 0), + ("created_at", ""), + ("updated_at", ""), + ("deleted_at", ""), + ("deleted_by", ""), + ), "node_events": ( ("uid", ""), ("device_uid", ""), @@ -2343,6 +2360,14 @@ def _ensure_nodes_tables() -> None: where="status IN ('queued','assigned')", ) _index(db, "node_requests", "idx_node_requests_created", ["created_at"]) + _index( + db, + "node_caller_daily", + "idx_node_caller_day", + ["caller_kind", "caller_id", "day"], + unique=True, + ) + _index(db, "node_caller_daily", "idx_node_caller_day_only", ["day"]) _index(db, "node_events", "idx_node_events_created", ["created_at"]) _index(db, "node_events", "idx_node_events_device", ["device_uid", "created_at"]) _index( @@ -2354,6 +2379,120 @@ def _ensure_nodes_tables() -> None: ) +SYNC_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = { + "sync_links": ( + ("uid", ""), + ("origin", ""), + ("kind", ""), + ("origin_uid", ""), + ("local_uid", ""), + ("fingerprint", ""), + ("origin_updated_at", ""), + ("first_synced_at", ""), + ("synced_at", ""), + ), + "sync_identities": ( + ("uid", ""), + ("origin", ""), + ("source_name", ""), + ("source_key", ""), + ("our_name", ""), + ("our_key", ""), + ("style_fingerprint", ""), + ("style_samples", ""), + ("ai_generated", 0), + ("created_at", ""), + ), +} + + +def _ensure_sync_tables() -> None: + for table_name, columns in SYNC_COLUMNS.items(): + table = get_table(table_name) + for column, example in columns: + if not table.has_column(column): + table.create_column_by_example(column, example) + _index( + db, + "sync_links", + "idx_sync_links_origin", + ["origin", "kind", "origin_uid"], + unique=True, + ) + _index(db, "sync_links", "idx_sync_links_local", ["kind", "local_uid"]) + _index( + db, + "sync_identities", + "idx_sync_identities_source", + ["origin", "source_key"], + unique=True, + ) + _index(db, "sync_identities", "idx_sync_identities_our", ["our_key"], unique=True) + + +COMMUNITY_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = { + "communities": ( + ("uid", ""), + ("slug", ""), + ("name", ""), + ("origin", ""), + ("url", ""), + ("query", ""), + ("status", ""), + ("description", ""), + ("rules", ""), + ("language", ""), + ("favicon", ""), + ("user_count", 0), + ("schema", ""), + ("watermark", ""), + ("enabled", 1), + ("interval_seconds", 0), + ("next_run_at", ""), + ("last_run_at", ""), + ("last_status", ""), + ("last_error", ""), + ("run_count", 0), + ("failure_count", 0), + ("totals", ""), + ("learned_at", ""), + ("created_at", ""), + ("created_by", ""), + ("login_username", ""), + ("login_email", ""), + ("login_password", ""), + ("sync_options", ""), + ("claimed_at", ""), + ("deleted_at", ""), + ("deleted_by", ""), + ), + "community_runs": ( + ("uid", ""), + ("community_uid", ""), + ("started_at", ""), + ("finished_at", ""), + ("status", ""), + ("dry_run", 0), + ("trigger", ""), + ("collected", ""), + ("pushed", ""), + ("errors", ""), + ("log", ""), + ), +} + + +def _ensure_community_tables() -> None: + for table_name, columns in COMMUNITY_COLUMNS.items(): + table = get_table(table_name) + for column, example in columns: + if not table.has_column(column): + table.create_column_by_example(column, example) + _index(db, "communities", "idx_communities_slug", ["slug"], unique=True, where="deleted_at IS NULL") + _index(db, "communities", "idx_communities_due", ["status", "enabled", "next_run_at"]) + _index(db, "community_runs", "idx_community_runs_community", ["community_uid", "started_at"]) + + def _ensure_moderation_tables() -> None: for table_name, columns in MODERATION_COLUMNS.items(): table = get_table(table_name) diff --git a/devplacepy/database/soft_delete.py b/devplacepy/database/soft_delete.py index 268777f4..0226ff65 100644 --- a/devplacepy/database/soft_delete.py +++ b/devplacepy/database/soft_delete.py @@ -18,11 +18,13 @@ SOFT_DELETE_TABLES = [ "bookmarks", "notes", "skills", + "communities", "node_devices", "node_settings", "node_usage_daily", "node_donors", "node_flags", + "node_caller_daily", "follows", "poll_votes", "polls", diff --git a/devplacepy/database/sync.py b/devplacepy/database/sync.py new file mode 100644 index 00000000..d12ecf6d --- /dev/null +++ b/devplacepy/database/sync.py @@ -0,0 +1,131 @@ +# retoor + +from __future__ import annotations + +from sqlalchemy import text + +from devplacepy.utils.text import generate_uid + +from .core import _now_iso, db, get_table + + +def get_link(origin: str, kind: str, origin_uid: str) -> dict | None: + return get_table("sync_links").find_one( + origin=origin, kind=kind, origin_uid=origin_uid + ) + + +def get_links_by_origin_uids( + origin: str, kind: str, origin_uids: list[str] +) -> dict[str, dict]: + uids = [uid for uid in dict.fromkeys(origin_uids) if uid] + if not uids: + return {} + table = get_table("sync_links") + column = table.table.columns.origin_uid + rows = table.find(column.in_(uids), origin=origin, kind=kind) + return {row["origin_uid"]: row for row in rows} + + +def claim_link( + origin: str, + kind: str, + origin_uid: str, + local_uid: str, + fingerprint: str, + origin_updated_at: str, +) -> bool: + now = _now_iso() + with db: + result = db.executable.execute( + text( + "INSERT OR IGNORE INTO sync_links (uid, origin, kind, origin_uid, " + "local_uid, fingerprint, origin_updated_at, first_synced_at, synced_at) " + "VALUES (:uid, :origin, :kind, :origin_uid, :local_uid, :fingerprint, " + ":origin_updated_at, :now, :now)" + ), + { + "uid": generate_uid(), + "origin": origin, + "kind": kind, + "origin_uid": origin_uid, + "local_uid": local_uid, + "fingerprint": fingerprint, + "origin_updated_at": origin_updated_at, + "now": now, + }, + ) + return result.rowcount == 1 + + +def release_link(origin: str, kind: str, origin_uid: str, local_uid: str) -> None: + get_table("sync_links").delete( + origin=origin, kind=kind, origin_uid=origin_uid, local_uid=local_uid + ) + + +def advance_link(link_uid: str, fingerprint: str, origin_updated_at: str) -> None: + get_table("sync_links").update( + { + "uid": link_uid, + "fingerprint": fingerprint, + "origin_updated_at": origin_updated_at, + "synced_at": _now_iso(), + }, + ["uid"], + ) + + +def username_taken(username: str) -> bool: + return get_table("users").find_one(username=username) is not None + + +def count_links(origin: str) -> int: + return get_table("sync_links").count(origin=origin) + + +def identity_map(origin: str) -> dict[str, dict]: + return {row["source_key"]: row for row in get_table("sync_identities").find(origin=origin)} + + +def identity_by_source(origin: str, source_key: str) -> dict | None: + return get_table("sync_identities").find_one(origin=origin, source_key=source_key) + + +def identity_name_taken(our_key: str) -> bool: + return get_table("sync_identities").find_one(our_key=our_key) is not None + + +def claim_identity( + origin: str, + source_name: str, + source_key: str, + our_name: str, + style_fingerprint: str, + ai_generated: bool, +) -> bool: + with db: + result = db.executable.execute( + text( + "INSERT OR IGNORE INTO sync_identities (uid, origin, source_name, source_key, " + "our_name, our_key, style_fingerprint, style_samples, ai_generated, created_at) " + "VALUES (:uid, :origin, :source_name, :source_key, :our_name, :our_key, " + ":style_fingerprint, '[]', :ai_generated, :now)" + ), + { + "uid": generate_uid(), + "origin": origin, + "source_name": source_name, + "source_key": source_key, + "our_name": our_name, + "our_key": our_name.lower(), + "style_fingerprint": style_fingerprint, + "ai_generated": int(ai_generated), + "now": _now_iso(), + }, + ) + return result.rowcount == 1 + + +def save_style_samples(identity_uid: str, samples_json: str) -> None: + get_table("sync_identities").update({"uid": identity_uid, "style_samples": samples_json}, ["uid"]) diff --git a/devplacepy/docs_api/groups/__init__.py b/devplacepy/docs_api/groups/__init__.py index 9e32e502..59eeb873 100644 --- a/devplacepy/docs_api/groups/__init__.py +++ b/devplacepy/docs_api/groups/__init__.py @@ -25,6 +25,8 @@ from . import ( battles, skills, nodes, + sync, + community, ) ORDERED_GROUPS = [ @@ -52,4 +54,6 @@ ORDERED_GROUPS = [ battles.GROUP, skills.GROUP, nodes.GROUP, + sync.GROUP, + community.GROUP, ] diff --git a/devplacepy/docs_api/groups/admin.py b/devplacepy/docs_api/groups/admin.py index 8ff7d9d9..5b6fe67e 100644 --- a/devplacepy/docs_api/groups/admin.py +++ b/devplacepy/docs_api/groups/admin.py @@ -545,6 +545,23 @@ four ways to sign requests. "Allow signups.", ["1", "0"], ), + field( + "sync_enabled", + "form", + "enum", + False, + "0", + "Allow community sync.", + ["1", "0"], + ), + field( + "sync_origins", + "form", + "string", + False, + "other.example.org", + "Comma separated origins allowed to sync; empty allows any.", + ), field( "maintenance_mode", "form", diff --git a/devplacepy/docs_api/groups/community.py b/devplacepy/docs_api/groups/community.py new file mode 100644 index 00000000..8b7747ca --- /dev/null +++ b/devplacepy/docs_api/groups/community.py @@ -0,0 +1,156 @@ +# retoor + +from .._shared import endpoint, field + +SLUG = field("slug", "path", "string", True, "other-example-org", "Community slug, as listed by GET /admin/communities.") +ACCOUNT = [ + field("login_username", "form", "string", False, "bot", "Username of an account on the community, for a login-protected community."), + field("login_email", "form", "string", False, "bot@example.com", "Email of that account, when the community logs in by email."), + field("login_password", "form", "string", False, "", "Password of that account. Stored for the collectors and never returned by any endpoint."), +] + +ITEM_SAMPLE = { + "uid": "COMMUNITY_UID", + "slug": "other-example-org", + "name": "Other Example", + "origin": "other.example.org", + "url": "https://other.example.org/", + "status": "ready", + "enabled": True, + "interval_seconds": 86400, + "run_count": 3, + "failure_count": 0, + "last_run_at": "2026-10-14T10:00:00+00:00", + "next_run_at": "2026-10-15T10:00:00+00:00", + "has_login": False, +} + +GROUP = { + "slug": "community", + "title": "Community discovery", + "admin": True, + "intro": """ +# Community discovery + +Administrators add an external community by name or address. The system searches for it, detects +its feeds (RSS, Atom, JSON or HTML lists, with JavaScript rendering when a page needs it), learns +the structure once, records the community's description, rules and size, generates collectors and +then runs them on a schedule, pushing users, posts and comments through the +[community sync API](/docs/sync.html). Members never see these communities. + +- **Statuses.** `pending`, `learning`, `ready`, `login_required`, `failed` and `failing` (five failed + runs in a row: the community is disabled and needs a human). +- **Login-protected communities.** Give `login_username` or `login_email` plus `login_password`; the + system finds the login form (or, for a page that needs JavaScript, drives it in the browser), logs + in before discovery and before every run. The password is never returned. +- **Dry run.** `dry_run=true` on a run validates and logs everything and writes nothing. +- **Asynchronous.** Adding, relearning and running return at once; read the result from the detail + endpoint. +- **AI assistance.** For HTML pages whose structure the heuristics cannot read, the internal AI model + proposes selectors, and a proposal is only used when it matches the sampled pages. + +Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html). +""", + "endpoints": [ + endpoint( + id="community-list", + method="GET", + path="/admin/communities", + title="List communities", + summary="Every community with status, schedule and last run, plus the state of the sync switches.", + auth="admin", + interactive=True, + sample_response={"items": [ITEM_SAMPLE], "sync_enabled": True, "service_enabled": True, "service_dry_run": True}, + ), + endpoint( + id="community-add", + method="POST", + path="/admin/communities", + title="Add a community", + summary="Starts discovery in the background and redirects to the community.", + auth="admin", + encoding="form", + interactive=False, + params=[field("query", "form", "string", True, "https://forum.example.com", "Community name or address."), *ACCOUNT], + sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"}, + ), + endpoint( + id="community-detail", + method="GET", + path="/admin/communities/{slug}", + title="Community detail", + summary="Metadata, learned feeds, totals and recent runs with their logs.", + auth="admin", + interactive=True, + params=[SLUG], + sample_response={ + "community": ITEM_SAMPLE, + "rules": "Be kind.", + "totals": {"posts_created": 120}, + "feeds": [{"id": "feed1", "type": "json_api", "url": "https://other.example.org/api/posts", "comments": "inline", "pagination": "page"}], + "runs": [{"status": "ok", "dry_run": 0, "trigger": "service", "collected": {"users": 4, "posts": 20, "comments": 31}}], + "busy": False, + }, + ), + endpoint( + id="community-run", + method="POST", + path="/admin/communities/{slug}/run", + title="Run now", + summary="Collects and pushes in the background. A live run needs the sync_enabled setting.", + auth="admin", + encoding="form", + interactive=False, + params=[SLUG, field("dry_run", "form", "boolean", False, "false", "Validate and log, write nothing, keep the watermark.")], + sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"}, + ), + endpoint( + id="community-relearn", + method="POST", + path="/admin/communities/{slug}/relearn", + title="Relearn", + summary="Learns the structure again with the stored account. A failed relearn keeps the working schema.", + auth="admin", + encoding="form", + interactive=False, + params=[SLUG], + sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"}, + ), + endpoint( + id="community-update", + method="POST", + path="/admin/communities/{slug}/update", + title="Update schedule or account", + summary="Changes the interval, the enabled flag and the stored account. An empty password keeps the stored one.", + auth="admin", + encoding="form", + interactive=False, + params=[ + SLUG, + field("interval_seconds", "form", "integer", True, "86400", "Seconds between runs, 900 to 2592000."), + field("enabled", "form", "boolean", False, "true", "Whether the scheduled service runs this community."), + *ACCOUNT, + field("clear_login", "form", "boolean", False, "false", "Remove the stored account."), + field("alias", "form", "boolean", False, "false", "Store authors under permanent pseudonyms and rewrite every mention. Refused once the origin has records under original names."), + field("translate_to", "form", "string", False, "en", "Language code to translate collected posts and comments into; empty turns it off."), + field("sample_percent", "form", "integer", False, "100", "Percentage of new posts accepted, 1 to 100, chosen deterministically per post."), + field("min_comments", "form", "integer", False, "0", "Minimum comments collected in the run for a new post to be accepted."), + field("topics", "form", "string", False, "devlog, rant", "Comma separated topic allow-list; empty accepts all."), + field("jitter_percent", "form", "integer", False, "0", "Randomise the next run time by up to this percentage of the interval, 0 to 50."), + ], + sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"}, + ), + endpoint( + id="community-delete", + method="POST", + path="/admin/communities/{slug}/delete", + title="Delete a community", + summary="Moves the community to the trash and stops its collectors. Content already synchronised stays.", + auth="admin", + encoding="form", + interactive=False, + params=[SLUG], + sample_response={"ok": True, "redirect": "/admin/communities"}, + ), + ], +} diff --git a/devplacepy/docs_api/groups/nodes.py b/devplacepy/docs_api/groups/nodes.py index fdad071b..87edac30 100644 --- a/devplacepy/docs_api/groups/nodes.py +++ b/devplacepy/docs_api/groups/nodes.py @@ -154,5 +154,265 @@ page. Every endpoint follows the shared [Conventions & Errors](/docs/conventions "`403` with a `reason` when the device is suspended or its owner no longer passes the consent gate.", ], ), + endpoint( + id="nodes-me", + method="GET", + path="/nodes/me", + title="Your donation status", + summary="Whether you may donate right now (and the reason when not), your settings, your devices and what you have donated.", + auth="user", + interactive=True, + sample_response={ + "consent": True, + "can_donate": True, + "reason": "", + "message": "", + "settings": {"enabled": True, "bandwidth_mb_day": 500, "max_concurrent": 2}, + "platforms": ["reddit"], + "devices": [DEVICE_SAMPLE], + "max_devices": 5, + "donated": {"lifetime_requests": 12, "lifetime_bytes": 482113, "today_requests": 3}, + "limits": {"max_donor_mb_per_day": 20480, "max_concurrent_per_node": 2, "default_donor_mb_per_day": 500}, + }, + ), + endpoint( + id="nodes-settings", + method="POST", + path="/nodes/settings", + title="Change your donation settings", + summary="Update bandwidth, concurrency, active hours, platforms and display. Unsent fields keep their value.", + auth="user", + ajax=True, + encoding="form", + params=[ + field("enabled", "form", "boolean", False, "true", "Switch donation on or off. Switching on needs the `bandwidth_sharing` consent and is refused when sent by the Devii assistant."), + field("bandwidth_mb_day", "form", "integer", False, "500", "Daily bandwidth cap in megabytes."), + field("max_concurrent", "form", "integer", False, "2", "Simultaneous requests."), + field("active_from", "form", "string", False, "08:00", "Start of the active window, HH:MM."), + field("active_to", "form", "string", False, "22:00", "End of the active window, HH:MM."), + field("timezone", "form", "string", False, "Europe/Amsterdam", "IANA timezone of the window."), + field("allowed_platforms", "form", "string", False, "reddit", "Comma separated platforms you are willing to serve."), + field("pause_on_battery", "form", "boolean", False, "true", "Stop donating on battery power."), + field("public_display", "form", "boolean", False, "false", "Show your name on the donor leaderboard."), + ], + sample_response={"consent": True, "can_donate": True, "settings": {"enabled": True}}, + notes=["`403` with `reason: no_consent` when switching on without the consent."], + ), + endpoint( + id="nodes-stats", + method="GET", + path="/nodes/stats", + title="Community donation totals", + summary="Aggregate numbers only: requests relayed, bytes relayed and number of donors. No per-member data.", + auth="public", + interactive=True, + sample_response={"requests": 1200, "bytes": 90311234, "donors": 7}, + ), + endpoint( + id="nodes-leaderboard", + method="GET", + path="/nodes/leaderboard", + title="Donor leaderboard", + summary="The top donors who chose to be shown. Members are not listed unless they switched on public display.", + auth="public", + interactive=True, + sample_response={"donors": [{"username": "alice", "requests": 400, "bytes": 31200000}]}, + ), + endpoint( + id="nodes-social-search", + method="GET", + path="/nodes/social/search", + title="Social search", + summary="Search public developer discussion on Hacker News, Stack Overflow, GitHub, dev.to, Mastodon and, when donors are online, Reddit.", + auth="user", + interactive=True, + params=[ + field("q", "query", "string", True, "sqlite wal", "What to search for, up to 200 characters."), + field("platforms", "query", "string", False, "hackernews,github", "Comma separated subset. Empty means all."), + field("limit", "query", "integer", False, "8", "Hits per platform, 1 to 20."), + ], + sample_response={ + "query": "sqlite wal", + "platforms": ["hackernews", "reddit"], + "hits": [ + { + "platform": "hackernews", + "title": "SQLite WAL mode explained", + "url": "https://news.ycombinator.com/item?id=1", + "snippet": "", + "author": "someone", + "score": 120, + "comments": 33, + "created_at": "2026-01-01T00:00:00Z", + "via": "direct", + } + ], + "unavailable": {"reddit": "no_nodes"}, + }, + notes=[ + "Platforms that could not answer are listed under `unavailable` with a reason code; results are never invented for them.", + "Counted against a daily per-member quota (`429` when reached). Bluesky and Lobsters are not covered because they block unauthenticated server access.", + ], + ), + endpoint( + id="nodes-internal-fetch", + method="POST", + path="/nodes/internal/fetch", + title="Fetch a page through the donor network", + summary="Fetch one page from an allowlisted site through a donor node. Meant for agents and services that hold an API key.", + auth="user", + encoding="form", + params=[ + field("url", "form", "string", True, "https://www.reddit.com/r/programming/.json", "Absolute URL on an allowlisted host."), + field("cache", "form", "boolean", False, "true", "Allow a cached copy."), + field("timeout", "form", "integer", False, "15", "Seconds to wait, 1 to 60."), + ], + sample_response={ + "url": "https://www.reddit.com/r/programming/.json", + "status": 200, + "via": "fetch", + "truncated": False, + "elapsed_ms": 812, + "content_type": "application/json", + "text": "{...}", + }, + notes=[ + "`403` with `reason: host_not_allowed` for any host that is not on the allowlist, `503` with `reason: no_nodes` when no donor is online, `429` when the daily node quota is reached.", + "`via` says how the body arrived: `fetch` (relayed by a donor browser, treat as unverified) or `cache`.", + ], + ), + endpoint( + id="nodes-client-version", + method="GET", + path="/nodes/client-version", + title="Client versions and checksums", + summary="The protocol versions this server speaks and every downloadable client with its SHA-256.", + auth="public", + interactive=True, + sample_response={ + "version": "0.1.0", + "protocol": 1, + "min_protocol": 1, + "artifacts": [{"name": "devplace-node-linux-amd64", "size": 6291456, "sha256": "ab12..."}], + }, + ), + endpoint( + id="nodes-download", + method="GET", + path="/nodes/download/{artifact}", + title="Download a client", + summary="Download one client artifact listed by the client version endpoint. The SHA-256 is also sent as `X-Content-SHA256`.", + auth="public", + params=[ + field("artifact", "path", "string", True, "devplace-node-linux-amd64", "Artifact name."), + ], + sample_response="(binary file)", + ), + endpoint( + id="admin-nodes", + method="GET", + path="/admin/nodes", + title="Node network overview", + summary="Pool, queue, device breakdowns, today's totals, open flags, recent devices, settings and the kill switch.", + auth="admin", + interactive=True, + sample_response={ + "enabled": True, + "pool": {"online": True, "connected": 12, "capacity": 2000, "accepted": 40, "kicked": 3, "jobs_done": 310, "jobs_failed": 4, "open_circuits": 0, "heartbeat": "2026-01-01T00:00:00+00:00"}, + "queue": {"depth": 2, "max_depth": 500, "queued": 1, "assigned": 1, "done": 30, "failed": 1, "expired": 0}, + "devices_by_status": {"active": 14, "suspended": 1}, + "devices_by_kind": {"native": 10, "extension": 5}, + "devices_by_asn_class": {"unknown": 15}, + "devices_by_country": {"unknown": 15}, + "usage_today": {"requests_ok": 120, "requests_failed": 3, "bytes_up": 40000, "bytes_down": 9000000, "cache_hits_served": 0, "integrity_failures": 0}, + "flags": [{"uid": "FLAG_UID", "kind": "refusal_mismatch", "detail": "", "status": "open", "device_uid": "DEVICE_UID", "device_label": "Home laptop", "user_uid": "USER_UID", "owner_username": "alice", "created_at": "2026-01-01T00:00:00+00:00"}], + "devices": [dict(DEVICE_SAMPLE, owner_uid="USER_UID", owner_username="alice", suspend_reason="")], + "settings": [{"key": "nodes_max_total", "label": "Maximum connected nodes", "kind": "int", "value": "2000", "default": "2000", "ceiling": 2000}], + "limits": {"max_devices_per_user": 5, "max_connected_nodes": 2000}, + }, + notes=[ + "Per-destination circuit state lives in the memory of the worker that holds the sockets and is not readable from other workers; only the number of open circuits is reported.", + "`usage_today` is the current UTC day. Lists are capped at 100 rows.", + ], + ), + endpoint( + id="admin-nodes-killswitch", + method="POST", + path="/admin/nodes/killswitch", + title="Switch community donation on or off", + summary="The global kill switch. Switching it off disconnects every node within one reconcile tick.", + auth="admin", + encoding="form", + destructive=True, + params=[ + field("enabled", "form", "boolean", False, "false", "True to switch on, false to switch off. Omit to flip the current state."), + ], + sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"enabled": False}}, + notes=["Audited as `node.killswitch`."], + ), + endpoint( + id="admin-nodes-settings", + method="POST", + path="/admin/nodes/settings", + title="Update node network settings", + summary="Change the caps and quotas. Only the fields you send are changed; values above the ceiling are clamped to it.", + auth="admin", + encoding="form", + params=[ + field("nodes_max_total", "form", "integer", False, "2000", "Maximum connected nodes, 0 to 2000."), + field("nodes_daily_quota_user", "form", "integer", False, "200", "Daily node requests per member, 0 to 10000."), + field("nodes_daily_quota_guest", "form", "integer", False, "0", "Daily node requests per guest, 0 to 1000."), + field("nodes_cache_ttl_seconds", "form", "integer", False, "600", "Result cache lifetime in seconds, 0 to 600."), + field("nodes_daily_social_user", "form", "integer", False, "300", "Daily social searches per member, 0 to 10000."), + field("nodes_social_via_nodes", "form", "string", False, "0", "`1` routes social search through nodes, `0` keeps it direct."), + ], + sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"changed": ["nodes_max_total"], "clamped": []}}, + notes=["Audited as `node.admin.settings` with the old and new values."], + ), + endpoint( + id="admin-nodes-suspend", + method="POST", + path="/admin/nodes/devices/{device_uid}/suspend", + title="Suspend a donor device", + summary="Suspend one device for 1 to 10080 minutes. The device is disconnected on the next tick and cannot reconnect until the period ends.", + auth="admin", + encoding="form", + destructive=True, + params=[ + field("device_uid", "path", "string", True, "DEVICE_UID", "Device uid."), + field("minutes", "form", "integer", True, "60", "Suspension length in minutes, 1 to 10080."), + field("reason", "form", "string", False, "Repeated refusals", "Reason, up to 500 characters."), + ], + sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "DEVICE_UID", "minutes": 60}}, + notes=["Refused (audited as `denied`) when the device owner is a more senior administrator. Audited as `node.suspend`."], + ), + endpoint( + id="admin-nodes-unsuspend", + method="POST", + path="/admin/nodes/devices/{device_uid}/unsuspend", + title="Lift a device suspension", + summary="Return a suspended device to active.", + auth="admin", + encoding="form", + params=[ + field("device_uid", "path", "string", True, "DEVICE_UID", "Device uid."), + ], + sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "DEVICE_UID"}}, + notes=["`409` when the device is not suspended. Audited as `node.unsuspend`."], + ), + endpoint( + id="admin-nodes-flag-resolve", + method="POST", + path="/admin/nodes/flags/{flag_uid}/resolve", + title="Resolve a node flag", + summary="Mark an open integrity or refusal flag as handled.", + auth="admin", + encoding="form", + params=[ + field("flag_uid", "path", "string", True, "FLAG_UID", "Flag uid."), + ], + sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "FLAG_UID"}}, + notes=["`409` when the flag is already resolved. Audited as `node.flag.resolve`."], + ), ], } diff --git a/devplacepy/docs_api/groups/sync.py b/devplacepy/docs_api/groups/sync.py new file mode 100644 index 00000000..c498f82e --- /dev/null +++ b/devplacepy/docs_api/groups/sync.py @@ -0,0 +1,119 @@ +# retoor + +from .._shared import endpoint, field + +ENVELOPE_PARAMS = [ + field("origin", "body", "string", True, "other.example.org", "Stable id of the sending community, 2-63 characters of a-z, 0-9, dots and hyphens. Lower-cased by the server."), + field("records", "body", "array", True, "[]", "1-50 records of the kind this endpoint takes. Each record is validated on its own."), + field("dry_run", "body", "boolean", False, "false", "Validate, normalize and report what would happen without writing anything."), +] + +RESULT_SAMPLE = { + "origin": "other.example.org", + "kind": "posts", + "dry_run": False, + "total": 2, + "counts": {"created": 1, "updated": 0, "unchanged": 0, "stale": 0, "rejected": 1, "failed": 0}, + "results": [ + {"index": 0, "origin_uid": "p-1", "status": "created", "reason": "", "local_uid": "LOCAL_UID", "errors": [], "ignored": ["stars"]}, + {"index": 1, "origin_uid": "p-2", "status": "rejected", "reason": "unknown_author", "local_uid": "", "errors": ["author u-9 is not synced yet, send users first"], "ignored": []}, + ], +} + +GROUP = { + "slug": "sync", + "title": "Community sync", + "admin": True, + "intro": """ +# Community sync + +Copy users, posts and comments from another community into this one. The API is idempotent +and safe to repeat: every record carries the id it has in the **origin** community +(`origin_uid`), the server remembers which local row that id maps to, and a repeated record is +an update or a no-op, never a duplicate. + +- **Validated and normalized here.** The sender posts raw data; the server fixes what can be + fixed (usernames, whitespace, control characters, unknown topics, timestamps) and rejects what + cannot, record by record. One bad record never fails the batch. +- **Counters are not imported.** Votes, stars, XP and levels cannot be verified across + communities, so they are ignored and listed under `ignored` in the result. +- **No side effects.** Synced content sends no notifications, awards no XP and runs no AI + correction. Content that matches the content filter is rejected. +- **Order matters.** Send users first, then posts, then comments (parents before replies). +- **Off by default.** An administrator enables it at `/admin/settings` (`sync_enabled`) and may + restrict senders with `sync_origins` (comma separated; empty allows any origin). +- **JSON or form.** The body is JSON. A form-encoded body is accepted too, with `records` as a + JSON array string; Devii uses that form through the `sync_users`, `sync_posts` and + `sync_comments` tools. + +The accepted fields and every limit are published at `/sync/spec.json`. Every endpoint follows +the shared [Conventions & Errors](/docs/conventions.html). +""", + "endpoints": [ + endpoint( + id="sync-spec", + method="GET", + path="/sync/spec.json", + title="Sync specification", + summary="Limits, statuses, refusal reasons and the JSON schema of each record kind.", + auth="public", + interactive=True, + sample_response={ + "limits": {"max_batch_records": 50}, + "kinds": ["users", "posts", "comments"], + "statuses": ["created", "updated", "unchanged", "stale", "rejected", "failed"], + "reasons": ["invalid", "unknown_author"], + "records": {"users": {"title": "UserRecord", "type": "object"}}, + }, + ), + endpoint( + id="sync-users", + method="POST", + path="/sync/users", + title="Upsert users", + summary="Create or update synced accounts. A synced account cannot sign in; it only owns the content that was synced for it.", + auth="admin", + encoding="json", + interactive=False, + params=ENVELOPE_PARAMS, + sample_response=RESULT_SAMPLE, + notes=[ + "Record fields: `origin_uid`, `username`, `created_at`, optional `updated_at`, `bio`, `location`, `git_link`, `website`, `avatar_seed`, `deleted`.", + "A username that is invalid is normalized to letters, digits, `_` and `-`. A name already taken by a different account gets the origin as a suffix, so the result may differ from the one sent.", + "`deleted: true` deactivates the account. Email, password, role, XP and level are never accepted.", + ], + ), + endpoint( + id="sync-posts", + method="POST", + path="/sync/posts", + title="Upsert posts", + summary="Create or update posts. The author must already be synced.", + auth="admin", + encoding="json", + interactive=False, + params=ENVELOPE_PARAMS, + sample_response=RESULT_SAMPLE, + notes=[ + "Record fields: `origin_uid`, `author_uid` (the author's `origin_uid`), `content`, `created_at`, optional `title`, `topic`, `updated_at`, `deleted`.", + "An unknown `topic` becomes `random`. Content shorter than 10 characters is rejected.", + ], + ), + endpoint( + id="sync-comments", + method="POST", + path="/sync/comments", + title="Upsert comments", + summary="Create or update comments on synced posts. The author, the post and any parent comment must already be synced.", + auth="admin", + encoding="json", + interactive=False, + params=ENVELOPE_PARAMS, + sample_response=RESULT_SAMPLE, + notes=[ + "Record fields: `origin_uid`, `author_uid`, `post_uid` (the post's `origin_uid`), `content`, `created_at`, optional `parent_uid`, `updated_at`, `deleted`.", + "A parent that belongs to a different post is rejected with `parent_mismatch`.", + ], + ), + ], +} diff --git a/devplacepy/main.py b/devplacepy/main.py index 01584baa..0779ec3f 100644 --- a/devplacepy/main.py +++ b/devplacepy/main.py @@ -53,6 +53,7 @@ from devplacepy.routers import ( auth, battles, nodes, + sync, feed, posts, topics, @@ -128,6 +129,7 @@ from devplacepy.services.audit import AuditService from devplacepy.services.moderation.service import ModerationService from devplacepy.services.moderation.screening import ContentRefused from devplacepy.services.acceptance.service import AcceptanceService +from devplacepy.services.community.service import CommunityService from devplacepy.services.nodes.service import NodesService from devplacepy.services.audit import record as audit from devplacepy.services.push import PushService @@ -298,6 +300,7 @@ async def lifespan(app: FastAPI): service_manager.register(AuditService()) service_manager.register(ModerationService()) service_manager.register(AcceptanceService()) + service_manager.register(CommunityService()) service_manager.register(NodesService()) service_manager.register(PushService()) service_manager.register(TelegramService()) @@ -529,6 +532,7 @@ app.include_router(game.router, prefix="/game") app.include_router(quizzes.router, prefix="/quizzes") app.include_router(battles.router, prefix="/battles") app.include_router(nodes.router, prefix="/nodes") +app.include_router(sync.router, prefix="/sync") app.include_router(workspaces.router, prefix="/workspaces") diff --git a/devplacepy/models.py b/devplacepy/models.py index ee17e623..777e263f 100644 --- a/devplacepy/models.py +++ b/devplacepy/models.py @@ -328,6 +328,37 @@ class ProjectEditForm(BaseModel): return normalize_website_url(value) +class CommunityAddForm(BaseModel): + query: str = Field(min_length=2, max_length=200) + login_username: str = Field(default="", max_length=200) + login_email: str = Field(default="", max_length=254) + login_password: str = Field(default="", max_length=500) + + @field_validator("query") + @classmethod + def trimmed(cls, value): + return value.strip() + + +class CommunityUpdateForm(BaseModel): + interval_seconds: int = Field(ge=900, le=2592000) + enabled: bool = False + login_username: str = Field(default="", max_length=200) + login_email: str = Field(default="", max_length=254) + login_password: str = Field(default="", max_length=500) + clear_login: bool = False + alias: Optional[bool] = None + translate_to: Optional[str] = Field(default=None, max_length=12) + sample_percent: Optional[int] = Field(default=None, ge=1, le=100) + min_comments: Optional[int] = Field(default=None, ge=0, le=1000) + topics: Optional[str] = Field(default=None, max_length=500) + jitter_percent: Optional[int] = Field(default=None, ge=0, le=50) + + +class CommunityRunForm(BaseModel): + dry_run: bool = False + + class BackupRunForm(BaseModel): target: Literal["database", "uploads", "keys", "full"] = "full" @@ -721,6 +752,16 @@ class AdminSettingsForm(BaseModel): modifier_model: str = Field(default="", max_length=120) quiz_grading_model: str = Field(default="", max_length=120) extra_head: str = Field(default="", max_length=50000) + sync_enabled: str = Field(default="", max_length=1) + sync_origins: str = Field(default="", max_length=1000) + + @field_validator("sync_origins") + @classmethod + def validate_sync_origins(cls, value): + from devplacepy.services.sync.normalize import normalize_origin + + items = [item for item in value.replace("\n", ",").split(",") if item.strip()] + return ",".join(dict.fromkeys(normalize_origin(item) for item in items)) @field_validator("moderation_filter_mode") @classmethod @@ -1124,6 +1165,59 @@ class BanForm(BaseModel): reason: str = Field(default="", max_length=200) +class NodeSettingsForm(BaseModel): + enabled: Optional[bool] = None + bandwidth_mb_day: Optional[int] = Field(default=None, ge=1, le=20 * 1024) + max_concurrent: Optional[int] = Field(default=None, ge=1, le=16) + active_from: Optional[str] = Field(default=None, max_length=5) + active_to: Optional[str] = Field(default=None, max_length=5) + timezone: Optional[str] = Field(default=None, max_length=64) + allowed_platforms: Optional[list[str]] = Field(default=None, max_length=16) + pause_on_battery: Optional[bool] = None + public_display: Optional[bool] = None + + @field_validator("allowed_platforms", mode="before") + @classmethod + def split_platforms(cls, value): + if isinstance(value, str): + return [item.strip() for item in value.split(",") if item.strip()] + return value + + +class NodeFetchForm(BaseModel): + url: str = Field(max_length=2048) + cache: bool = True + timeout: int = Field(default=15, ge=1, le=60) + + +NODE_SUSPEND_MIN_MINUTES = 1 +NODE_SUSPEND_MAX_MINUTES = 10080 +NODE_SETTING_FORM_MAX = 1_000_000 + + +class AdminNodeKillswitchForm(BaseModel): + enabled: Optional[bool] = None + + +class AdminNodeSettingsForm(BaseModel): + nodes_max_total: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX) + nodes_daily_quota_user: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX) + nodes_daily_quota_guest: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX) + nodes_cache_ttl_seconds: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX) + nodes_social_via_nodes: Optional[Literal["0", "1"]] = None + nodes_daily_social_user: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX) + + +class AdminNodeSuspendForm(BaseModel): + minutes: int = Field(ge=NODE_SUSPEND_MIN_MINUTES, le=NODE_SUSPEND_MAX_MINUTES) + reason: str = Field(default="", max_length=500) + + @field_validator("reason") + @classmethod + def trimmed(cls, value): + return value.strip() + + class NodeDeviceForm(BaseModel): label: str = Field(default="", max_length=60) kind: str = Field(default="native", max_length=20) diff --git a/devplacepy/routers/CLAUDE.md b/devplacepy/routers/CLAUDE.md index e313e776..41df9718 100644 --- a/devplacepy/routers/CLAUDE.md +++ b/devplacepy/routers/CLAUDE.md @@ -53,7 +53,8 @@ Prefixes are wired in `main.py`: | (none) | push.py - push + PWA: `GET /push.json` (VAPID public key + the providers accepting registrations; `apns` includes `environment` when active), `POST /push.json` (register with any active provider; a body without `provider` is a `webpush` body; an APNs body is `token` plus optional `client_id` for device-stable upsert; created/revived rows are probed and the JSON may include `delivered`/`error`; an unknown, disabled or unconfigured provider is a 400), `DELETE /push.json` (unregister exactly the one registration named by `endpoint`/`token`/`client_id`, same identity priority as registration; idempotent, always 200 `{unregistered}`; wired into `PushManager.js`'s logout-link interceptor so a webpush subscription is dropped before the browser navigates to `/auth/logout`), `GET /service-worker.js`, `GET /manifest.json`. Provider protocol and delivery live in `devplacepy/push/` - see `devplacepy/push/CLAUDE.md` | | (none) | docs.py (docs/ package) - the documentation site (prose pages + API reference). See `routers/docs/CLAUDE.md` for the deep detail on this tree (`DOCS_PAGES`, audience tiers, prose rendering pipeline) | | `/skills` | skills.py - per-user Agent Skills: `GET ""` (list), `GET /{name}` (detail), `POST /install` (fetch a `SKILL.md` from a URL, update in place), `POST /{name}/enabled`, `POST /{name}/delete` (owner, or an administrator naming `owner`). Private to the owner, `noindex`. See `devplacepy/services/skills/CLAUDE.md` | -| `/nodes` | nodes/ package - community node network. `spec.py` (`GET /nodes/spec.json`, public, `SPEC.describe()`), `devices.py` (`GET/POST /nodes/devices`, `POST /nodes/devices/{uid}/rotate|delete`, `require_user_api`, secret returned once, `no-store`), `ticket.py` (`POST /nodes/ticket`, authenticated by `Authorization: Bearer dpn_...`, never a session), `ws.py` (`WS /nodes/ws?ticket=`, lock-owner gated, `4013` retry). Needs its own nginx upgrade block. See `devplacepy/services/nodes/CLAUDE.md` | +| `/nodes` | nodes/ package - community node network. `spec.py` (`GET /nodes/spec.json`, public, `SPEC.describe()`), `devices.py` (`GET/POST /nodes/devices`, `POST /nodes/devices/{uid}/rotate|delete`, `require_user_api`, secret returned once, `no-store`), `ticket.py` (`POST /nodes/ticket`, authenticated by `Authorization: Bearer dpn_...`, never a session), `ws.py` (`WS /nodes/ws?ticket=`, lock-owner gated, `4013` retry). Needs its own nginx upgrade block. The administrator side is `admin/nodes.py` (`GET /admin/nodes`, `POST /admin/nodes/{killswitch,settings,devices/{uid}/suspend,devices/{uid}/unsuspend,flags/{uid}/resolve}`). See `devplacepy/services/nodes/CLAUDE.md` | +| `/sync` | sync/ package - community sync. `spec.py` (`GET /sync/spec.json`, public), `users.py`, `posts.py`, `comments.py` (`POST /sync/{kind}`, administrator API key, batch of up to 50 records, per-record results, `dry_run`). Shared auth, kill switch (`sync_enabled`), origin allowlist (`sync_origins`) and audit in `_shared.py`. See `devplacepy/services/sync/CLAUDE.md` | | `/pubsub` | pubsub.py - **database-free** publish/subscribe bus. `WS /pubsub/ws` (subscribe/unsubscribe/publish frames, `foo.*` wildcards), `POST /pubsub/publish` + `GET /pubsub/topics` (admin/internal). Lock-owner-gated WS (`4013` retry) so subscribers converge on one worker; topic authz in `services/pubsub/policy.py` (`user.{uid}.*` private, `public.*` shared, admin/internal anywhere, guests opt-in). In-process `services.pubsub.publish(topic, data)` for backends; frontend `app.pubsub`. See `devplacepy/services/pubsub/CLAUDE.md` | | (none) | seo.py - `/robots.txt`, `/sitemap.xml` | diff --git a/devplacepy/routers/admin/__init__.py b/devplacepy/routers/admin/__init__.py index 94290618..fe196c14 100644 --- a/devplacepy/routers/admin/__init__.py +++ b/devplacepy/routers/admin/__init__.py @@ -7,6 +7,7 @@ from devplacepy.routers.admin import ( auditlog, backups, bots, + communities, containers, workspaces, devii_tasks, @@ -16,6 +17,7 @@ from devplacepy.routers.admin import ( media, moderation, news, + nodes, notifications, services, settings, @@ -40,6 +42,8 @@ router.include_router(issues.router) router.include_router(auditlog.router) router.include_router(backups.router) router.include_router(bots.router) +router.include_router(communities.router) +router.include_router(nodes.router) router.include_router(gateway_configs.router) router.include_router(devii_tasks.router) router.include_router(game.router) diff --git a/devplacepy/routers/admin/communities.py b/devplacepy/routers/admin/communities.py new file mode 100644 index 00000000..d4f00cf3 --- /dev/null +++ b/devplacepy/routers/admin/communities.py @@ -0,0 +1,263 @@ +# retoor + +import asyncio +import logging +from typing import Annotated + +from fastapi import APIRouter, Form, Request +from fastapi.responses import HTMLResponse + +from devplacepy.database import get_setting +from devplacepy.database.sync import count_links, identity_map +from devplacepy.responses import action_result, json_error, respond +from devplacepy.schemas import AdminCommunitiesOut, AdminCommunityOut +from devplacepy.models import CommunityAddForm, CommunityRunForm, CommunityUpdateForm +from devplacepy.seo import base_seo_context, site_url, website_schema +from devplacepy.services.audit import record as audit +from devplacepy.services.community.discoverer import slug_for_query +from devplacepy.services.community.files import files +from devplacepy.services.community.runner import TRIGGER_ADMIN, CommunityRunner +from devplacepy.services.community.service import DRY_RUN_KEY +from devplacepy.services.community.spec import STATUS_LEARNING +from devplacepy.services.community.store import iso, now, store +from devplacepy.services.sync.gate import sync_enabled +from devplacepy.services.sync.options import SyncOptions +from devplacepy.utils import not_found, require_admin + +logger = logging.getLogger(__name__) +router = APIRouter() + +background_jobs: set[asyncio.Task] = set() +LIST_URL = "/admin/communities" + + +def spawn(coroutine) -> None: + task = asyncio.create_task(coroutine) + background_jobs.add(task) + task.add_done_callback(background_jobs.discard) + + +def service_flags() -> dict: + return { + "service_enabled": (get_setting("service_community_enabled", "0") or "0") == "1", + "service_dry_run": (get_setting(DRY_RUN_KEY, "1") or "1") == "1", + } + + +def public(row: dict) -> dict: + view = {key: value for key, value in row.items() if key != "login_password"} + view["has_login"] = bool(row.get("login_password")) + return view + + +def require_community(slug: str) -> dict: + row = store.get(slug) + if row is None: + raise not_found("Unknown community") + return row + + +def credentials_of(data) -> dict: + return {"username": data.login_username.strip(), "email": data.login_email.strip(), "password": data.login_password} + + +def seo(request: Request, title: str, path: str, crumb: str): + base = site_url(request) + return base_seo_context( + request, + title=f"{title} - Admin", + description="External communities synchronised into this instance.", + breadcrumbs=[ + {"name": "Home", "url": "/feed"}, + {"name": "Admin", "url": "/admin"}, + {"name": crumb, "url": path}, + ], + schemas=[website_schema(base)], + robots="noindex,nofollow", + ) + + +def feed_summary(feed: dict) -> dict: + return { + "id": feed["id"], + "type": feed["type"], + "url": feed["url"], + "list_path": feed.get("list_path"), + "item_selector": feed.get("item_selector"), + "post_fields": feed["post"]["fields"], + "author_fields": (feed.get("author") or {}).get("fields", {}), + "comments": (feed.get("comments") or {}).get("mode", "none"), + "comments_reason": (feed.get("comments") or {}).get("reason", ""), + "pagination": (feed.get("pagination") or {}).get("style", "none"), + } + + +@router.get("/communities", response_class=HTMLResponse) +async def admin_communities(request: Request): + admin = require_admin(request) + return respond( + request, + "admin_communities.html", + { + **seo(request, "Communities", LIST_URL, "Communities"), + "request": request, + "user": admin, + "items": [public(row) for row in store.list()], + "sync_enabled": sync_enabled(), + **service_flags(), + "admin_section": "communities", + }, + model=AdminCommunitiesOut, + ) + + +@router.post("/communities") +async def admin_community_add(request: Request, data: Annotated[CommunityAddForm, Form()]): + admin = require_admin(request) + slug = slug_for_query(data.query) + if not slug: + return json_error(400, "Enter a community name or address") + existing = store.get(slug) + if existing is None: + secrets = credentials_of(data) + existing = store.create( + slug, + data.query, + admin["uid"], + name=data.query, + status=STATUS_LEARNING, + claimed_at=iso(now()), + login_username=secrets["username"], + login_email=secrets["email"], + login_password=secrets["password"], + ) + spawn(CommunityRunner().discover(data.query, admin["uid"], relearn_slug=slug)) + audit.record( + request, + "admin.community.add", + user=admin, + target_type="community", + target_uid=existing["uid"], + target_label=slug, + metadata={"query": data.query}, + summary=f"{admin['username']} added community {data.query!r}", + ) + return action_result(request, f"{LIST_URL}/{existing['slug']}") + + +@router.get("/communities/{slug}", response_class=HTMLResponse) +async def admin_community(request: Request, slug: str): + admin = require_admin(request) + row = require_community(slug) + source = files.read(row["slug"]) + return respond( + request, + "admin_community.html", + { + **seo(request, row["name"] or slug, f"{LIST_URL}/{slug}", "Communities"), + "request": request, + "user": admin, + "community": public(row), + "rules": row.get("rules") or "", + "favicon": row.get("favicon") or "", + "totals": row.get("totals") or {}, + "feeds": [feed_summary(feed) for feed in (row.get("schema") or {}).get("feeds", [])], + "runs": store.runs(row["uid"]), + "collectors_path": str(files.collectors_path(row["slug"])) if source else "", + "collectors_lines": len(source.splitlines()), + "busy": store.busy(row["uid"]), + "sync_enabled": sync_enabled(), + "admin_section": "communities", + }, + model=AdminCommunityOut, + ) + + +@router.post("/communities/{slug}/run") +async def admin_community_run(request: Request, slug: str, data: Annotated[CommunityRunForm, Form()]): + admin = require_admin(request) + row = require_community(slug) + if store.busy(row["uid"]): + return json_error(409, "A run is already in progress for this community") + spawn(CommunityRunner().run(row, dry_run=data.dry_run, trigger=TRIGGER_ADMIN)) + audit.record( + request, + "admin.community.run", + user=admin, + target_type="community", + target_uid=row["uid"], + target_label=slug, + metadata={"dry_run": data.dry_run}, + summary=f"{admin['username']} started a {'dry run' if data.dry_run else 'run'} of community {slug}", + ) + return action_result(request, f"{LIST_URL}/{slug}") + + +@router.post("/communities/{slug}/relearn") +async def admin_community_relearn(request: Request, slug: str): + admin = require_admin(request) + row = require_community(slug) + if store.busy(row["uid"]): + return json_error(409, "A run is already in progress for this community") + spawn(CommunityRunner().discover(row["query"] or row["url"], admin["uid"], relearn_slug=slug)) + audit.record( + request, + "admin.community.relearn", + user=admin, + target_type="community", + target_uid=row["uid"], + target_label=slug, + summary=f"{admin['username']} started relearning community {slug}", + ) + return action_result(request, f"{LIST_URL}/{slug}") + + +@router.post("/communities/{slug}/update") +async def admin_community_update(request: Request, slug: str, data: Annotated[CommunityUpdateForm, Form()]): + admin = require_admin(request) + row = require_community(slug) + changes = {"interval_seconds": data.interval_seconds, "enabled": int(data.enabled)} + if data.clear_login: + changes.update(login_username="", login_email="", login_password="") + else: + for column, value in (("login_username", data.login_username.strip()), ("login_email", data.login_email.strip()), ("login_password", data.login_password)): + if value: + changes[column] = value + submitted = {name: getattr(data, name) for name in SyncOptions.__dataclass_fields__ if getattr(data, name) is not None} + if submitted: + try: + options = SyncOptions.from_row(row).merged(**submitted) + except ValueError as exc: + return json_error(400, str(exc)) + if options.alias and not SyncOptions.from_row(row).alias and not identity_map(row["origin"]) and count_links(row["origin"]): + return json_error(400, "This community already has records stored under their original names; aliasing cannot be switched on afterwards") + changes["sync_options"] = options.to_dict() + store.update(row["uid"], **changes) + audit.record( + request, + "admin.community.update", + user=admin, + target_type="community", + target_uid=row["uid"], + target_label=slug, + metadata={"interval_seconds": data.interval_seconds, "enabled": data.enabled, "sync_options": changes.get("sync_options"), "login_changed": data.clear_login or bool(data.login_password or data.login_username or data.login_email)}, + summary=f"{admin['username']} updated community {slug}", + ) + return action_result(request, f"{LIST_URL}/{slug}") + + +@router.post("/communities/{slug}/delete") +async def admin_community_delete(request: Request, slug: str): + admin = require_admin(request) + row = require_community(slug) + store.delete(row["uid"], admin["uid"]) + audit.record( + request, + "admin.community.delete", + user=admin, + target_type="community", + target_uid=row["uid"], + target_label=slug, + summary=f"{admin['username']} deleted community {slug}", + ) + return action_result(request, LIST_URL) diff --git a/devplacepy/routers/admin/nodes.py b/devplacepy/routers/admin/nodes.py new file mode 100644 index 00000000..75a1c3cb --- /dev/null +++ b/devplacepy/routers/admin/nodes.py @@ -0,0 +1,376 @@ +# retoor + +import json +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Annotated + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import HTMLResponse + +from devplacepy.database import get_int_setting, get_setting, get_table, get_users_by_uids, set_setting +from devplacepy.database.nodes import ( + ADMIN_LIST_LIMIT, + DEVICE_BREAKDOWN_COLUMNS, + LIMITS, + add_event, + count_open_requests, + device_breakdown, + effective_status, + get_device, + list_flags, + recent_devices, + request_status_counts, + resolve_flag, + suspend_device, + unsuspend_device, + usage_day_totals, +) +from devplacepy.dependencies import json_or_form +from devplacepy.models import AdminNodeKillswitchForm, AdminNodeSettingsForm, AdminNodeSuspendForm +from devplacepy.responses import action_result, json_error, respond +from devplacepy.routers.admin._shared import deny_senior, is_senior_admin +from devplacepy.routers.nodes.social import SOCIAL_DAILY_DEFAULT +from devplacepy.schemas import ( + AdminNodeDeviceOut, + AdminNodeFlagOut, + AdminNodePoolOut, + AdminNodeQueueOut, + AdminNodeSettingOut, + AdminNodesOut, +) +from devplacepy.seo import base_seo_context, site_url, website_schema +from devplacepy.services.audit import record as audit +from devplacepy.services.base import BaseService +from devplacepy.utils import not_found, require_admin + +logger = logging.getLogger(__name__) +router = APIRouter() + +LIST_URL = "/admin/nodes" +SERVICE_NAME = "nodes" +KILLSWITCH_KEY = "nodes_enabled" +QUOTA_USER_CEILING = 10_000 +QUOTA_GUEST_CEILING = 1_000 +QUOTA_SOCIAL_CEILING = 10_000 +POOL_STATS = ( + ("connected", "connected"), + ("capacity", "capacity"), + ("accepted", "accepted"), + ("kicked", "kicked"), + ("jobs done", "jobs_done"), + ("jobs failed", "jobs_failed"), + ("open circuits", "open_circuits"), +) + + +@dataclass(frozen=True) +class NodeSettingField: + key: str + label: str + default: int + ceiling: int | None + kind: str = "int" + + +SETTING_FIELDS: tuple[NodeSettingField, ...] = ( + NodeSettingField("nodes_max_total", "Maximum connected nodes", LIMITS.max_connected_nodes, LIMITS.max_connected_nodes), + NodeSettingField("nodes_daily_quota_user", "Daily node requests per member", LIMITS.default_daily_requests_user, QUOTA_USER_CEILING), + NodeSettingField("nodes_daily_quota_guest", "Daily node requests per guest", LIMITS.default_daily_requests_guest, QUOTA_GUEST_CEILING), + NodeSettingField("nodes_cache_ttl_seconds", "Result cache lifetime (seconds)", LIMITS.cache_ttl_seconds, LIMITS.cache_ttl_seconds), + NodeSettingField("nodes_daily_social_user", "Daily social searches per member", SOCIAL_DAILY_DEFAULT, QUOTA_SOCIAL_CEILING), + NodeSettingField("nodes_social_via_nodes", "Route social search through nodes", 0, None, "flag"), +) +SETTING_FIELDS_BY_KEY = {field.key: field for field in SETTING_FIELDS} + + +def kill_switch_on() -> bool: + return (get_setting(KILLSWITCH_KEY, "0") or "0") == "1" + + +def pool_view() -> AdminNodePoolOut: + row = get_table("service_state").find_one(name=SERVICE_NAME) + if not row or not row.get("heartbeat"): + return AdminNodePoolOut() + try: + beat = datetime.fromisoformat(row["heartbeat"]) + except ValueError: + return AdminNodePoolOut() + if beat.tzinfo is None: + beat = beat.replace(tzinfo=timezone.utc) + if (datetime.now(timezone.utc) - beat).total_seconds() > BaseService.STALE_SECONDS * 2: + return AdminNodePoolOut(heartbeat=row["heartbeat"]) + try: + stats = json.loads(row.get("metrics") or "{}").get("stats") or [] + except ValueError: + stats = [] + by_label = {item.get("label"): item.get("value") for item in stats} + values = {field: int(by_label.get(label) or 0) for label, field in POOL_STATS} + return AdminNodePoolOut(online=True, heartbeat=row["heartbeat"], **values) + + +def queue_view() -> AdminNodeQueueOut: + return AdminNodeQueueOut( + depth=count_open_requests(), + max_depth=LIMITS.max_queue_depth, + **request_status_counts(), + ) + + +def setting_value(field: NodeSettingField) -> str: + if field.kind == "flag": + return "1" if (get_setting(field.key, str(field.default)) or "0") == "1" else "0" + return str(get_int_setting(field.key, field.default)) + + +def settings_view() -> list[AdminNodeSettingOut]: + return [ + AdminNodeSettingOut( + key=field.key, + label=field.label, + kind=field.kind, + value=setting_value(field), + default=str(field.default), + ceiling=field.ceiling, + ) + for field in SETTING_FIELDS + ] + + +def device_views(rows: list[dict], owners: dict) -> list[AdminNodeDeviceOut]: + return [ + AdminNodeDeviceOut( + uid=row["uid"], + label=row.get("label") or "", + kind=row.get("kind") or "", + platform=row.get("platform") or "", + client_version=row.get("client_version") or "", + status=effective_status(row), + suspended_until=row.get("suspended_until") or None, + suspend_reason=row.get("suspend_reason") or "", + country=row.get("country") or "", + asn_class=row.get("asn_class") or "unknown", + last_seen_at=row.get("last_seen_at") or None, + created_at=row.get("created_at"), + owner_uid=row.get("user_uid") or "", + owner_username=(owners.get(row.get("user_uid")) or {}).get("username") or "", + ) + for row in rows + ] + + +def flag_views(rows: list[dict], owners: dict, devices: dict) -> list[AdminNodeFlagOut]: + return [ + AdminNodeFlagOut( + uid=row["uid"], + kind=row.get("kind") or "", + detail=row.get("detail") or "", + status=row.get("status") or "open", + device_uid=row.get("device_uid") or "", + device_label=(devices.get(row.get("device_uid")) or {}).get("label") or "", + user_uid=row.get("user_uid") or "", + owner_username=(owners.get(row.get("user_uid")) or {}).get("username") or "", + created_at=row.get("created_at"), + ) + for row in rows + ] + + +def overview() -> dict: + devices = recent_devices(ADMIN_LIST_LIMIT) + flags = list_flags()[:ADMIN_LIST_LIMIT] + device_by_uid = {row["uid"]: row for row in devices} + missing = [row["device_uid"] for row in flags if row.get("device_uid") and row["device_uid"] not in device_by_uid] + for uid in missing: + found = get_device(uid) + if found: + device_by_uid[uid] = found + owner_uids = [row["user_uid"] for row in devices + flags if row.get("user_uid")] + owners = get_users_by_uids(list(dict.fromkeys(owner_uids))) + breakdowns = {f"devices_by_{column}": device_breakdown(column) for column in DEVICE_BREAKDOWN_COLUMNS} + return { + "enabled": kill_switch_on(), + "pool": pool_view(), + "queue": queue_view(), + **breakdowns, + "usage_today": usage_day_totals(), + "flags": flag_views(flags, owners, device_by_uid), + "devices": device_views(devices, owners), + "settings": settings_view(), + "limits": LIMITS.describe(), + "admin_section": "nodes", + } + + +def seo(request: Request): + base = site_url(request) + return base_seo_context( + request, + title="Community nodes - Admin", + description="Community donor node network: pool, queue, devices and flags.", + breadcrumbs=[ + {"name": "Home", "url": "/feed"}, + {"name": "Admin", "url": "/admin"}, + {"name": "Community nodes", "url": LIST_URL}, + ], + schemas=[website_schema(base)], + robots="noindex,nofollow", + ) + + +@router.get("/nodes", response_class=HTMLResponse) +async def admin_nodes(request: Request): + admin = require_admin(request) + return respond( + request, + "admin_nodes.html", + {**seo(request), "request": request, "user": admin, **overview()}, + model=AdminNodesOut, + ) + + +@router.post("/nodes/killswitch") +async def admin_nodes_killswitch( + request: Request, + data: Annotated[AdminNodeKillswitchForm, Depends(json_or_form(AdminNodeKillswitchForm))], +): + admin = require_admin(request) + before = kill_switch_on() + after = (not before) if data.enabled is None else data.enabled + set_setting(KILLSWITCH_KEY, "1" if after else "0") + audit.record( + request, + "node.killswitch", + user=admin, + target_type="setting", + target_uid=KILLSWITCH_KEY, + target_label=KILLSWITCH_KEY, + old_value="1" if before else "0", + new_value="1" if after else "0", + summary=f"admin {admin['username']} switched community donation {'on' if after else 'off'}", + links=[audit.setting(KILLSWITCH_KEY)], + ) + return action_result(request, LIST_URL, data={"enabled": after}) + + +@router.post("/nodes/settings") +async def admin_nodes_settings( + request: Request, + data: Annotated[AdminNodeSettingsForm, Depends(json_or_form(AdminNodeSettingsForm))], +): + admin = require_admin(request) + changes: dict[str, dict] = {} + clamped: list[str] = [] + for key, submitted in data.model_dump().items(): + if submitted is None: + continue + field = SETTING_FIELDS_BY_KEY[key] + value = str(submitted) + if field.ceiling is not None and int(submitted) > field.ceiling: + value = str(field.ceiling) + clamped.append(key) + before = setting_value(field) + if before == value: + continue + set_setting(key, value) + changes[key] = {"old": before, "new": value} + audit.record( + request, + "node.admin.settings", + user=admin, + target_type="setting", + target_uid="nodes", + target_label="nodes", + metadata={"changes": changes, "clamped": clamped}, + summary=f"admin {admin['username']} updated community node settings ({len(changes)} changed)", + links=[audit.setting(key) for key in changes], + ) + return action_result(request, LIST_URL, data={"changed": list(changes), "clamped": clamped}) + + +def require_device(device_uid: str) -> dict: + device = get_device(device_uid) + if device is None: + raise not_found("Unknown device") + return device + + +def owner_of(device: dict) -> dict | None: + return get_table("users").find_one(uid=device.get("user_uid")) + + +@router.post("/nodes/devices/{device_uid}/suspend") +async def admin_nodes_suspend( + request: Request, + device_uid: str, + data: Annotated[AdminNodeSuspendForm, Depends(json_or_form(AdminNodeSuspendForm))], +): + admin = require_admin(request) + device = require_device(device_uid) + owner = owner_of(device) + if is_senior_admin(admin, owner): + return deny_senior(request, admin, owner["uid"], owner, "node.suspend", LIST_URL) + if not suspend_device(device_uid, data.reason, data.minutes): + return json_error(409, "This device can no longer be suspended") + add_event("admin_suspend", device_uid=device_uid, user_uid=device.get("user_uid") or "", detail=data.reason) + audit.record( + request, + "node.suspend", + user=admin, + target_type="node_device", + target_uid=device_uid, + target_label=device.get("label") or device_uid, + metadata={"minutes": data.minutes, "reason": data.reason, "owner_uid": device.get("user_uid")}, + summary=f"admin {admin['username']} suspended node device {device.get('label') or device_uid} for {data.minutes} minutes", + links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))], + ) + return action_result(request, LIST_URL, data={"uid": device_uid, "minutes": data.minutes}) + + +@router.post("/nodes/devices/{device_uid}/unsuspend") +async def admin_nodes_unsuspend(request: Request, device_uid: str): + admin = require_admin(request) + device = require_device(device_uid) + owner = owner_of(device) + if is_senior_admin(admin, owner): + return deny_senior(request, admin, owner["uid"], owner, "node.unsuspend", LIST_URL) + if not unsuspend_device(device_uid): + return json_error(409, "This device is not suspended") + add_event("admin_unsuspend", device_uid=device_uid, user_uid=device.get("user_uid") or "") + audit.record( + request, + "node.unsuspend", + user=admin, + target_type="node_device", + target_uid=device_uid, + target_label=device.get("label") or device_uid, + metadata={"owner_uid": device.get("user_uid")}, + summary=f"admin {admin['username']} lifted the suspension of node device {device.get('label') or device_uid}", + links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))], + ) + return action_result(request, LIST_URL, data={"uid": device_uid}) + + +@router.post("/nodes/flags/{flag_uid}/resolve") +async def admin_nodes_flag_resolve(request: Request, flag_uid: str): + admin = require_admin(request) + flag = get_table("node_flags").find_one(uid=flag_uid, deleted_at=None) + if flag is None: + raise not_found("Unknown flag") + owner = get_table("users").find_one(uid=flag.get("user_uid")) if flag.get("user_uid") else None + if is_senior_admin(admin, owner): + return deny_senior(request, admin, owner["uid"], owner, "node.flag.resolve", LIST_URL) + if not resolve_flag(flag_uid, admin["uid"]): + return json_error(409, "This flag is already resolved") + audit.record( + request, + "node.flag.resolve", + user=admin, + target_type="node_flag", + target_uid=flag_uid, + target_label=flag.get("kind") or flag_uid, + metadata={"device_uid": flag.get("device_uid"), "owner_uid": flag.get("user_uid")}, + summary=f"admin {admin['username']} resolved node flag {flag.get('kind') or flag_uid}", + ) + return action_result(request, LIST_URL, data={"uid": flag_uid}) diff --git a/devplacepy/routers/admin/settings.py b/devplacepy/routers/admin/settings.py index 292d92b7..1b67eaaf 100644 --- a/devplacepy/routers/admin/settings.py +++ b/devplacepy/routers/admin/settings.py @@ -16,7 +16,7 @@ from devplacepy.dependencies import json_or_form logger = logging.getLogger(__name__) router = APIRouter() -CLEARABLE_SETTINGS = {"extra_head", "ios_app_url"} +CLEARABLE_SETTINGS = {"extra_head", "ios_app_url", "sync_origins"} @router.get("/settings", response_class=HTMLResponse) async def admin_settings(request: Request): diff --git a/devplacepy/routers/docs/pages.py b/devplacepy/routers/docs/pages.py index a0e84bb4..6ba203ea 100644 --- a/devplacepy/routers/docs/pages.py +++ b/devplacepy/routers/docs/pages.py @@ -75,6 +75,12 @@ DOCS_PAGES = [ "kind": "prose", "section": SECTION_GENERAL, }, + { + "slug": "nodes", + "title": "Donate your connection", + "kind": "prose", + "section": SECTION_GENERAL, + }, { "slug": "quizzes", "title": "Quizzes", diff --git a/devplacepy/routers/nodes/__init__.py b/devplacepy/routers/nodes/__init__.py index 03f36231..e578ef47 100644 --- a/devplacepy/routers/nodes/__init__.py +++ b/devplacepy/routers/nodes/__init__.py @@ -2,10 +2,13 @@ from fastapi import APIRouter -from . import devices, spec, ticket, ws +from . import clients, devices, me, social, spec, ticket, ws router = APIRouter() router.include_router(spec.router) +router.include_router(clients.router) +router.include_router(me.router) router.include_router(devices.router) router.include_router(ticket.router) +router.include_router(social.router) router.include_router(ws.router) diff --git a/devplacepy/routers/nodes/clients.py b/devplacepy/routers/nodes/clients.py new file mode 100644 index 00000000..eb365eef --- /dev/null +++ b/devplacepy/routers/nodes/clients.py @@ -0,0 +1,95 @@ +# retoor + +import hashlib +import json +import re + +from fastapi import APIRouter +from fastapi.responses import FileResponse, JSONResponse + +from devplacepy import config +from devplacepy.responses import json_error +from devplacepy.schemas.nodes import NodeArtifactOut, NodeClientVersionOut +from devplacepy.services.nodes.spec import SPEC + +router = APIRouter() + +MANIFEST_NAME = "manifest.json" +ARTIFACT_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$") +MAX_ARTIFACTS = 64 +MAX_ARTIFACT_BYTES = 200 * 1024 * 1024 +_digests: dict[tuple[str, int, int], str] = {} + + +def _digest(path) -> str: + stat = path.stat() + key = (path.name, int(stat.st_mtime_ns), stat.st_size) + found = _digests.get(key) + if found is None: + sha = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + sha.update(block) + found = sha.hexdigest() + if len(_digests) > MAX_ARTIFACTS * 4: + _digests.clear() + _digests[key] = found + return found + + +def _manifest_version() -> str: + path = config.NODE_CLIENTS_DIR / MANIFEST_NAME + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return "" + if isinstance(data, dict): + return str(data.get("version") or "")[:40] + if isinstance(data, list) and data and isinstance(data[0], dict): + return str(data[0].get("version") or "")[:40] + return "" + + +def _artifacts() -> list[NodeArtifactOut]: + root = config.NODE_CLIENTS_DIR + if not root.is_dir(): + return [] + found = [] + for path in sorted(root.iterdir())[:MAX_ARTIFACTS]: + if ( + path.name == MANIFEST_NAME + or not path.is_file() + or not ARTIFACT_NAME.match(path.name) + or path.stat().st_size > MAX_ARTIFACT_BYTES + ): + continue + found.append( + NodeArtifactOut(name=path.name, size=path.stat().st_size, sha256=_digest(path)) + ) + return found + + +@router.get("/client-version") +async def client_version(): + payload = NodeClientVersionOut( + version=_manifest_version(), + protocol=SPEC.version, + min_protocol=SPEC.min_supported, + artifacts=_artifacts(), + ) + return JSONResponse(payload.model_dump(mode="json")) + + +@router.get("/download/{artifact}") +async def download_client(artifact: str): + if not ARTIFACT_NAME.match(artifact) or artifact == MANIFEST_NAME: + return json_error(404, "No such client") + path = config.NODE_CLIENTS_DIR / artifact + if not path.is_file() or path.is_symlink() or path.stat().st_size > MAX_ARTIFACT_BYTES: + return json_error(404, "No such client") + return FileResponse( + path, + media_type="application/octet-stream", + filename=artifact, + headers={"X-Content-SHA256": _digest(path), "X-Content-Type-Options": "nosniff"}, + ) diff --git a/devplacepy/routers/nodes/me.py b/devplacepy/routers/nodes/me.py new file mode 100644 index 00000000..8fabf9cf --- /dev/null +++ b/devplacepy/routers/nodes/me.py @@ -0,0 +1,143 @@ +# retoor + +import logging +import os +from typing import Annotated + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse + +from devplacepy.cache import TTLCache +from devplacepy.database import get_users_by_uids +from devplacepy.database.moderation import consent_granted +from devplacepy.database.nodes import ( + LIMITS, + PLATFORMS, + get_node_settings, + list_devices, + public_stats, + save_node_settings, + top_donors, +) +from devplacepy.dependencies import json_or_form +from devplacepy.models import NodeSettingsForm +from devplacepy.responses import json_error +from devplacepy.routers.nodes._shared import GATE, NO_STORE, device_view +from devplacepy.schemas.nodes import ( + NodeDonorOut, + NodeLeaderboardOut, + NodeMeOut, + NodeStatsOut, +) +from devplacepy.services.audit import record as audit +from devplacepy.services.nodes.ledger import UsageLedger +from devplacepy.utils import require_user_api + +logger = logging.getLogger(__name__) +router = APIRouter() + +AGENT_HEADER = "x-devii-agent" +LEADERBOARD_LIMIT = 10 +STATS_TTL_SECONDS = int(os.environ.get("DEVPLACE_NODES_STATS_TTL", "30")) +_public_cache = TTLCache(STATS_TTL_SECONDS, max_size=4) + + +def _me(user: dict) -> NodeMeOut: + verdict = GATE.allows(user["uid"]) + return NodeMeOut( + consent=consent_granted("user", user["uid"], "bandwidth_sharing"), + can_donate=verdict.allowed, + reason=verdict.reason, + message=verdict.message, + settings=get_node_settings(user["uid"]), + platforms=list(PLATFORMS), + devices=[device_view(row) for row in list_devices(user["uid"])], + max_devices=LIMITS.max_devices_per_user, + donated=UsageLedger().summary(user["uid"]).describe(), + limits={ + "max_donor_mb_per_day": LIMITS.max_donor_mb_per_day, + "max_concurrent_per_node": LIMITS.max_concurrent_per_node, + "default_donor_mb_per_day": LIMITS.default_donor_mb_per_day, + }, + ) + + +@router.get("/me") +async def my_donation(request: Request): + user = require_user_api(request) + return JSONResponse(_me(user).model_dump(mode="json"), headers=NO_STORE) + + +@router.post("/settings") +async def save_my_settings( + request: Request, + data: Annotated[NodeSettingsForm, Depends(json_or_form(NodeSettingsForm))], +): + user = require_user_api(request) + values = data.model_dump(exclude_none=True) + if values.get("enabled"): + if request.headers.get(AGENT_HEADER) == "1": + audit.record( + request, + "security.authz.denied", + user=user, + result="denied", + target_type="user", + target_uid=user["uid"], + metadata={"reason": "agent cannot enable donation"}, + summary="agent denied enabling bandwidth donation", + ) + return json_error( + 403, "Donation can only be switched on by the member in their own profile" + ) + if not consent_granted("user", user["uid"], "bandwidth_sharing"): + return json_error( + 403, + "Give the bandwidth sharing consent before switching donation on", + reason="no_consent", + ) + before = get_node_settings(user["uid"]) + saved = save_node_settings(user["uid"], values) + audit.record( + request, + "node.settings.update", + user=user, + target_type="user", + target_uid=user["uid"], + target_label=user["username"], + old_value=str(before.get("enabled")), + new_value=str(saved.get("enabled")), + metadata={key: saved[key] for key in values if key in saved}, + summary="updated bandwidth donation settings", + ) + return JSONResponse(_me(user).model_dump(mode="json"), headers=NO_STORE) + + +@router.get("/stats") +async def donation_stats(): + cached = _public_cache.get("stats") + if cached is None: + cached = NodeStatsOut(**public_stats()).model_dump(mode="json") + _public_cache.set("stats", cached) + return JSONResponse(cached) + + +@router.get("/leaderboard") +async def donation_leaderboard(): + cached = _public_cache.get("leaderboard") + if cached is None: + rows = top_donors(LEADERBOARD_LIMIT) + users = get_users_by_uids([row["user_uid"] for row in rows]) + cached = NodeLeaderboardOut( + donors=[ + NodeDonorOut( + username=users[row["user_uid"]]["username"], + requests=int(row["requests_ok"] or 0), + bytes=int(row["bytes_total"] or 0), + ) + for row in rows + if row["user_uid"] in users + ] + ).model_dump(mode="json") + _public_cache.set("leaderboard", cached) + return JSONResponse(cached) diff --git a/devplacepy/routers/nodes/social.py b/devplacepy/routers/nodes/social.py new file mode 100644 index 00000000..3efd754c --- /dev/null +++ b/devplacepy/routers/nodes/social.py @@ -0,0 +1,98 @@ +# retoor + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse + +from devplacepy.database import get_int_setting +from devplacepy.database.nodes import reserve_caller_request +from devplacepy.dependencies import json_or_form +from devplacepy.models import NodeFetchForm +from devplacepy.responses import json_error +from devplacepy.routers.nodes._shared import NO_STORE +from devplacepy.schemas.nodes import NodeFetchOut, SocialSearchOut +from devplacepy.services.nodes.fabric import ( + USER_KIND, + FabricQuotaExceeded, + FabricRefused, + FabricUnavailable, + shared_fabric, +) +from devplacepy.services.nodes.social import DEFAULT_LIMIT, SocialSearch +from devplacepy.utils import require_user_api + +logger = logging.getLogger(__name__) +router = APIRouter() + +SOCIAL_KIND = "social" +SOCIAL_DAILY_DEFAULT = 300 +FETCH_TEXT_MAX = 1024 * 1024 + + +def social_quota_ok(user_uid: str) -> bool: + cap = get_int_setting("nodes_daily_social_user", SOCIAL_DAILY_DEFAULT) + return reserve_caller_request(SOCIAL_KIND, user_uid, cap) + + +@router.get("/social/search") +async def social_search( + request: Request, + q: str = "", + platforms: str = "", + limit: int = DEFAULT_LIMIT, +): + user = require_user_api(request) + query = " ".join(q.split()) + if not query: + return json_error(400, "A search query is required") + wanted = [item.strip().lower() for item in platforms.split(",") if item.strip()] + if not social_quota_ok(user["uid"]): + return json_error(429, "Daily social search quota reached") + search = SocialSearch() + outcome = await search.search( + query, + caller_kind=USER_KIND, + caller_id=user["uid"], + platforms=wanted, + limit=limit, + ) + payload = SocialSearchOut( + query=query[:200], + platforms=search.platforms_for(wanted), + **outcome.as_dict(), + ) + return JSONResponse(payload.model_dump(mode="json"), headers=NO_STORE) + + +@router.post("/internal/fetch") +async def internal_fetch( + request: Request, + data: Annotated[NodeFetchForm, Depends(json_or_form(NodeFetchForm))], +): + user = require_user_api(request) + try: + result = await shared_fabric().fetch( + data.url, + caller_kind=USER_KIND, + caller_id=user["uid"], + use_cache=data.cache, + timeout=data.timeout, + ) + except FabricRefused as exc: + return json_error(403, exc.message, reason=exc.code) + except FabricQuotaExceeded as exc: + return json_error(429, exc.message, reason=exc.code) + except FabricUnavailable as exc: + return json_error(503, exc.message, reason=exc.code) + payload = NodeFetchOut( + url=result.url, + status=result.status, + via=result.via, + truncated=result.truncated or len(result.body) > FETCH_TEXT_MAX, + elapsed_ms=result.elapsed_ms, + content_type=str(result.headers.get("content-type", ""))[:120], + text=result.body[:FETCH_TEXT_MAX].decode("utf-8", errors="replace"), + ) + return JSONResponse(payload.model_dump(mode="json"), headers=NO_STORE) diff --git a/devplacepy/routers/nodes/ws.py b/devplacepy/routers/nodes/ws.py index ab6a36e1..bf172fd4 100644 --- a/devplacepy/routers/nodes/ws.py +++ b/devplacepy/routers/nodes/ws.py @@ -67,6 +67,8 @@ async def node_ws(websocket: WebSocket, ticket: str = ""): await websocket.close(code=CLOSE_POLICY, reason=verdict.reason) return session = NodeSession(websocket, device) + service.dispatcher.attach(session) + service.ensure_dispatching() admitted, previous = registry.admit(session) if not admitted: await websocket.close(code=CLOSE_CAPACITY, reason="capacity") @@ -99,6 +101,7 @@ async def node_ws(websocket: WebSocket, ticket: str = ""): if ended == "heartbeat": counters.inc("evicted_heartbeat") registry.release(session) + service.dispatcher.release_session(session) add_event( "disconnect", device_uid=device["uid"], diff --git a/devplacepy/routers/profile/index.py b/devplacepy/routers/profile/index.py index fdb2ad29..cb7b9b68 100644 --- a/devplacepy/routers/profile/index.py +++ b/devplacepy/routers/profile/index.py @@ -38,6 +38,7 @@ from devplacepy.database.awards import ( get_user_awards, ) from devplacepy.content import can_view_project, enrich_items +from devplacepy.routers.nodes.me import _me as node_me_view from devplacepy.routers.profile.consent import consent_view from devplacepy.services.moderation.deletion import grace_hours from devplacepy.services.moderation.screening import ( @@ -401,8 +402,18 @@ async def profile_page( } ) + node_me = ( + node_me_view(current_user).model_dump(mode="json") + if (tab == "donate" and is_owner) + else {} + ) + base = site_url(request) - robots = "noindex,follow" if posts_count < 2 else "index,follow" + robots = ( + "noindex,follow" + if posts_count < 2 or (tab == "donate" and is_owner) + else "index,follow" + ) bio = profile_user.get("bio", "") desc = ( bio @@ -473,6 +484,7 @@ async def profile_page( "cust_disable_pagetype": customization_prefs["disable_pagetype"], "notification_prefs": notification_prefs, "consents": consents, + "node_me": node_me, **privacy_fields, "current_terms_version": get_setting("terms_version", "1") or "1", "deletion_grace_hours": grace_hours(), diff --git a/devplacepy/routers/sync/__init__.py b/devplacepy/routers/sync/__init__.py new file mode 100644 index 00000000..5e99488b --- /dev/null +++ b/devplacepy/routers/sync/__init__.py @@ -0,0 +1,11 @@ +# retoor + +from fastapi import APIRouter + +from . import comments, posts, spec, users + +router = APIRouter() +router.include_router(spec.router) +router.include_router(users.router) +router.include_router(posts.router) +router.include_router(comments.router) diff --git a/devplacepy/routers/sync/_shared.py b/devplacepy/routers/sync/_shared.py new file mode 100644 index 00000000..f447351c --- /dev/null +++ b/devplacepy/routers/sync/_shared.py @@ -0,0 +1,66 @@ +# retoor + +from fastapi import HTTPException, Request +from fastapi.responses import JSONResponse + +from devplacepy.responses import json_error +from devplacepy.services.audit import record as audit +from devplacepy.services.sync.batch import BatchRunner +from devplacepy.services.sync.gate import allowed_origins, sync_enabled +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.utils import is_admin, require_user_api + +NO_STORE = {"Cache-Control": "no-store"} + + +def require_sync_caller(request: Request) -> dict: + user = require_user_api(request) + if not is_admin(user): + audit.record( + request, + "security.authz.denied", + user=user, + result="denied", + summary=f"non-admin denied access to {request.url.path}", + ) + raise HTTPException(status_code=403, detail="Administrator access required") + return user + + +def run_batch( + request: Request, user: dict, kind: str, data: SyncBatchForm +) -> JSONResponse: + if not sync_enabled(): + return json_error(503, "Community sync is disabled") + allowed = allowed_origins() + if allowed and data.origin not in allowed: + audit.record( + request, + "sync.batch", + user=user, + result="denied", + target_type="sync", + target_label=f"{data.origin}/{kind}", + summary=f"sync from origin {data.origin} refused: not in sync_origins", + ) + return json_error(403, f"origin {data.origin} is not allowed") + outcome = BatchRunner(kind, request=request).run(data) + audit.record( + request, + "sync.batch", + user=user, + target_type="sync", + target_label=f"{data.origin}/{kind}", + metadata={ + "origin": data.origin, + "kind": kind, + "dry_run": data.dry_run, + "total": outcome.total, + "counts": outcome.counts, + }, + summary=( + f"{user['username']} synced {outcome.total} {kind} from {data.origin}" + f"{' (dry run)' if data.dry_run else ''}" + ), + ) + return JSONResponse(outcome.model_dump(mode="json"), headers=NO_STORE) diff --git a/devplacepy/routers/sync/comments.py b/devplacepy/routers/sync/comments.py new file mode 100644 index 00000000..a572b604 --- /dev/null +++ b/devplacepy/routers/sync/comments.py @@ -0,0 +1,21 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request + +from devplacepy.dependencies import json_or_form +from devplacepy.routers.sync._shared import require_sync_caller, run_batch +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KIND_COMMENTS + +router = APIRouter() + + +@router.post("/comments") +async def sync_comments( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))], +): + return run_batch(request, user, KIND_COMMENTS, data) diff --git a/devplacepy/routers/sync/posts.py b/devplacepy/routers/sync/posts.py new file mode 100644 index 00000000..c62ea4d3 --- /dev/null +++ b/devplacepy/routers/sync/posts.py @@ -0,0 +1,21 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request + +from devplacepy.dependencies import json_or_form +from devplacepy.routers.sync._shared import require_sync_caller, run_batch +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KIND_POSTS + +router = APIRouter() + + +@router.post("/posts") +async def sync_posts( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))], +): + return run_batch(request, user, KIND_POSTS, data) diff --git a/devplacepy/routers/sync/spec.py b/devplacepy/routers/sync/spec.py new file mode 100644 index 00000000..e5ce4686 --- /dev/null +++ b/devplacepy/routers/sync/spec.py @@ -0,0 +1,18 @@ +# retoor + +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +from devplacepy.services.sync.records import RECORD_MODELS +from devplacepy.services.sync.spec import SPEC + +router = APIRouter() + + +@router.get("/spec.json") +async def sync_spec() -> JSONResponse: + description = SPEC.describe() + description["records"] = { + kind: model.model_json_schema() for kind, model in RECORD_MODELS.items() + } + return JSONResponse(description) diff --git a/devplacepy/routers/sync/users.py b/devplacepy/routers/sync/users.py new file mode 100644 index 00000000..ab6a30fc --- /dev/null +++ b/devplacepy/routers/sync/users.py @@ -0,0 +1,21 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request + +from devplacepy.dependencies import json_or_form +from devplacepy.routers.sync._shared import require_sync_caller, run_batch +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KIND_USERS + +router = APIRouter() + + +@router.post("/users") +async def sync_users( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))], +): + return run_batch(request, user, KIND_USERS, data) diff --git a/devplacepy/schemas/__init__.py b/devplacepy/schemas/__init__.py index 98bfc9bf..7e67a5ae 100644 --- a/devplacepy/schemas/__init__.py +++ b/devplacepy/schemas/__init__.py @@ -113,10 +113,19 @@ from devplacepy.schemas.backups import ( BackupStoragePathOut, ) from devplacepy.schemas.admin import ( + AdminCommunitiesOut, + AdminCommunityItemOut, + AdminCommunityOut, AdminDeviiTaskItemOut, AdminDeviiTasksOut, AdminGameOut, AdminMediaItemOut, + AdminNodeDeviceOut, + AdminNodeFlagOut, + AdminNodePoolOut, + AdminNodeQueueOut, + AdminNodeSettingOut, + AdminNodesOut, AdminMediaOut, AdminNewsItemOut, AdminNewsOut, diff --git a/devplacepy/schemas/admin.py b/devplacepy/schemas/admin.py index 79406434..75bfaba1 100644 --- a/devplacepy/schemas/admin.py +++ b/devplacepy/schemas/admin.py @@ -107,3 +107,128 @@ class AdminGameOut(_Out): era_started_at: str = "" era_ends_at: str = "" admin_section: Optional[str] = None + + +class AdminCommunityItemOut(_Out): + uid: str + slug: str + name: str = "" + origin: str = "" + url: str = "" + query: str = "" + status: str = "" + enabled: bool = False + description: str = "" + language: str = "" + user_count: int = 0 + interval_seconds: int = 0 + sync_options: dict = {} + run_count: int = 0 + failure_count: int = 0 + last_run_at: Optional[str] = None + next_run_at: Optional[str] = None + last_status: Optional[str] = None + last_error: Optional[str] = None + learned_at: Optional[str] = None + created_at: Optional[str] = None + login_username: str = "" + login_email: str = "" + has_login: bool = False + + +class AdminCommunitiesOut(_Out): + items: list[AdminCommunityItemOut] = [] + sync_enabled: bool = False + service_enabled: bool = False + service_dry_run: bool = True + admin_section: Optional[str] = None + + +class AdminCommunityOut(_Out): + community: AdminCommunityItemOut + rules: str = "" + favicon: str = "" + totals: dict = {} + feeds: list[dict] = [] + runs: list[dict] = [] + collectors_path: str = "" + collectors_lines: int = 0 + busy: bool = False + sync_enabled: bool = False + admin_section: Optional[str] = None + + +class AdminNodeDeviceOut(_Out): + uid: str = "" + label: str = "" + kind: str = "" + platform: str = "" + client_version: str = "" + status: str = "active" + suspended_until: Optional[str] = None + suspend_reason: str = "" + country: str = "" + asn_class: str = "unknown" + last_seen_at: Optional[str] = None + created_at: Optional[str] = None + owner_uid: str = "" + owner_username: str = "" + + +class AdminNodeFlagOut(_Out): + uid: str = "" + kind: str = "" + detail: str = "" + status: str = "open" + device_uid: str = "" + device_label: str = "" + user_uid: str = "" + owner_username: str = "" + created_at: Optional[str] = None + + +class AdminNodeSettingOut(_Out): + key: str + label: str = "" + kind: str = "int" + value: str = "" + default: str = "" + ceiling: Optional[int] = None + + +class AdminNodePoolOut(_Out): + online: bool = False + connected: int = 0 + capacity: int = 0 + accepted: int = 0 + kicked: int = 0 + jobs_done: int = 0 + jobs_failed: int = 0 + open_circuits: int = 0 + heartbeat: Optional[str] = None + + +class AdminNodeQueueOut(_Out): + depth: int = 0 + max_depth: int = 0 + queued: int = 0 + assigned: int = 0 + done: int = 0 + failed: int = 0 + expired: int = 0 + + +class AdminNodesOut(_Out): + enabled: bool = False + pool: AdminNodePoolOut = AdminNodePoolOut() + queue: AdminNodeQueueOut = AdminNodeQueueOut() + devices_by_status: dict[str, int] = {} + devices_by_kind: dict[str, int] = {} + devices_by_asn_class: dict[str, int] = {} + devices_by_country: dict[str, int] = {} + usage_today: dict[str, int] = {} + flags: list[AdminNodeFlagOut] = [] + devices: list[AdminNodeDeviceOut] = [] + settings: list[AdminNodeSettingOut] = [] + limits: dict[str, int] = {} + admin_section: Optional[str] = None diff --git a/devplacepy/schemas/nodes.py b/devplacepy/schemas/nodes.py index a8fb5f48..e55c4cc6 100644 --- a/devplacepy/schemas/nodes.py +++ b/devplacepy/schemas/nodes.py @@ -40,3 +40,74 @@ class NodeTicketOut(_Out): ticket: str expires_in: int ws_path: str + + +class SocialHitOut(_Out): + platform: str = "" + title: str = "" + url: str = "" + snippet: str = "" + author: str = "" + score: int = 0 + comments: int = 0 + created_at: str = "" + via: str = "direct" + + +class SocialSearchOut(_Out): + query: str = "" + hits: list[SocialHitOut] = [] + unavailable: dict = {} + platforms: list[str] = [] + + +class NodeFetchOut(_Out): + url: str = "" + status: int = 0 + via: str = "" + truncated: bool = False + elapsed_ms: int = 0 + content_type: str = "" + text: str = "" + + +class NodeMeOut(_Out): + consent: bool = False + can_donate: bool = False + reason: str = "" + message: str = "" + settings: dict = {} + platforms: list[str] = [] + devices: list[NodeDeviceOut] = [] + max_devices: int = 0 + donated: dict = {} + limits: dict = {} + + +class NodeStatsOut(_Out): + requests: int = 0 + bytes: int = 0 + donors: int = 0 + + +class NodeDonorOut(_Out): + username: str = "" + requests: int = 0 + bytes: int = 0 + + +class NodeLeaderboardOut(_Out): + donors: list[NodeDonorOut] = [] + + +class NodeArtifactOut(_Out): + name: str = "" + size: int = 0 + sha256: str = "" + + +class NodeClientVersionOut(_Out): + version: str = "" + protocol: int = 0 + min_protocol: int = 0 + artifacts: list[NodeArtifactOut] = [] diff --git a/devplacepy/schemas/profile.py b/devplacepy/schemas/profile.py index c024b53f..35fbd056 100644 --- a/devplacepy/schemas/profile.py +++ b/devplacepy/schemas/profile.py @@ -84,6 +84,7 @@ class ProfileOut(_Out): prominent_award: Optional[AwardOut] = None can_give_award: bool = False consents: list[Any] = [] + node_me: dict = {} mature_opt_in: bool = False age_band: Optional[str] = None terms_version: Optional[str] = None diff --git a/devplacepy/schemas/sync.py b/devplacepy/schemas/sync.py new file mode 100644 index 00000000..6b37b796 --- /dev/null +++ b/devplacepy/schemas/sync.py @@ -0,0 +1,24 @@ +# retoor + +from __future__ import annotations + +from devplacepy.schemas.base import _Out + + +class SyncRecordResultOut(_Out): + index: int + origin_uid: str = "" + status: str + reason: str = "" + local_uid: str = "" + errors: list[str] = [] + ignored: list[str] = [] + + +class SyncBatchOut(_Out): + origin: str + kind: str + dry_run: bool = False + total: int = 0 + counts: dict[str, int] = {} + results: list[SyncRecordResultOut] = [] diff --git a/devplacepy/services/audit/categories.py b/devplacepy/services/audit/categories.py index 1a211280..dc43cb16 100644 --- a/devplacepy/services/audit/categories.py +++ b/devplacepy/services/audit/categories.py @@ -50,6 +50,7 @@ CATEGORY_BY_PREFIX: dict[str, str] = { "devii": "devii", "skill": "devii", "node": "nodes", + "sync": "sync", "email": "email", "cli": "cli", "reward": "reward", diff --git a/devplacepy/services/community/CLAUDE.md b/devplacepy/services/community/CLAUDE.md new file mode 100644 index 00000000..9a3434a1 --- /dev/null +++ b/devplacepy/services/community/CLAUDE.md @@ -0,0 +1,66 @@ +This file documents community discovery (`devplacepy/services/community/`, `routers/admin/communities.py`, `cli/community.py`): finding an external community, learning its structure once, generating collectors, and feeding what they collect through the sync API. Claude Code auto-loads it whenever a file under `devplacepy/services/community/` is read or edited. The design record is `add_community.md`. + +## What exists today + +| Piece | File | State | +|-------|------|-------| +| Constants, caps, statuses | `spec.py` (`CommunityLimits`) | built | +| Path, feed and CSS extraction (stdlib only) | `extract.py` | built, tested | +| Collector engine (retry, backoff, pagination, incremental, comments) | `runtime.py` | built, tested | +| Search (rsearch), feed detection, structure learning, metadata | `search.py`, `feeds.py`, `learner.py`, `discoverer.py` | built, tested | +| Code generation | `generator.py` | built, tested | +| Persistence | `store.py` (`communities`, `community_runs`), `files.py` | built | +| Push into sync, with dry run | `pusher.py` | built | +| Run orchestration | `runner.py` (`CommunityRunner`) | built | +| Scheduled service | `service.py` (`CommunityService`, opt-in, dry run by default) | built | +| Admin pages | `/admin/communities`, `/admin/communities/{slug}` | built | +| CLI | `devplace community discover\|run\|login\|list` | built | +| Devii tools (`community_*`), docs group, privacy row | `services/devii/actions/catalog/community.py`, `docs_api/groups/community.py` | built, all administrator-only | +| AI structure assistant | `ai.py` | built, optional, every proposal validated against the sample | +| Login and credentials | `extract.find_login_form`, `runtime.Collector.login`, `discoverer.LoginLearner` | built | + +## Flow + +`discover` -> `SearchPhase` (a direct address skips rsearch; otherwise `rsearch.app.molodetz.nl/search`, ranked by host/title match, social sites dropped, at most 5 candidates) -> `FeedDetector` (the page itself, ``, same-host "latest/posts/..." links one level deep, then a fixed list of probe paths; JS-only pages escalate to `Fetcher.render`) -> `StructureLearner` per feed -> `MetadataLearner` (name, description, language, favicon, member count, rules text) -> `Collector` verification run on a small sample. A community is only `ready` when the verification run collected at least one post. No feed plus a login wall is `login_required`; anything else is `failed` with the reason. + +Feed types: `rss`, `atom`, `json_api`, `html_list`. Learned field maps use the **sync record field names** (`id`, `title`, `content`, `created_at`, `updated_at`, author `username`/`id`, comment `parent_id`), each an ordered list of alternates; the collector emits records ready for `UserRecord`/`PostRecord`/`CommentRecord`. JSON paths are `$.a.b[0]`, `[*]`. XML feeds are converted to dicts so the same paths work (`$.link.@href`). HTML fields are `selector::text` or `selector::attr(name)`. + +## Browser, JavaScript, login, AI + +- **stealthii is a project dependency** (`pyproject.toml`, git URL; the Dockerfile installs `git`). `fetcher.Fetcher` shares one stealth Chromium per event loop (`SharedBrowser`), applies the SSRF guard (`guard_public_url`) before every request and every `page.goto`, and waits at least `delay_seconds` between requests to one host. Without the browser it falls back to the guarded plain client and logs that JavaScript rendering is unavailable. `CommunityService.on_disable` closes the browser. +- **JavaScript pages.** If a page yields nothing statically, detection renders it (`Fetcher.render_html`, `page.content()`); the feed is then marked `"render": true` and the collector renders it on every run. The same fallback applies to post pages when comments only exist after rendering. +- **Comments and post bodies from HTML pages.** For any feed (RSS, Atom, JSON, HTML list) whose posts expose an address, `PageProbe` opens up to three post pages, finds the repeated comment element (class hint `comment|reply|response|answer`, nesting from DOM containment, id from `::attr(id)` else a stable hash), and learns a `mode: "page"` comments spec. It also learns a `post.detail.content` selector when the post page carries more text than the listing. JSON feeds still prefer `inline`/`url` comment modes. +- **Login.** A community can store a username, an email and a password (columns `login_*`, never returned by any endpoint, admin form, Devii tool or CLI). Discovery finds the login form (page, login links, `/login`-style paths, rendered page), logs in with the session `community:`, then learns behind the wall. `schema.login` holds the form (action, field names, hidden defaults re-read at login for CSRF tokens, or selectors for a JavaScript page); `login_required` makes a missing account a clear `LoginRequired`. A rejected password raises `LoginFailed` and the community goes to `login_required`. Not supported: JSON-API logins, two-factor, captchas. +- **AI.** When HTML heuristics are weak (no author, no title, content only raw text, no date) or find no item container, comment structure or post body, `AiAssistant` asks the internal gateway model (site setting `community_ai_model`, default `molodetz`; `community_ai=0` turns it off) for selectors. A proposal is accepted per field only if it matches at least 60% of the sampled elements. If the gateway is unreachable the learner continues with heuristics and logs it. +- **Duplicate feeds.** Feeds whose sample posts overlap by half or more are learned once (the one with comments wins); comment feeds (`/comments.rss`) are never treated as post feeds; generic probe paths are only tried when no feed is advertised. + +## Output: one engine, concrete file + +`runtime.py` is the only collector implementation. `CodeGenerator` writes `data/communities//collectors.py` by inlining `extract.py` and `runtime.py` (stdlib only), the learned `SCHEMA` as a literal, and one `collect_` wrapper per feed with the selectors in comments; plus `schema.json`. The service imports that file, so a human edit to `SCHEMA` takes effect. `python collectors.py` runs it standalone with `stealthii`; a login-protected community reads `COMMUNITY_USERNAME`/`COMMUNITY_EMAIL`/`COMMUNITY_PASSWORD` from the environment. The database row holds the schema; the files are regenerated when missing. + +## Rules that are deliberate + +- **Dry run is first class.** `devplace community run --dry-run`, the admin "Dry run" button and the service setting `community_dry_run` (default on) collect and validate through the real sync processors with `dry_run=True`, log every step, write nothing and keep the watermark. Dependent records (a post whose author this dry run would create) are projected as `created` instead of `unknown_author`. A live run needs `sync_enabled=1` and honors `sync_origins` (`services/sync/gate.py`, shared with the `/sync` routes). +- **Nothing unbounded.** Pages, posts, comment fetches, request delay, retries, body size, run time, runs kept (30) and communities per tick are static constants in `CommunityLimits`; no autotuning. +- **Floor.** After `max_failures` (5) consecutive failed runs a community is disabled with status `failing` and the log says to relearn. The machine never deletes; a human relearns or deletes (`delete` is soft, `communities` is in `SOFT_DELETE_TABLES` and `UNREPORTABLE_TABLES`). +- **Incremental.** The watermark is the newest `updated_at`/`created_at` per feed; a page that is entirely older ends pagination. The newest 10 posts have their comments re-fetched every run; duplicates are absorbed by the sync fingerprint (`unchanged`). +- **Outbound HTTP** goes through `Fetcher` (stealthii plus the SSRF guard), never a bare client. +- **Order.** users, posts, comments, in batches of 50. A comment's post and parent are always earlier in the batch. +- Admin add and relearn run as per-worker `asyncio` tasks; the page shows a `busy` banner and a run row while they work. + +## Measured + +Real run against `https://lobste.rs` (dry run, scratch database): RSS, JSON and HTML feeds detected, description/favicon/language/rules text learned, 25 posts and 24 authors collected from one page. Fixture tests cover RSS, Atom, JSON with inline comments and page pagination, and HTML lists. With stealthii and Chromium the same run learned lobste.rs from its `/rss` feed plus post-page comments (`li.comments_subtree`). Not measured: a real login-protected site, cursor pagination against a live site, the AI path against a live gateway (the local gateway rejected the scratch key, so heuristics ran). Fixtures cover JavaScript rendering, form login, a rejected password, RSS and HTML comments from post pages, feed deduplication and the AI validation. + +## Gaps + +- Comment pagination on HTML post pages is not followed (first page only); `max_comment_posts` bounds the pages opened per run. +- No two-factor, captcha or JSON-API login. +- HTML learning is heuristic plus AI; a site whose markup defeats both needs a hand-edited `SCHEMA`. +- Votes, reactions and attachments are not collected (the sync API does not take them). +- Cross-feed duplicate detection compares the sampled post addresses at discovery time only. +- Passwords are stored in plain text in the database, as chosen for now; they are masked everywhere they could be displayed. + +## Selection, aliasing, translation + +Per-community options (`sync_options`) and the `community_alias_enabled` kill switch are documented in `devplacepy/services/sync/CLAUDE.md` ("Selection, identity aliasing and translation"). The runner calls `prepare` between collection and `SyncPusher`; skip counts appear in the run's `pushed["skipped"]` and in the totals as `skipped_`. diff --git a/devplacepy/services/community/__init__.py b/devplacepy/services/community/__init__.py new file mode 100644 index 00000000..95ee7c3e --- /dev/null +++ b/devplacepy/services/community/__init__.py @@ -0,0 +1 @@ +# retoor diff --git a/devplacepy/services/community/ai.py b/devplacepy/services/community/ai.py new file mode 100644 index 00000000..c8726a6c --- /dev/null +++ b/devplacepy/services/community/ai.py @@ -0,0 +1,183 @@ +# retoor + +from __future__ import annotations + +import asyncio +import json +import logging +import re + +from devplacepy.config import INTERNAL_MODEL +from devplacepy.database import get_setting +from devplacepy.database.settings import internal_gateway_key +from devplacepy.services.community.extract import Node, field_text, select_nodes +from devplacepy.services.community.spec import LIMITS + +logger = logging.getLogger("community.ai") + +SELECTOR_SYNTAX = ( + "Selector syntax is a small CSS subset: tag, .class, #id, [attr] or [attr=value], descendant (space) " + "and child (>) combinators. A field is a selector plus an accessor: 'h2 a::text' for the text, " + "'h2 a::attr(href)' for an attribute, and '::attr(id)' (empty selector) for the item element itself. " + "No :nth-child, no pseudo-classes, no commas." +) +FENCE = re.compile(r"^```(?:json)?\s*|\s*```$", re.I) +OUTLINE_LINES = 160 +SAMPLE_HTML_CHARS = 1800 +AI_TIMEOUT_SECONDS = 90 + + +def outline(root: Node, limit: int = OUTLINE_LINES) -> str: + lines: list[str] = [] + + def walk(node: Node, depth: int) -> None: + if len(lines) >= limit or depth > 9: + return + seen: dict[str, int] = {} + for child in node.children: + if len(lines) >= limit: + return + if child.tag in ("script", "style", "svg", "noscript", "link", "meta", "head"): + continue + label = child.tag + (f"#{child.attrs['id']}" if child.attrs.get("id") else "") + "".join(f".{name}" for name in sorted(child.classes())) + seen[label] = seen.get(label, 0) + 1 + if seen[label] > 2: + continue + text = child.text()[:40] + lines.append(f"{' ' * depth}{label}" + (f" \"{text}\"" if text and not child.children else "")) + walk(child, depth + 1) + + walk(root, 0) + return "\n".join(lines) + + +def to_html(node: Node, limit: int = SAMPLE_HTML_CHARS) -> str: + parts: list[str] = [] + + def walk(current: Node) -> None: + attributes = "".join(f' {key}="{value[:80]}"' for key, value in current.attrs.items() if key in ("class", "id", "href", "datetime", "rel", "itemprop", "data-id", "name")) + parts.append(f"<{current.tag}{attributes}>") + for chunk in current.chunks: + if isinstance(chunk, Node): + if chunk.tag not in ("script", "style", "svg"): + walk(chunk) + elif chunk.strip(): + parts.append(chunk.strip()[:160]) + parts.append(f"") + + walk(node) + return "".join(parts)[:limit] + + +def parse_json(text: str): + cleaned = FENCE.sub("", text.strip()) + start, end = cleaned.find("{"), cleaned.rfind("}") + if start < 0 or end <= start: + return None + try: + return json.loads(cleaned[start : end + 1]) + except ValueError: + return None + + +def coverage(nodes: list[Node], spec: str) -> float: + if not nodes or not spec: + return 0.0 + try: + return sum(1 for node in nodes if field_text(node, spec)) / len(nodes) + except ValueError: + return 0.0 + + +class AiAssistant: + def enabled(self) -> bool: + return (get_setting("community_ai", "1") or "1") == "1" + + async def ask(self, system: str, user: str) -> dict | None: + if not self.enabled(): + return None + from devplacepy.services.correction import gateway_complete + + model = get_setting("community_ai_model", "") or INTERNAL_MODEL + reply, usage = await asyncio.to_thread( + gateway_complete, internal_gateway_key(), system, user, AI_TIMEOUT_SECONDS, None, model, True, False + ) + if usage is None: + logger.warning("ai: the %s model is unreachable, continuing with heuristics only", model) + return None + answer = parse_json(reply) + if answer is None: + logger.warning("ai: the model did not return JSON") + else: + logger.info("ai: %s answered %s", model, json.dumps(answer)[:300]) + return answer + + def validated(self, nodes: list[Node], proposal: dict | None, names: tuple[str, ...]) -> dict: + accepted: dict[str, str] = {} + for name in names: + spec = (proposal or {}).get(name) + if isinstance(spec, str) and coverage(nodes, spec) >= LIMITS.min_field_coverage: + accepted[name] = spec + elif spec: + logger.info("ai: rejected %s=%r, it matches too few sampled items", name, spec) + return accepted + + async def item_selector(self, root: Node) -> str | None: + answer = await self.ask( + "You analyse the HTML outline of a community listing page. Reply with JSON only: " + '{"item_selector": ""}. ' + + SELECTOR_SYNTAX, + outline(root), + ) + selector = (answer or {}).get("item_selector") + if isinstance(selector, str): + try: + if len(select_nodes(root, selector)) >= 3: + return selector + except ValueError: + pass + return None + + async def post_fields(self, nodes: list[Node]) -> dict: + samples = "\n\n".join(to_html(node) for node in nodes[:3]) + answer = await self.ask( + "You analyse HTML of community listing items. Reply with JSON only, keys: title, link, content, " + "author, created_at. Each value is a field selector relative to ONE item (omit a key when the item " + "does not contain it). link should end in ::attr(href), created_at in ::attr(datetime) or ::text. " + + SELECTOR_SYNTAX, + samples, + ) + return self.validated(nodes, answer, ("title", "link", "content", "author", "created_at")) + + async def comment_structure(self, root: Node) -> dict | None: + answer = await self.ask( + "You analyse the HTML outline of a community post page that has comments. Reply with JSON only: " + '{"item_selector": "", "id": "", "content": "...", ' + '"author": "...", "created_at": "..."}. Field selectors are relative to one comment. ' + SELECTOR_SYNTAX, + outline(root, OUTLINE_LINES * 2), + ) + selector = (answer or {}).get("item_selector") + if not isinstance(selector, str): + return None + try: + nodes = select_nodes(root, selector) + except ValueError: + return None + if not nodes: + return None + fields = self.validated(nodes, answer, ("id", "content", "author", "created_at")) + if "content" not in fields: + return None + return {"item_selector": selector, **fields} + + async def detail_content(self, root: Node) -> str | None: + answer = await self.ask( + "You analyse the HTML outline of a single community post page. Reply with JSON only: " + '{"content": ""}. ' + SELECTOR_SYNTAX, + outline(root), + ) + spec = (answer or {}).get("content") + if isinstance(spec, str) and len(field_text(root, spec)) > 40: + return spec + return None diff --git a/devplacepy/services/community/discoverer.py b/devplacepy/services/community/discoverer.py new file mode 100644 index 00000000..f25ecbfa --- /dev/null +++ b/devplacepy/services/community/discoverer.py @@ -0,0 +1,261 @@ +# retoor + +from __future__ import annotations + +import logging +import re +from urllib.parse import urljoin +from dataclasses import dataclass, field +from urllib.parse import urlsplit + +from devplacepy.services.community.ai import AiAssistant +from devplacepy.services.community.extract import field_text, find_login_form, parse_html, select_nodes +from devplacepy.services.community.feeds import FeedDetector +from devplacepy.services.community.learner import MetadataLearner, StructureLearner +from devplacepy.services.community.runtime import Collector, LoginFailed, LoginRequired +from devplacepy.services.community.search import SearchPhase +from devplacepy.services.community.spec import ( + LIMITS, + SCHEMA_VERSION, + STATUS_FAILED, + STATUS_LOGIN, + STATUS_READY, +) + +logger = logging.getLogger("community.discover") + +MAX_FEEDS = 4 + + +@dataclass +class Discovery: + query: str + status: str = STATUS_FAILED + name: str = "" + slug: str = "" + origin: str = "" + url: str = "" + schema: dict = field(default_factory=dict) + metadata: dict = field(default_factory=dict) + sample: dict = field(default_factory=dict) + error: str = "" + + +def origin_of(url: str) -> str: + host = (urlsplit(url).hostname or "").lower() + return host[4:] if host.startswith("www.") else host + + +def slug_of(origin: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", origin).strip("-")[:63] + + +def slug_for_query(query: str) -> str: + from devplacepy.services.community.search import direct_url + + address = direct_url(query) + return slug_of(origin_of(address)) if address else re.sub(r"[^a-z0-9]+", "-", query.lower()).strip("-")[:63] + + +LOGIN_PATHS = ("/login", "/signin", "/sign-in", "/accounts/login", "/users/sign_in", "/auth/login", "/user/login") +LOGIN_LABEL = re.compile(r"\b(log ?in|sign ?in)\b", re.I) +MAX_LOGIN_PAGES = 6 + + +class LoginLearner: + def __init__(self, fetcher): + self.fetcher = fetcher + + async def page(self, url: str, render: bool) -> str: + try: + if render: + status, text = await self.fetcher.render_html(url) + else: + response = await self.fetcher.get(url) + status, text = response.status, await response.text() + except Exception as exc: + logger.info("login: %s unreachable: %s: %s", url, type(exc).__name__, exc) + return "" + return text if status < 400 else "" + + async def find(self, base_url: str, html: str) -> dict | None: + spec = find_login_form(parse_html(html), base_url) if html else None + if spec: + return spec + urls = [] + if html: + for node in select_nodes(parse_html(html), "a[href]"): + if LOGIN_LABEL.search(node.text()[:30]): + urls.append(urljoin(base_url, node.attrs["href"])) + parts = urlsplit(base_url) + urls.extend(f"{parts.scheme}://{parts.netloc}{path}" for path in LOGIN_PATHS) + urls = list(dict.fromkeys(urls))[:MAX_LOGIN_PAGES] + for render in (False, True): + for url in urls[: MAX_LOGIN_PAGES if not render else 2]: + spec = find_login_form(parse_html(await self.page(url, render)), url) + if spec: + spec["render"] = render + logger.info("login: found a %s login form at %s%s", spec["mode"], url, " (needs JavaScript)" if render else "") + return spec + return None + + +class CommunityDiscoverer: + def __init__(self, fetcher, ai=None, credentials: dict | None = None): + self.fetcher = fetcher + self.ai = ai or AiAssistant() + self.credentials = credentials or {} + self.search = SearchPhase(fetcher) + self.detector = FeedDetector(fetcher, self.ai) + self.learner = StructureLearner(fetcher, self.ai) + self.metadata = MetadataLearner(fetcher) + self.logins = LoginLearner(fetcher) + + def has_credentials(self) -> bool: + return bool(self.credentials.get("password") and (self.credentials.get("username") or self.credentials.get("email"))) + + async def discover(self, query: str) -> Discovery: + result = Discovery(query=query) + logger.info("discover: starting for %r", query) + try: + candidates = await self.search.run(query) + except Exception as exc: + result.error = f"search failed: {exc}" + logger.error("discover: %s", result.error) + return result + if not candidates: + result.error = "search returned no usable candidates" + logger.warning("discover: %s", result.error) + return result + for candidate in candidates: + logger.info("discover: trying %s", candidate.url) + found = await self.try_candidate(query, candidate.url, result) + if found: + return result + if result.status == STATUS_LOGIN: + return result + result.error = result.error or "no candidate exposed a learnable feed" + logger.warning("discover: gave up: %s", result.error) + return result + + async def try_candidate(self, query: str, url: str, result: Discovery) -> bool: + origin = origin_of(url) + self.fetcher.session = f"community:{origin}" + candidates, context = await self.detector.detect(url) + walled = bool(context.get("login_required")) or not candidates + login = None + if walled or self.has_credentials(): + login = await self.logins.find(url, context.get("html", "")) + if login and self.has_credentials(): + probe = {"community": origin, "origin": origin, "base_url": url, "login": login, "limits": LIMITS.runtime()} + try: + await Collector(self.fetcher, probe, credentials=self.credentials).login() + logger.info("discover: logged in to %s", origin) + except LoginFailed as exc: + result.error, result.status = str(exc), STATUS_FAILED + logger.error("discover: %s", exc) + return False + if walled: + candidates, context = await self.detector.detect(url) + logger.info("discover: %d feed candidate(s) at %s", len(candidates), url) + learned = [] + for candidate in candidates: + if len(learned) >= MAX_FEEDS * 2: + break + feed = await self.learner.learn(candidate, len(learned)) + if feed: + learned.append((candidate, feed)) + feeds = self.distinct(learned) + home = await self.home_page(url, context.get("html", "")) + meta = await self.metadata.learn(home[0], home[1], {}) + result.url, result.origin, result.slug = url, origin, slug_of(origin) + result.metadata = meta + result.name = meta.get("name") or origin + wall = walled and login is not None + if not feeds: + if walled and login: + result.status = STATUS_LOGIN + result.error = "the community requires a login; add the account's username or email and password and relearn" if not self.has_credentials() else "no feed could be learned even after logging in" + result.schema = self.schema(result, [], login_required=True, login=login) + logger.warning("discover: %s", result.error) + else: + result.error = f"no feed could be learned at {url}" + return False + schema = self.schema(result, feeds, login_required=wall, login=login) + sample = await self.verify(schema) + if not sample["records"]["posts"]: + result.error = f"learned {len(feeds)} feed(s) but the verification run collected no posts" + logger.warning("discover: %s", result.error) + return False + result.schema, result.sample = schema, sample + result.status = STATUS_READY + logger.info( + "discover: ready %s: %d feed(s), sample %d users / %d posts / %d comments", + origin, len(feeds), *(len(sample["records"][kind]) for kind in ("users", "posts", "comments")), + ) + return True + + @staticmethod + def canonical(link: str) -> str: + parts = urlsplit(link) + return f"{(parts.hostname or '').removeprefix('www.')}{parts.path.rstrip('/')}".lower() + + def links_of(self, candidate, feed: dict) -> set[str]: + spec = feed["post"]["fields"].get("url") or feed["post"]["fields"].get("id") + return { + self.canonical(urljoin(candidate.url, field_text(item, spec))) + for item in candidate.items[: LIMITS.sample_posts] + if field_text(item, spec) + } + + def distinct(self, learned: list) -> list[dict]: + ranked = sorted(enumerate(learned), key=lambda entry: (entry[1][1]["comments"]["mode"] == "none", entry[0])) + kept: list[tuple[set, dict]] = [] + for _, (candidate, feed) in ranked: + links = self.links_of(candidate, feed) + twin = next((other for other, _ in kept if links and other and len(links & other) / min(len(links), len(other)) >= 0.5), None) + if twin is not None: + logger.info("discover: %s lists the same posts as a better feed, skipping it", feed["url"]) + continue + kept.append((links, feed)) + if len(kept) >= MAX_FEEDS: + break + feeds = [feed for _, feed in kept] + for index, feed in enumerate(feeds): + feed["id"] = f"feed{index + 1}" + return feeds + + async def home_page(self, url: str, html: str) -> tuple[str, str]: + if html.lstrip()[:1] == "<" and not html.lstrip().startswith(" dict: + schema = { + "version": SCHEMA_VERSION, + "community": result.name, + "origin": result.origin, + "base_url": result.url, + "login_required": login_required, + "limits": LIMITS.runtime(), + "feeds": feeds, + } + if login: + schema["login"] = login + return schema + + async def verify(self, schema: dict) -> dict: + trial = {**schema, "limits": {**schema["limits"], "max_pages": 1, "max_posts": LIMITS.sample_posts, "max_comment_posts": LIMITS.sample_comment_posts, "delay_seconds": 0.2}} + collector = Collector(self.fetcher, trial, credentials=self.credentials) + try: + return await collector.run() + except LoginRequired as exc: + logger.warning("discover: verification hit a login wall: %s", exc) + return {"records": {"users": [], "posts": [], "comments": []}, "errors": [str(exc)]} diff --git a/devplacepy/services/community/extract.py b/devplacepy/services/community/extract.py new file mode 100644 index 00000000..a0c9003e --- /dev/null +++ b/devplacepy/services/community/extract.py @@ -0,0 +1,469 @@ +# retoor + +from __future__ import annotations + +import json +import re +import xml.etree.ElementTree as ElementTree +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from html.parser import HTMLParser +from urllib.parse import urljoin + +PATH_TOKEN = re.compile(r"""\.\s*([^.\[\]]+)|\[\s*(\*|-?\d+|"[^"]*"|'[^']*')\s*\]""") +SELECTOR_PART = re.compile(r"^([a-zA-Z][a-zA-Z0-9-]*|\*)?((?:[#.][\w-]+|\[[^\]]+\])*)$") +SELECTOR_ATOM = re.compile(r"#([\w-]+)|\.([\w-]+)|\[\s*([\w:-]+)\s*(?:([~^$*|]?=)\s*[\"']?([^\"'\]]*)[\"']?)?\s*\]") +VOID_TAGS = frozenset( + {"br", "hr", "img", "meta", "link", "input", "source", "area", "base", "col", "embed", "track", "wbr"} +) +SKIP_TEXT_TAGS = frozenset({"script", "style", "noscript", "template"}) +EPOCH_MS_THRESHOLD = 100_000_000_000 +MAX_PATH_DEPTH = 24 +NOW_TOKEN = "$now" + + +def parse_path(path: str) -> list: + text = path.strip() + if not text.startswith("$"): + raise ValueError(f"path must start with $: {path!r}") + rest = text[1:] + tokens: list = [] + position = 0 + while position < len(rest): + match = PATH_TOKEN.match(rest, position) + if not match: + raise ValueError(f"cannot parse path {path!r} at {rest[position:]!r}") + key, bracket = match.groups() + if key is not None: + tokens.append(key.strip()) + elif bracket == "*": + tokens.append(None) + elif bracket[0] in "\"'": + tokens.append(bracket[1:-1]) + else: + tokens.append(int(bracket)) + position = match.end() + if len(tokens) > MAX_PATH_DEPTH: + raise ValueError(f"path too deep: {path!r}") + return tokens + + +def select(data, path: str) -> list: + current = [data] + for token in parse_path(path): + following = [] + for node in current: + if token is None: + if isinstance(node, list): + following.extend(node) + elif isinstance(node, dict): + following.extend(node.values()) + elif isinstance(token, int): + if isinstance(node, list) and -len(node) <= token < len(node): + following.append(node[token]) + elif isinstance(node, dict) and token in node: + following.append(node[token]) + elif isinstance(node, list): + following.extend( + item[token] for item in node if isinstance(item, dict) and token in item + ) + current = following + return current + + +def scalar_text(value) -> str: + if value is None: + return "" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + return value.strip() + if isinstance(value, dict): + for key in ("#text", "text", "value", "@href", "href", "url", "name"): + if key in value: + return scalar_text(value[key]) + return "" + if isinstance(value, list): + for entry in value: + text = scalar_text(entry) + if text: + return text + return "" + + +def text_at(data, path: str) -> str: + for value in select(data, path): + text = scalar_text(value) + if text: + return text + return "" + + +def strip_namespace(tag: str) -> str: + if "}" in tag: + tag = tag.split("}", 1)[1] + return tag + + +def element_value(element): + children = list(element) + attributes = {f"@{strip_namespace(key)}": value for key, value in element.attrib.items()} + text = (element.text or "").strip() + if not children and not attributes: + return text + entry = dict(attributes) + if text: + entry["#text"] = text + for child in children: + name = strip_namespace(child.tag) + value = element_value(child) + if name in entry: + if not isinstance(entry[name], list): + entry[name] = [entry[name]] + entry[name].append(value) + else: + entry[name] = value + return entry + + +def feed_kind(text: str) -> str | None: + head = text.lstrip()[:2048].lower() + if " tuple[str, dict, list[dict]]: + kind = feed_kind(text) + if kind is None: + raise ValueError("not an RSS or Atom document") + root = ElementTree.fromstring(text.encode("utf-8") if isinstance(text, str) else text) + channel = root.find("channel") if kind == "rss" else root + if channel is None: + channel = root + item_tag = "item" if kind == "rss" else "entry" + meta: dict = {} + items: list[dict] = [] + for child in list(channel) + (list(root) if channel is not root else []): + name = strip_namespace(child.tag) + if name == item_tag: + value = element_value(child) + if isinstance(value, dict): + items.append(value) + elif child is not channel and name in ("title", "description", "subtitle", "language", "link"): + meta.setdefault(name, element_value(child)) + return kind, meta, items + + +class Node: + __slots__ = ("tag", "attrs", "children", "parent", "chunks") + + def __init__(self, tag: str, attrs: dict, parent: "Node | None" = None): + self.tag = tag + self.attrs = attrs + self.children: list[Node] = [] + self.parent = parent + self.chunks: list = [] + + def text(self) -> str: + parts = [] + for chunk in self.chunks: + if isinstance(chunk, Node): + if chunk.tag not in SKIP_TEXT_TAGS: + parts.append(chunk.text()) + else: + parts.append(chunk) + return re.sub(r"\s+", " ", " ".join(parts)).strip() + + def classes(self) -> set[str]: + return set(self.attrs.get("class", "").split()) + + def descendants(self): + for child in self.children: + yield child + yield from child.descendants() + + +class TreeBuilder(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.root = Node("#root", {}) + self.current = self.root + + def handle_starttag(self, tag, attrs): + node = Node(tag, {key: (value or "") for key, value in attrs}, self.current) + self.current.children.append(node) + self.current.chunks.append(node) + if tag not in VOID_TAGS: + self.current = node + + def handle_startendtag(self, tag, attrs): + node = Node(tag, {key: (value or "") for key, value in attrs}, self.current) + self.current.children.append(node) + self.current.chunks.append(node) + + def handle_endtag(self, tag): + cursor = self.current + while cursor is not self.root and cursor.tag != tag: + cursor = cursor.parent + if cursor is not self.root: + self.current = cursor.parent + + def handle_data(self, data): + self.current.chunks.append(data) + + +def parse_html(text: str) -> Node: + builder = TreeBuilder() + builder.feed(text) + builder.close() + return builder.root + + +def compile_compound(part: str): + match = SELECTOR_PART.match(part) + if not match: + raise ValueError(f"unsupported selector: {part!r}") + tag, rest = match.groups() + ident = None + classes: list[str] = [] + attributes: list[tuple] = [] + for atom in SELECTOR_ATOM.finditer(rest): + atom_id, atom_class, name, operator, value = atom.groups() + if atom_id: + ident = atom_id + elif atom_class: + classes.append(atom_class) + else: + attributes.append((name, operator, value)) + return (None if tag in (None, "*") else tag.lower(), ident, classes, attributes) + + +def compound_matches(node: Node, compound) -> bool: + tag, ident, classes, attributes = compound + if tag and node.tag != tag: + return False + if ident and node.attrs.get("id") != ident: + return False + if classes and not set(classes) <= node.classes(): + return False + for name, operator, value in attributes: + if name not in node.attrs: + return False + actual = node.attrs[name] + if operator == "=" and actual != value: + return False + if operator == "^=" and not actual.startswith(value): + return False + if operator == "$=" and not actual.endswith(value): + return False + if operator == "*=" and value not in actual: + return False + if operator == "~=" and value not in actual.split(): + return False + return True + + +def parse_selector(selector: str) -> list[tuple[str, tuple]]: + spaced = re.sub(r"\s*>\s*", " > ", selector.strip()) + steps: list[tuple[str, tuple]] = [] + combinator = " " + for part in spaced.split(): + if part == ">": + combinator = ">" + continue + steps.append((combinator, compile_compound(part))) + combinator = " " + if not steps: + raise ValueError("empty selector") + return steps + + +def path_matches(node: Node, steps: list[tuple[str, tuple]], index: int) -> bool: + combinator, compound = steps[index] + if not compound_matches(node, compound): + return False + if index == 0: + return True + parent = node.parent + if combinator == ">": + return parent is not None and path_matches(parent, steps, index - 1) + while parent is not None and parent.tag != "#root": + if path_matches(parent, steps, index - 1): + return True + parent = parent.parent + return False + + +def select_nodes(root: Node, selector: str) -> list[Node]: + steps = parse_selector(selector) + return [node for node in root.descendants() if path_matches(node, steps, len(steps) - 1)] + + +def html_field(node: Node, spec: str) -> str: + selector, _, accessor = spec.partition("::") + target = node + if selector.strip(): + matches = select_nodes(node, selector) + if not matches: + return "" + target = matches[0] + accessor = accessor or "text" + if accessor == "text": + return target.text() + if accessor.startswith("attr(") and accessor.endswith(")"): + return target.attrs.get(accessor[5:-1], "").strip() + raise ValueError(f"unsupported accessor: {accessor!r}") + + +def field_text(item, spec) -> str: + specs = spec if isinstance(spec, (list, tuple)) else [spec] + for entry in specs: + if not entry: + continue + if entry == NOW_TOKEN: + return datetime.now(timezone.utc).isoformat() + text = html_field(item, entry) if isinstance(item, Node) else text_at(item, entry) + if text: + return text + return "" + + +def normalize_instant(value) -> str | None: + if value is None or value == "": + return None + if isinstance(value, (int, float)) or (isinstance(value, str) and value.strip().isdigit()): + number = float(value) + if number >= EPOCH_MS_THRESHOLD: + number /= 1000 + try: + return datetime.fromtimestamp(number, timezone.utc).isoformat() + except (OverflowError, OSError, ValueError): + return None + text = str(value).strip() + try: + moment = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + try: + moment = parsedate_to_datetime(text) + except (TypeError, ValueError): + return None + if moment.tzinfo is None: + moment = moment.replace(tzinfo=timezone.utc) + return moment.astimezone(timezone.utc).isoformat() + + +BLOCK_TAGS = frozenset( + {"p", "div", "section", "article", "li", "ul", "ol", "tr", "blockquote", "pre", "br", + "h1", "h2", "h3", "h4", "h5", "h6"} +) + + +def block_text(node: Node) -> str: + parts = [] + for chunk in node.chunks: + if isinstance(chunk, Node): + if chunk.tag in SKIP_TEXT_TAGS: + continue + inner = block_text(chunk) + parts.append(f"\n\n{inner}\n\n" if chunk.tag in BLOCK_TAGS else inner) + else: + parts.append(chunk) + return "".join(parts) + + +def strip_tags(text: str) -> str: + if "<" not in text: + return text.strip() + raw = block_text(parse_html(text)) + lines = [re.sub(r"[ \t]+", " ", line).strip() for line in raw.split("\n")] + return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip() + + +def json_or_none(text: str): + try: + return json.loads(text) + except ValueError: + return None + + +def leaf_paths(data, prefix: str = "$", depth: int = 0) -> dict[str, object]: + found: dict[str, object] = {} + if depth > 4: + return found + if isinstance(data, dict): + for key, value in data.items(): + if not re.fullmatch(r"[\w@#:-]+", str(key)): + continue + found.update(leaf_paths(value, f"{prefix}.{key}", depth + 1)) + elif isinstance(data, list): + if data and all(not isinstance(entry, (dict, list)) for entry in data[:3]): + found[prefix] = data + elif data: + found.update(leaf_paths(data[0], f"{prefix}[0]", depth + 1)) + else: + found[prefix] = data + return found + + +IDENTITY_NAME = re.compile(r"(user|login|email|mail|account|identifier|handle|name)", re.I) +SKIP_INPUT_TYPES = frozenset({"submit", "button", "image", "reset", "checkbox", "radio", "file"}) + + +def find_login_form(root: Node, base_url: str) -> dict | None: + for form in select_nodes(root, "form"): + inputs = [node for node in form.descendants() if node.tag == "input"] + passwords = [node for node in inputs if node.attrs.get("type", "").lower() == "password"] + if not passwords: + continue + password = passwords[0] + identity = next( + ( + node + for node in inputs + if node.attrs.get("type", "text").lower() in ("text", "email", "tel") + and node.attrs.get("name") + ), + None, + ) + if identity is None or not password.attrs.get("name"): + continue + defaults = { + node.attrs["name"]: node.attrs.get("value", "") + for node in inputs + if node.attrs.get("name") + and node.attrs.get("type", "text").lower() not in SKIP_INPUT_TYPES + and node is not password + and node is not identity + } + kind = identity.attrs.get("type", "text").lower() + name = identity.attrs["name"] + wants_email = kind == "email" or bool(re.search(r"mail", name, re.I)) + return { + "mode": "form", + "url": base_url, + "action": urljoin(base_url, form.attrs.get("action") or base_url), + "method": (form.attrs.get("method") or "post").upper(), + "fields": defaults, + "username_field": name, + "password_field": password.attrs["name"], + "identity": "email" if wants_email else "username", + "username_selector": f'input[name="{name}"]', + "password_selector": f'input[name="{password.attrs["name"]}"]', + "submit_selector": "button[type=submit]", + } + passwords = select_nodes(root, "input[type=password]") + if passwords: + return { + "mode": "page", + "url": base_url, + "identity": "username", + "username_selector": "input[type=email], input[type=text]", + "password_selector": "input[type=password]", + "submit_selector": "button[type=submit], input[type=submit]", + } + return None diff --git a/devplacepy/services/community/feeds.py b/devplacepy/services/community/feeds.py new file mode 100644 index 00000000..135e95d2 --- /dev/null +++ b/devplacepy/services/community/feeds.py @@ -0,0 +1,262 @@ +# retoor + +from __future__ import annotations + +import logging +import re +import statistics +from dataclasses import dataclass, field +from urllib.parse import urljoin, urlsplit + +from devplacepy.services.community.extract import ( + Node, + json_or_none, + parse_feed, + parse_html, + select_nodes, +) +from devplacepy.services.community.spec import ( + FEED_ATOM, + FEED_HTML, + FEED_JSON, + FEED_PROBE_PATHS, + FEED_RSS, + LIMITS, +) + +logger = logging.getLogger("community.feeds") + +ITEM_KEYS = re.compile( + r"^(title|name|subject|headline|content|body|text|description|summary|id|uid|guid|url|link|" + r"created_at|created|published|date|author|user|username|creator|content_html|excerpt)$", + re.I, +) +ITEM_HINT = re.compile(r"post|thread|topic|item|entry|article|story|rant|card|result|row", re.I) +ACTIVITY_LINK = re.compile(r"\b(latest|recent|new|activity|posts|threads|topics|discussions|feed)\b", re.I) +LOGIN_WORDS = re.compile(r"(sign in|log in|login|password)", re.I) +MIN_REPEAT = 3 +COMMENT_FEED = re.compile(r"/(comments?|repl(?:y|ies)|responses?)(?:[./_-][\w.]*)?/?$", re.I) + + +@dataclass +class FeedCandidate: + url: str + type: str + items: list = field(default_factory=list) + document: object = None + list_path: str | None = None + item_selector: str | None = None + login_required: bool = False + render: bool = False + + +def item_score(item) -> int: + if not isinstance(item, dict): + return 0 + return sum(1 for key in item if ITEM_KEYS.match(str(key).lstrip("@"))) + + +def find_item_list(document, path: str = "$", depth: int = 0): + best = (0, None, None) + if depth > 3: + return best + if isinstance(document, list): + dicts = [entry for entry in document if isinstance(entry, dict)] + if len(dicts) >= 2: + score = statistics.median(item_score(entry) for entry in dicts[:8]) * min(len(dicts), 20) + best = (score, path, document) + return best + if isinstance(document, dict): + for key, value in document.items(): + if isinstance(value, (list, dict)): + found = find_item_list(value, f"{path}.{key}", depth + 1) + if found[0] > best[0]: + best = found + return best + + +def find_item_selector(root: Node) -> tuple[str, list[Node]] | None: + best_score = 0.0 + best: tuple[str, list[Node]] | None = None + groups: dict[tuple, list[Node]] = {} + for node in root.descendants(): + if node.tag in ("script", "style", "nav", "header", "footer", "select", "option", "a", "span", "td", "th"): + continue + key = (node.tag, tuple(sorted(node.classes()))) + groups.setdefault(key, []).append(node) + for (tag, classes), nodes in groups.items(): + if len(nodes) < MIN_REPEAT: + continue + linked = [node for node in nodes if any(child.tag == "a" and child.attrs.get("href") for child in node.descendants())] + if len(linked) < MIN_REPEAT: + continue + lengths = [len(node.text()) for node in linked] + median = statistics.median(lengths) + if median < 25: + continue + selector = tag + "".join(f".{name}" for name in classes) + hint = 2.0 if ITEM_HINT.search(selector) else 1.0 + score = min(len(linked), 30) * min(median, 400) * hint + if score > best_score: + best_score = score + best = (selector, linked) + return best + + +def classify_text(url: str, text: str) -> FeedCandidate | None: + stripped = text.lstrip() + if stripped.startswith(("= 1: + return FeedCandidate(url, FEED_RSS if kind == "rss" else FEED_ATOM, items=items) + return None + if stripped[:1] in "[{": + document = json_or_none(text) + if document is None: + return None + score, path, items = find_item_list(document) + if path and score >= 4: + return FeedCandidate(url, FEED_JSON, items=items, document=document, list_path=path) + return None + return None + + +COMMENT_HINT = re.compile(r"comment|repl(?:y|ies)|response|answer", re.I) + + +def find_comment_selector(roots: list[Node]) -> str | None: + tallies: dict[str, int] = {} + for root in roots: + for node in root.descendants(): + if node.tag in ("script", "style", "a", "span", "form", "textarea", "input", "button", "ul", "ol", "section", "nav"): + continue + hinted = sorted(name for name in node.classes() if COMMENT_HINT.search(name) and not re.search(r"form|count|list|wrap|section|header|input|box|area|title|link|button", name, re.I)) + if not hinted: + continue + selector = f"{node.tag}.{hinted[0]}" + if len(node.text()) < 4: + continue + tallies[selector] = tallies.get(selector, 0) + 1 + ranked = sorted(tallies.items(), key=lambda entry: entry[1], reverse=True) + return ranked[0][0] if ranked and ranked[0][1] >= 2 else None + + +class FeedDetector: + def __init__(self, fetcher, ai=None): + self.fetcher = fetcher + self.ai = ai + + async def fetch(self, url: str): + try: + response = await self.fetcher.get(url) + return response.status, await response.text() + except Exception as exc: + logger.warning("detect: %s unreachable: %s: %s", url, type(exc).__name__, exc) + return 0, "" + + async def detect(self, base_url: str) -> tuple[list[FeedCandidate], dict]: + parts = urlsplit(base_url) + origin = f"{parts.scheme}://{parts.netloc}" + status, text = await self.fetch(base_url) + context = {"status": status, "html": text, "login_required": status in (401, 403)} + found: dict[str, FeedCandidate] = {} + direct = classify_text(base_url, text) if text else None + if direct: + found[base_url] = direct + root = parse_html(text) if text and not direct else None + queue: list[str] = [] + if root is not None: + queue.extend(self.advertised(root, base_url)) + queue.extend(self.activity_links(root, base_url)) + context["login_required"] = context["login_required"] or self.looks_like_login(root) + for url in dict.fromkeys(queue): + if len(found) < LIMITS.max_candidates and url not in found and not COMMENT_FEED.search(urlsplit(url).path): + await self.consider(url, found, context) + if not found: + for path in FEED_PROBE_PATHS[: LIMITS.max_probe_paths]: + await self.consider(urljoin(origin, path), found, context) + if not found and root is not None: + candidate = await self.html_with_ai(base_url, text, root) + if candidate: + found[base_url] = candidate + if not found and not direct: + rendered = await self.rendered(base_url) + if rendered: + context["html"] = rendered + candidate = await self.html_with_ai(base_url, rendered, parse_html(rendered)) + if candidate: + candidate.render = True + logger.info("detect: %s only renders with JavaScript, using the browser", base_url) + found[base_url] = candidate + return sorted(found.values(), key=self.preference), context + + async def consider(self, url: str, found: dict, context: dict) -> None: + status, body = await self.fetch(url) + if status == 401: + context["login_required"] = True + if status >= 400 or not body: + return + candidate = classify_text(url, body) + if candidate is None and status < 300: + candidate = self.html_candidate(url, body) + if candidate: + logger.info("detect: %s feed at %s (%d sample items)", candidate.type, url, len(candidate.items)) + found[url] = candidate + + async def rendered(self, url: str) -> str: + try: + status, html = await self.fetcher.render_html(url) + except Exception as exc: + logger.info("detect: rendering %s failed: %s: %s", url, type(exc).__name__, exc) + return "" + return html if status < 400 else "" + + async def html_with_ai(self, url: str, text: str, root: Node) -> FeedCandidate | None: + candidate = self.html_candidate(url, text) + if candidate or self.ai is None: + return candidate + selector = await self.ai.item_selector(root) + if not selector: + return None + return FeedCandidate(url, FEED_HTML, items=select_nodes(root, selector), document=root, item_selector=selector) + + def preference(self, candidate: FeedCandidate) -> int: + order = {FEED_JSON: 0, FEED_ATOM: 1, FEED_RSS: 2, FEED_HTML: 3} + return order[candidate.type] + + def advertised(self, root: Node, base_url: str) -> list[str]: + urls = [] + for node in select_nodes(root, "link[rel=alternate]"): + kind = node.attrs.get("type", "").lower() + if any(marker in kind for marker in ("rss", "atom", "json")) and node.attrs.get("href"): + urls.append(urljoin(base_url, node.attrs["href"])) + return urls + + def activity_links(self, root: Node, base_url: str) -> list[str]: + urls = [] + host = urlsplit(base_url).hostname + for node in select_nodes(root, "a[href]"): + label = node.text()[:40] + target = urljoin(base_url, node.attrs["href"]) + if urlsplit(target).hostname == host and ACTIVITY_LINK.search(label) and target != base_url: + urls.append(target) + return urls[: LIMITS.max_candidates] + + def html_candidate(self, url: str, text: str) -> FeedCandidate | None: + if "<" not in text[:2000]: + return None + root = parse_html(text) + found = find_item_selector(root) + if found is None: + return None + selector, nodes = found + return FeedCandidate(url, FEED_HTML, items=nodes, document=root, item_selector=selector) + + def looks_like_login(self, root: Node) -> bool: + if select_nodes(root, "input[type=password]") and not find_item_selector(root): + return True + return bool(LOGIN_WORDS.search(root.text()[:400])) and not find_item_selector(root) diff --git a/devplacepy/services/community/fetcher.py b/devplacepy/services/community/fetcher.py new file mode 100644 index 00000000..e8a4eee7 --- /dev/null +++ b/devplacepy/services/community/fetcher.py @@ -0,0 +1,153 @@ +# retoor + +from __future__ import annotations + +import asyncio +import logging +import time +from urllib.parse import urlsplit +from contextlib import asynccontextmanager + +import httpx + +from devplacepy.net_guard import guard_public_url, guarded_async_client +from devplacepy.services.community.spec import LIMITS + +logger = logging.getLogger("community.fetch") + +RENDER_SETTLE_MS = 800 + + +class SharedBrowser: + instance = None + loop = None + lock: asyncio.Lock | None = None + + @classmethod + async def acquire(cls): + running = asyncio.get_running_loop() + if cls.instance is not None and cls.loop is running: + return cls.instance + if cls.instance is not None: + logger.info("browser: event loop changed, starting a new Chromium") + cls.instance = None + from stealthii import Stealthii + + browser = Stealthii(request_timeout=LIMITS.timeout_seconds) + await browser.start() + cls.instance, cls.loop = browser, running + logger.info("browser: stealth Chromium started") + return browser + + @classmethod + async def close(cls) -> None: + if cls.instance is not None: + try: + await cls.instance.shutdown() + finally: + cls.instance = None + cls.loop = None + + +class GuardedPage: + def __init__(self, page): + self.page = page + + async def goto(self, url, **kwargs): + await guard_public_url(url) + return await self.page.goto(url, **kwargs) + + def __getattr__(self, name): + return getattr(self.page, name) + + +class FetchResponse: + def __init__(self, response: httpx.Response): + self.status = response.status_code + self.url = str(response.url) + self.headers = response.headers + self._response = response + + async def text(self) -> str: + return self._response.text + + async def read(self) -> bytes: + return self._response.content + + async def json(self): + return self._response.json() + + +class Fetcher: + def __init__(self): + self.browser = None + self.client: httpx.AsyncClient | None = None + self.session: str | None = None + self.last_request: dict[str, float] = {} + + async def polite(self, url: str) -> None: + host = urlsplit(url).hostname or "" + wait = LIMITS.delay_seconds - (time.monotonic() - self.last_request.get(host, 0.0)) + if wait > 0: + await asyncio.sleep(wait) + self.last_request[host] = time.monotonic() + + async def start(self) -> "Fetcher": + try: + self.browser = await SharedBrowser.acquire() + except Exception as exc: + logger.warning("browser: stealthii unavailable (%s: %s); falling back to plain HTTP, no JavaScript rendering", type(exc).__name__, exc) + self.client = guarded_async_client(follow_redirects=True, max_redirects=5, timeout=LIMITS.timeout_seconds) + return self + + async def shutdown(self) -> None: + if self.client is not None: + await self.client.aclose() + self.client = None + + async def __aenter__(self) -> "Fetcher": + return await self.start() + + async def __aexit__(self, *exc) -> None: + await self.shutdown() + + async def ready(self) -> None: + if self.browser is None and self.client is None: + await self.start() + + async def get(self, url: str, params: dict | None = None, timeout: float | None = None, session: str | None = None): + await self.ready() + await guard_public_url(url) + await self.polite(url) + logger.info("GET %s params=%s", url, params or {}) + if self.browser is not None: + response = await self.browser.get(url, params=params, timeout=timeout or LIMITS.timeout_seconds, session=session or self.session) + logger.info("GET %s -> %s", url, response.status) + return response + raw = await self.client.get(url, params=params, timeout=timeout or LIMITS.timeout_seconds) + logger.info("GET %s -> %s", url, raw.status_code) + return FetchResponse(raw) + + async def post(self, url: str, data: dict | None = None, timeout: float | None = None, session: str | None = None): + await self.ready() + await guard_public_url(url) + await self.polite(url) + logger.info("POST %s fields=%s", url, sorted((data or {}).keys())) + if self.browser is not None: + return await self.browser.post(url, data=data, timeout=timeout or LIMITS.timeout_seconds, session=session or self.session) + return FetchResponse(await self.client.post(url, data=data, timeout=timeout or LIMITS.timeout_seconds)) + + @asynccontextmanager + async def page(self, session: str | None = None): + await self.ready() + if self.browser is None: + raise RuntimeError("JavaScript rendering needs stealthii and its Chromium") + async with self.browser.page(session=session or self.session) as page: + yield GuardedPage(page) + + async def render_html(self, url: str, session: str | None = None) -> tuple[int, str]: + async with self.page(session=session) as page: + response = await page.goto(url, wait_until="domcontentloaded") + await page.wait_for_timeout(RENDER_SETTLE_MS) + logger.info("RENDER %s -> %s", url, response.status if response else "?") + return (response.status if response else 200), await page.content() diff --git a/devplacepy/services/community/files.py b/devplacepy/services/community/files.py new file mode 100644 index 00000000..28e89d4d --- /dev/null +++ b/devplacepy/services/community/files.py @@ -0,0 +1,40 @@ +# retoor + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import ModuleType + +from devplacepy import config +from devplacepy.services.community.generator import CodeGenerator + + +class CommunityFiles: + def __init__(self, root: Path | None = None): + self.root = root + + def directory(self, slug: str) -> Path: + return (self.root or config.COMMUNITIES_DIR) / slug + + def collectors_path(self, slug: str) -> Path: + return self.directory(slug) / "collectors.py" + + def write(self, slug: str, schema: dict) -> Path: + return CodeGenerator(schema).write(self.directory(slug)) + + def load(self, slug: str, schema: dict) -> ModuleType: + path = self.collectors_path(slug) + if not path.exists(): + self.write(slug, schema) + spec = importlib.util.spec_from_file_location(f"community_collectors_{slug.replace('-', '_')}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def read(self, slug: str) -> str: + path = self.collectors_path(slug) + return path.read_text() if path.exists() else "" + + +files = CommunityFiles() diff --git a/devplacepy/services/community/generator.py b/devplacepy/services/community/generator.py new file mode 100644 index 00000000..a9aaa529 --- /dev/null +++ b/devplacepy/services/community/generator.py @@ -0,0 +1,117 @@ +# retoor + +from __future__ import annotations + +import json +import re +from pathlib import Path + +from devplacepy.services.community import extract, runtime + +PACKAGE_IMPORT = re.compile(r"^from devplacepy\.services\.community\.extract import \([^)]*\)\n", re.M) +FUTURE_IMPORT = re.compile(r"^from __future__ import annotations\n", re.M) +HEADER_LINE = re.compile(r"^# retoor <[^>]*>\n", re.M) + +ENTRYPOINT = ''' + +async def collect_all(stealth, state=None, only=None, sleep=None, credentials=None): + collector = Collector(stealth, SCHEMA, state, sleep, credentials) + return await collector.run(only) +{wrappers} + +async def main(): + import os + import sys + from pathlib import Path + try: + from stealthii import Stealthii + except ImportError: + sys.exit("stealthii is required to run this file directly: pip install stealthii") + here = Path(__file__).resolve().parent + state_file = here / "last_run.json" + state = json.loads(state_file.read_text()) if state_file.exists() else None + stealth = Stealthii() + await stealth.start() + try: + credentials = {{ + "username": os.environ.get("COMMUNITY_USERNAME", ""), + "email": os.environ.get("COMMUNITY_EMAIL", ""), + "password": os.environ.get("COMMUNITY_PASSWORD", ""), + }} + result = await collect_all(stealth, state, credentials=credentials) + finally: + await stealth.shutdown() + state_file.write_text(json.dumps(result["state"], indent=2)) + (here / "collected.json").write_text(json.dumps(result["records"], indent=2)) + counts = {{kind: len(rows) for kind, rows in result["records"].items()}} + print(json.dumps({{"ok": result["ok"], "counts": counts, "errors": result["errors"]}}, indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) +''' + +WRAPPER = ''' + +async def collect_{name}(stealth, state=None, sleep=None, credentials=None): + # {kind} feed {url} + # post fields: {fields} + # comments: {comments} + # pagination: {pagination} + return await collect_all(stealth, state, only="{feed_id}", sleep=sleep, credentials=credentials) +''' + + +class CodeGenerator: + def __init__(self, schema: dict): + self.schema = schema + + @staticmethod + def module_source(module) -> str: + text = Path(module.__file__).read_text() + text = FUTURE_IMPORT.sub("", text) + text = HEADER_LINE.sub("", text) + return PACKAGE_IMPORT.sub("", text).strip("\n") + + def wrappers(self) -> str: + parts = [] + for feed in self.schema["feeds"]: + parts.append( + WRAPPER.format( + name=re.sub(r"\W", "_", feed["id"]), + kind=feed["type"], + url=feed["url"], + fields=json.dumps(feed["post"]["fields"]), + comments=(feed.get("comments") or {}).get("mode", "none"), + pagination=(feed.get("pagination") or {}).get("style", "none"), + feed_id=feed["id"], + ) + ) + return "".join(parts) + + def render(self) -> str: + schema_json = json.dumps(self.schema, indent=2, ensure_ascii=False).replace("'", "\\u0027") + header = ( + "# Generated by DevPlace community discovery. Community: " + f"{self.schema.get('community')} (origin {self.schema.get('origin')}).\n" + "# Self-contained: standard library plus stealthii when run directly.\n" + "# A login-protected community reads COMMUNITY_USERNAME / COMMUNITY_EMAIL / COMMUNITY_PASSWORD from the environment.\n" + "# Adjust SCHEMA below to change a selector or path; collect_all() reads it at runtime.\n" + "from __future__ import annotations\n\n" + ) + schema_block = f"\n\nSCHEMA = json.loads(r'''{schema_json}''')\n" + return ( + header + + self.module_source(extract) + + "\n\n" + + self.module_source(runtime) + + schema_block + + ENTRYPOINT.format(wrappers=self.wrappers()) + ) + + def write(self, directory: Path) -> Path: + directory.mkdir(parents=True, exist_ok=True) + target = directory / "collectors.py" + target.write_text(self.render()) + (directory / "schema.json").write_text(json.dumps(self.schema, indent=2, ensure_ascii=False)) + return target diff --git a/devplacepy/services/community/learner.py b/devplacepy/services/community/learner.py new file mode 100644 index 00000000..803294d8 --- /dev/null +++ b/devplacepy/services/community/learner.py @@ -0,0 +1,581 @@ +# retoor + +from __future__ import annotations + +import logging +import re +import statistics +from urllib.parse import urljoin, urlsplit + +from devplacepy.services.community.extract import ( + NOW_TOKEN, + Node, + field_text, + html_field, + json_or_none, + leaf_paths, + normalize_instant, + parse_html, + scalar_text, + select, + select_nodes, + strip_tags, + text_at, +) +from devplacepy.services.community.feeds import FeedCandidate, find_comment_selector, find_item_list +from devplacepy.services.community.spec import ( + FEED_HTML, + FEED_JSON, + LIMITS, +) + +logger = logging.getLogger("community.learn") + +PATTERNS = { + "id": r"^(id|uid|guid|post_id|postid|uuid|slug|key|@id|@rdf:about)$", + "url": r"^(url|link|permalink|href|@href|html_url|canonical_url|web_url|share_url)$", + "title": r"^(title|headline|subject|topic_title|name)$", + "content": r"^(content_html|content|body_html|body|text|selftext|message|encoded|raw|cooked|description|summary|excerpt)$", + "created_at": r"^(created_at|created|createdat|published|published_at|pubdate|date|timestamp|created_utc|posted_at|time|dateadded)$", + "updated_at": r"^(updated_at|updated|modified|edited_at|last_edited|bumped_at|last_activity)$", + "parent_id": r"^(parent_id|parent|in_reply_to|reply_to|parent_comment_id|parentid)$", +} +AUTHOR_KEYS = re.compile(r"^(author|user|creator|poster|by|owner|dc:creator|created_by|member|profile)$", re.I) +AUTHOR_NAME = re.compile(r"^(username|user_name|login|handle|name|display_name|screen_name|nickname|nick)$", re.I) +AUTHOR_ID = re.compile(r"^(id|uid|user_id|userid|uuid)$", re.I) +COMMENT_KEYS = re.compile(r"^(comments|replies|children|answers|responses|posts)$", re.I) +COMMENT_URL_KEYS = re.compile(r"^(comments_url|comment_url|comments_link|replies_url|comments_href)$", re.I) +REPLY_KEYS = re.compile(r"^(replies|children|answers|responses)$", re.I) +NEXT_KEYS = re.compile(r"^(next|next_page|next_url|next_page_url|nextpage)$", re.I) +CURSOR_KEYS = re.compile(r"^(next_cursor|cursor|after|next_token|nextcursor|continuation)$", re.I) +PAGE_PARAMS = ("page", "p", "paged") +USER_COUNT = re.compile(r"([\d][\d,.\s]{0,12}\d|\d)\s*(?:\+\s*)?(members|users|developers|registered|people)\b", re.I) +RULE_PATHS = ("/rules", "/about", "/faq", "/guidelines", "/terms") +COMMENT_PROBES = ("{feed}/{id}/comments", "{feed}/{id}", "/api/posts/{id}/comments", "/api/comments?post_id={id}") + + +def last_segment(path: str) -> str: + return re.split(r"[.\[]", path.rstrip("]"))[-1] if path else "" + + +class FieldInferrer: + def __init__(self, samples: list, coverage: float | None = None): + self.samples = samples + self.coverage = LIMITS.min_field_coverage if coverage is None else coverage + self.leaves = [leaf_paths(sample) for sample in samples] + self.paths = list(dict.fromkeys(path for leaf in self.leaves for path in leaf)) + + def values(self, path: str) -> list[str]: + return [text_at(sample, path) for sample in self.samples] + + def coverage_of(self, path: str, validator=None) -> float: + values = self.values(path) + good = [value for value in values if value and (validator is None or validator(value))] + return len(good) / len(self.samples) if self.samples else 0.0 + + def candidates(self, pattern: str) -> list[str]: + regex = re.compile(pattern, re.I) + return [path for path in self.paths if regex.match(last_segment(path).lstrip("@")) or regex.match(last_segment(path))] + + def pick_all(self, name: str, validator=None, longest: bool = False) -> list[str]: + scored = [] + for index, path in enumerate(self.candidates(PATTERNS[name])): + cover = self.coverage_of(path, validator) + if cover < self.coverage: + continue + median = statistics.median(len(value) for value in self.values(path)) if longest else 0 + scored.append((-median if longest else 0, index, path)) + scored.sort() + return [path for _, _, path in scored] + + def post_fields(self) -> dict: + valid_date = lambda value: normalize_instant(value) is not None + fields = { + "id": self.pick_all("id") or self.pick_all("url"), + "url": self.pick_all("url"), + "title": self.pick_all("title")[:1], + "content": self.pick_all("content", longest=True)[:3], + "created_at": self.pick_all("created_at", valid_date)[:1], + "updated_at": self.pick_all("updated_at", valid_date)[:1], + } + if not fields["created_at"]: + fields["created_at"] = [NOW_TOKEN] + return {key: value[:1] if key in ("id", "url") else value for key, value in fields.items() if value} + + def author_spec(self) -> dict: + regex_keys = [path for path in self.paths if AUTHOR_KEYS.match(last_segment(path))] + found: dict = {} + objects = [path for path in self.paths if AUTHOR_NAME.match(last_segment(path)) and any(AUTHOR_KEYS.match(part) for part in re.split(r"[.\[\]]", path)[:-1])] + objects.sort(key=lambda path: (len(path), path)) + for path in objects: + if self.coverage_of(path) >= self.coverage: + found["username"] = [path] + base = path.rsplit(".", 1)[0] + for other in self.paths: + if other.startswith(base + ".") and AUTHOR_ID.match(last_segment(other)) and self.coverage_of(other) >= self.coverage: + found["id"] = [other] + break + return {"fields": found} + for path in regex_keys: + if self.coverage_of(path) >= self.coverage: + return {"fields": {"username": [path]}} + flat = [path for path in self.paths if re.match(r"^\$\.(username|author_name|user|author_username)$", path)] + for path in flat: + if self.coverage_of(path) >= self.coverage: + return {"fields": {"username": [path]}} + return {"fields": {}} + + def list_paths(self, pattern) -> list[str]: + found = [] + for path in self.paths: + if "[0]" not in path: + continue + candidate = path.split("[0]")[0] + if pattern.match(last_segment(candidate)) and candidate not in found: + found.append(candidate) + return found + + def comment_lists(self) -> list[str]: + return [ + path + for path in self.list_paths(COMMENT_KEYS) + if any(isinstance(match, list) and match and isinstance(match[0], dict) for sample in self.samples for match in select(sample, path)) + ] + + +class HtmlInferrer: + def __init__(self, nodes: list[Node], base_url: str): + self.nodes = nodes + self.base_url = base_url + + def share(self, spec: str) -> float: + hits = sum(1 for node in self.nodes if field_text(node, spec)) + return hits / len(self.nodes) if self.nodes else 0.0 + + def first(self, specs: list[str]) -> str | None: + for spec in specs: + if self.share(spec) >= LIMITS.min_field_coverage: + return spec + return None + + def post_fields(self) -> dict: + link = self.first(["h1 a::attr(href)", "h2 a::attr(href)", "h3 a::attr(href)", "a.title::attr(href)", "a[href]::attr(href)"]) + title = self.first(["h1", "h2", "h3", "a.title", "a[href]"]) + content = self.first(["p", ".content", ".body", ".excerpt", ".summary", "::text"]) + created = self.first(["time::attr(datetime)", "time", "[datetime]::attr(datetime)"]) + fields: dict = {} + if link: + fields["id"] = [link] + fields["url"] = [link] + if title: + fields["title"] = [title] + if content: + fields["content"] = [content, "::text"] if content != "::text" else [content] + fields["created_at"] = [created] if created else [NOW_TOKEN] + return fields + + @staticmethod + def weak(post: dict, author: dict) -> bool: + return ( + not author.get("fields") + or "title" not in post + or post.get("content") in (None, ["::text"]) + or post.get("created_at") == [NOW_TOKEN] + ) + + @staticmethod + def merge(post: dict, author: dict, proposal: dict) -> tuple[dict, dict]: + post, author = dict(post), dict(author) + if proposal.get("title"): + post["title"] = [proposal["title"]] + if proposal.get("link"): + post["id"] = post["url"] = [proposal["link"]] + if proposal.get("content"): + post["content"] = [proposal["content"], "::text"] + if proposal.get("created_at"): + post["created_at"] = [proposal["created_at"]] + if proposal.get("author") and not author.get("fields"): + author = {"fields": {"username": [proposal["author"]]}} + return post, author + + def author_spec(self) -> dict: + name = self.first([".author", ".username", ".user", "a[rel=author]", "[itemprop=author]", ".byline"]) + return {"fields": {"username": [name]} if name else {}} + + +class PaginationProbe: + def __init__(self, fetcher): + self.fetcher = fetcher + + def from_document(self, candidate: FeedCandidate) -> dict | None: + document = candidate.document + if isinstance(candidate.document, Node): + for spec in ("a[rel=next]::attr(href)", "link[rel=next]::attr(href)", "a.next::attr(href)", ".pagination .next a::attr(href)"): + if html_field(candidate.document, spec): + return {"style": "next_link", "next": spec} + return None + if isinstance(document, dict): + leaves = leaf_paths(document) + for path, value in leaves.items(): + if NEXT_KEYS.match(last_segment(path)) and isinstance(value, str) and value.startswith(("http", "/")): + return {"style": "next_link", "next": path} + for path, value in leaves.items(): + if CURSOR_KEYS.match(last_segment(path)) and value: + return {"style": "cursor", "cursor_path": path, "param": last_segment(path).replace("next_", "")} + return None + + def signature(self, items: list) -> str: + return "|".join( + (item.text()[:60] if isinstance(item, Node) else scalar_text(select(item, "$.id")) or str(item)[:60]) + for item in items[:3] + ) + + async def probe(self, candidate: FeedCandidate, load_items) -> dict: + declared = self.from_document(candidate) + if declared: + return declared + baseline = self.signature(candidate.items) + feed_like = candidate.type in ("rss", "atom") + for param in PAGE_PARAMS[:1] if feed_like else PAGE_PARAMS: + try: + other = await load_items(candidate, {param: 2}) + except Exception as exc: + logger.info("pagination probe %s=2 failed: %s", param, exc) + continue + if other and self.signature(other) != baseline: + start = 1 + return {"style": "page", "param": param, "start": start} + per_page = len(candidate.items) + if per_page and not feed_like: + for param in ("offset", "skip"): + try: + other = await load_items(candidate, {param: per_page}) + except Exception: + continue + if other and self.signature(other) != baseline: + return {"style": "offset", "param": param, "per_page": per_page} + return {"style": "none"} + + +class CommentProbe: + def __init__(self, fetcher): + self.fetcher = fetcher + + def comment_fields(self, entries: list) -> dict: + inferrer = FieldInferrer(entries) + fields = { + "id": inferrer.pick_all("id")[:1], + "content": inferrer.pick_all("content", longest=True)[:2], + "created_at": (inferrer.pick_all("created_at", lambda v: normalize_instant(v) is not None)[:1] or [NOW_TOKEN]), + "parent_id": inferrer.pick_all("parent_id")[:1], + "updated_at": inferrer.pick_all("updated_at", lambda v: normalize_instant(v) is not None)[:1], + } + spec = {"fields": {key: value for key, value in fields.items() if value}, "author": inferrer.author_spec()} + for path in inferrer.list_paths(REPLY_KEYS): + if any(isinstance(match, list) and match for sample in entries for match in select(sample, path)): + spec["replies_path"] = path + break + return spec + + def usable(self, spec: dict) -> bool: + return "id" in spec["fields"] and "content" in spec["fields"] + + async def learn(self, candidate: FeedCandidate, post_fields: dict, inferrer: FieldInferrer | None) -> dict: + if candidate.type != FEED_JSON or inferrer is None or not candidate.items: + return {"mode": "none", "reason": f"comments are not learned for {candidate.type} feeds"} + for path in inferrer.comment_lists(): + entries = [entry for sample in candidate.items for match in select(sample, path) if isinstance(match, list) for entry in match if isinstance(entry, dict)] + if len(entries) >= 1: + spec = self.comment_fields(entries) + if self.usable(spec): + return {"mode": "inline", "list_path": path, **spec} + id_path = (post_fields.get("id") or [None])[0] + sample = candidate.items[0] + identifier = text_at(sample, id_path) if id_path else "" + for path in inferrer.paths: + if COMMENT_URL_KEYS.match(last_segment(path)): + target = text_at(sample, path) + if target and identifier and identifier in target: + template = urljoin(candidate.url, target.replace(identifier, "{id}")) + result = await self.fetch_comments(template, identifier, candidate) + if result: + return result + if not identifier: + return {"mode": "none", "reason": "posts expose no id to build a comments address"} + feed = candidate.url.split("?")[0].rstrip("/") + for probe in COMMENT_PROBES[: LIMITS.max_probe_paths]: + template = probe.replace("{feed}", feed) + template = template if template.startswith("http") else urljoin(candidate.url, template) + result = await self.fetch_comments(template, identifier, candidate) + if result: + return result + return {"mode": "none", "reason": "no comments source found"} + + async def fetch_comments(self, template: str, identifier: str, candidate: FeedCandidate) -> dict | None: + try: + response = await self.fetcher.get(template.replace("{id}", identifier)) + if response.status >= 400: + return None + document = json_or_none(await response.text()) + except Exception as exc: + logger.info("comments probe %s failed: %s", template, exc) + return None + if document is None: + return None + score, path, entries = find_item_list(document) + if not path or not entries: + return None + spec = self.comment_fields([entry for entry in entries if isinstance(entry, dict)]) + if not self.usable(spec): + return None + return {"mode": "url", "url_template": template, "list_path": path, **spec} + + +class PageProbe: + CONTENT_SELECTORS = ("article", "[itemprop=articleBody]", ".post-content", ".entry-content", ".post-body", ".markdown-body", ".content", "main") + BODY_SELECTORS = (".comment-body", ".comment-content", ".comment-text", ".comment__body", ".body", ".content", ".text", "p") + TIME_SELECTORS = ("time::attr(datetime)", "[datetime]::attr(datetime)", "time", ".date", ".timestamp") + AUTHOR_SELECTORS = (".comment-author", ".author", ".username", ".user", "a[rel=author]", "[itemprop=author]", ".fn", ".byline") + + def __init__(self, fetcher, ai=None): + self.fetcher = fetcher + self.ai = ai + + async def load(self, link: str, render: bool): + try: + if render: + status, text = await self.fetcher.render_html(link) + else: + response = await self.fetcher.get(link) + status, text = response.status, await response.text() + except Exception as exc: + logger.info("pages: %s unreachable: %s: %s", link, type(exc).__name__, exc) + return None + return parse_html(text) if status < 400 and text else None + + def links(self, candidate: FeedCandidate, post: dict) -> list[str]: + found = [] + for item in candidate.items: + target = field_text(item, post.get("url")) + if target: + absolute = urljoin(candidate.url, target) + if absolute not in found: + found.append(absolute) + if len(found) >= LIMITS.sample_comment_posts + 1: + break + return found + + @staticmethod + def first_spec(nodes: list[Node], specs: tuple[str, ...], minimum: int = 0) -> str | None: + for spec in specs: + hits = [field_text(node, spec) for node in nodes] + if sum(1 for hit in hits if len(hit) > minimum) / max(len(nodes), 1) >= LIMITS.min_field_coverage: + return spec + return None + + def comment_spec(self, roots: list[Node], selector: str, render: bool) -> dict | None: + nodes = [node for root in roots for node in select_nodes(root, selector)] + if not nodes: + return None + fields: dict = {} + content = self.first_spec(nodes, self.BODY_SELECTORS, 3) + if not content: + return None + fields["content"] = [content] + for name, specs in (("id", ("::attr(id)", "::attr(data-id)")), ("created_at", self.TIME_SELECTORS)): + spec = self.first_spec(nodes, specs) + if spec: + fields[name] = [spec] + author = self.first_spec(nodes, self.AUTHOR_SELECTORS) + return { + "mode": "page", + "item_selector": selector, + "fields": fields, + "author": {"fields": {"username": [author]} if author else {}}, + "render": render, + } + + def ai_comment_spec(self, proposal: dict, render: bool) -> dict: + fields = {"content": [proposal["content"]]} + for name in ("id", "created_at"): + if proposal.get(name): + fields[name] = [proposal[name]] + return { + "mode": "page", + "item_selector": proposal["item_selector"], + "fields": fields, + "author": {"fields": {"username": [proposal["author"]]} if proposal.get("author") else {}}, + "render": render, + } + + async def learn(self, candidate: FeedCandidate, post: dict) -> tuple[dict, dict | None]: + none = {"mode": "none", "reason": "no comments found on the post pages"} + if not post.get("url"): + return {"mode": "none", "reason": "posts expose no address to open"}, None + links = self.links(candidate, post) + render = candidate.render + roots = [root for root in [await self.load(link, render) for link in links[: LIMITS.sample_comment_posts]] if root is not None] + if not roots: + return {"mode": "none", "reason": "post pages could not be opened"}, None + selector = find_comment_selector(roots) + if not selector and not render: + rendered = [root for root in [await self.load(link, True) for link in links[: LIMITS.sample_comment_posts]] if root is not None] + selector = find_comment_selector(rendered) + if selector: + roots, render = rendered, True + logger.info("pages: comments only appear after JavaScript rendering") + comments = none + if selector: + comments = self.comment_spec(roots, selector, render) or none + elif self.ai is not None: + proposal = await self.ai.comment_structure(roots[0]) + if proposal: + comments = self.ai_comment_spec(proposal, render) + detail = await self.detail(candidate, post, roots, render) + if comments["mode"] == "page": + logger.info("pages: comments via %s (%s)", comments["item_selector"], sorted(comments["fields"])) + return comments, detail + + async def detail(self, candidate: FeedCandidate, post: dict, roots: list[Node], render: bool) -> dict | None: + listed = statistics.median(len(field_text(item, post.get("content"))) for item in candidate.items[: LIMITS.sample_posts]) if post.get("content") else 0 + chosen = None + for spec in self.CONTENT_SELECTORS: + lengths = [len(field_text(root, spec)) for root in roots] + if lengths and statistics.median(lengths) >= 80: + chosen = spec + break + if chosen is None and self.ai is not None: + chosen = await self.ai.detail_content(roots[0]) + if chosen is None: + return None + gain = statistics.median(len(field_text(root, chosen)) for root in roots) + if gain <= listed * 1.2: + return None + logger.info("pages: full text from %s on the post page (listing shows %d chars, page %d)", chosen, listed, gain) + return {"content": [chosen], "render": render} + + +class MetadataLearner: + def __init__(self, fetcher): + self.fetcher = fetcher + + def from_html(self, html: str, base_url: str) -> dict: + if not html: + return {} + root = parse_html(html) + meta: dict = {} + + def pick(selector: str, accessor: str = "::attr(content)") -> str: + nodes = select_nodes(root, selector) + return html_field(nodes[0], accessor) if nodes else "" + + meta["name"] = pick("meta[property=og:site_name]") or (select_nodes(root, "title")[0].text() if select_nodes(root, "title") else "") + meta["description"] = pick("meta[property=og:description]") or pick("meta[name=description]") + icon = pick("link[rel=icon]", "::attr(href)") or pick("link[rel=shortcut]", "::attr(href)") + meta["favicon"] = urljoin(base_url, icon) if icon else "" + html_nodes = select_nodes(root, "html") + meta["language"] = html_nodes[0].attrs.get("lang", "") if html_nodes else "" + counted = USER_COUNT.search(root.text()) + if counted: + digits = re.sub(r"[^\d]", "", counted.group(1)) + meta["user_count"] = int(digits) if digits else None + return {key: value for key, value in meta.items() if value} + + async def rules(self, base_url: str) -> str: + parts = urlsplit(base_url) + origin = f"{parts.scheme}://{parts.netloc}" + for path in RULE_PATHS[:3]: + try: + response = await self.fetcher.get(origin + path) + except Exception: + continue + if response.status >= 400: + continue + text = strip_tags(await response.text()) + if len(text) > 120: + logger.info("metadata: rules text from %s (%d chars)", path, len(text)) + return text[:2000] + return "" + + async def learn(self, base_url: str, html: str, feed_meta: dict) -> dict: + meta = self.from_html(html, base_url) + for key in ("name", "description", "language"): + if not meta.get(key) and feed_meta.get(key if key != "name" else "title"): + meta[key] = scalar_text(feed_meta[key if key != "name" else "title"]) + if not meta.get("user_count"): + try: + response = await self.fetcher.get(urljoin(base_url, "/about")) + if response.status < 400: + counted = USER_COUNT.search(strip_tags(await response.text())) + if counted: + meta["user_count"] = int(re.sub(r"[^\d]", "", counted.group(1)) or 0) or None + except Exception as exc: + logger.info("metadata: about page unavailable: %s", exc) + meta["rules"] = await self.rules(base_url) + return {key: value for key, value in meta.items() if value} + + +class StructureLearner: + def __init__(self, fetcher, ai=None): + self.fetcher = fetcher + self.ai = ai + self.pagination = PaginationProbe(fetcher) + self.comments = CommentProbe(fetcher) + self.pages = PageProbe(fetcher, ai) + + async def load_items(self, candidate: FeedCandidate, params: dict) -> list: + from devplacepy.services.community.runtime import Collector + + collector = Collector(self.fetcher, {"feeds": [], "limits": {**LIMITS.runtime(), "delay_seconds": 0, "retries": 1}}) + text = await collector.fetch(candidate.url, params) + _, items = collector.parse(self.as_feed(candidate, {}, {}, {}), text) + return items + + def as_feed(self, candidate: FeedCandidate, post: dict, author: dict, pagination: dict) -> dict: + feed = { + "id": "probe", + "url": candidate.url, + "type": candidate.type, + "post": {"fields": post}, + "author": author, + "pagination": pagination, + } + if candidate.render: + feed["render"] = True + if candidate.list_path: + feed["list_path"] = candidate.list_path + if candidate.item_selector: + feed["item_selector"] = candidate.item_selector + return feed + + async def learn(self, candidate: FeedCandidate, index: int) -> dict | None: + samples = candidate.items[: LIMITS.sample_posts] + inferrer = None + if candidate.type == FEED_HTML: + html = HtmlInferrer(samples, candidate.url) + post, author = html.post_fields(), html.author_spec() + if self.ai is not None and html.weak(post, author): + logger.info("learn: heuristics are weak for %s, asking the AI model", candidate.url) + post, author = html.merge(post, author, await self.ai.post_fields(samples)) + else: + inferrer = FieldInferrer(samples) + post, author = inferrer.post_fields(), inferrer.author_spec() + if "id" not in post or not ({"title", "content"} & set(post)): + logger.info("learn: %s rejected, no stable id or no title/content (%s)", candidate.url, sorted(post)) + return None + pagination = await self.pagination.probe(candidate, self.load_items) + comments = await self.comments.learn(candidate, post, inferrer) + detail = None + if comments["mode"] == "none": + comments, detail = await self.pages.learn(candidate, post) + feed = self.as_feed(candidate, post, author, pagination) + if detail: + feed["post"]["detail"] = detail + feed["id"] = f"feed{index + 1}" + feed["comments"] = comments + feed["post"]["source_footer"] = True + logger.info( + "learn: %s as %s: fields=%s author=%s pagination=%s comments=%s", + candidate.url, candidate.type, sorted(post), sorted(author.get("fields", {})), pagination["style"], comments["mode"], + ) + return feed diff --git a/devplacepy/services/community/logbook.py b/devplacepy/services/community/logbook.py new file mode 100644 index 00000000..637c7f2f --- /dev/null +++ b/devplacepy/services/community/logbook.py @@ -0,0 +1,39 @@ +# retoor + +from __future__ import annotations + +import logging +import sys +from collections import deque +from datetime import datetime, timezone + +ROOT = "community" +MAX_LINES = 400 + + +class LogBook(logging.Handler): + def __init__(self, echo: bool = False, level: int = logging.INFO): + super().__init__(level) + self.lines: deque[str] = deque(maxlen=MAX_LINES) + self.echo = echo + self.setFormatter(logging.Formatter("%(message)s")) + + def emit(self, record: logging.LogRecord) -> None: + stamp = datetime.now(timezone.utc).strftime("%H:%M:%S") + name = record.name.split(".", 1)[-1] if "." in record.name else record.name + line = f"{stamp} {record.levelname:<7} [{name}] {self.format(record)}" + self.lines.append(line) + if self.echo: + print(line, file=sys.stderr, flush=True) + + def __enter__(self) -> "LogBook": + logger = logging.getLogger(ROOT) + logger.addHandler(self) + logger.setLevel(self.level) + return self + + def __exit__(self, *exc) -> None: + logging.getLogger(ROOT).removeHandler(self) + + def text(self) -> str: + return "\n".join(self.lines) diff --git a/devplacepy/services/community/pusher.py b/devplacepy/services/community/pusher.py new file mode 100644 index 00000000..8da73f41 --- /dev/null +++ b/devplacepy/services/community/pusher.py @@ -0,0 +1,73 @@ +# retoor + +from __future__ import annotations + +import logging +from collections import Counter +from dataclasses import dataclass, field + +from devplacepy.services.community.spec import LIMITS +from devplacepy.services.sync.batch import BatchRunner +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KINDS, STATUSES + +logger = logging.getLogger("community.push") + + +@dataclass +class PushReport: + dry_run: bool + counts: dict = field(default_factory=dict) + problems: list = field(default_factory=list) + + def total(self, status: str) -> int: + return sum(per.get(status, 0) for per in self.counts.values()) + + +DEPENDENCIES = { + "unknown_author": ("users", "author_uid"), + "unknown_post": ("posts", "post_uid"), + "unknown_parent": ("comments", "parent_uid"), +} + + +class SyncPusher: + def __init__(self, origin: str, dry_run: bool = False): + self.origin = origin + self.dry_run = dry_run + + @staticmethod + def satisfied_by_projection(result, raw: dict, projected: dict) -> bool: + if result.status != "rejected" or result.reason not in DEPENDENCIES: + return False + kind, field_name = DEPENDENCIES[result.reason] + return str(raw.get(field_name)) in projected[kind] + + def push(self, records: dict[str, list[dict]]) -> PushReport: + report = PushReport(dry_run=self.dry_run) + mode = "DRY RUN" if self.dry_run else "LIVE" + projected: dict[str, set[str]] = {kind: set() for kind in KINDS} + for kind in KINDS: + rows = records.get(kind, []) + tally: Counter = Counter() + logger.info("push[%s]: %s %d %s record(s) in batches of %d", mode, self.origin, len(rows), kind, LIMITS.batch_records) + for start in range(0, len(rows), LIMITS.batch_records): + chunk = rows[start : start + LIMITS.batch_records] + outcome = BatchRunner(kind).run(SyncBatchForm(origin=self.origin, records=chunk, dry_run=self.dry_run)) + tally.update(outcome.counts) + for result, raw in zip(outcome.results, chunk): + if result.status == "created": + projected[kind].add(result.origin_uid) + if self.dry_run and self.satisfied_by_projection(result, raw, projected): + tally["rejected"] -= 1 + tally["created"] += 1 + projected[kind].add(result.origin_uid) + logger.info("push[%s]: %s %s depends on a record this dry run would create; counted as created", mode, kind, result.origin_uid) + continue + if result.status in ("rejected", "failed"): + line = f"{kind} {result.origin_uid}: {result.status} {result.reason} {'; '.join(result.errors[:2])}".strip() + report.problems.append(line) + logger.warning("push[%s]: %s", mode, line) + report.counts[kind] = {status: tally.get(status, 0) for status in STATUSES if tally.get(status, 0)} + logger.info("push[%s]: %s result %s", mode, kind, report.counts[kind] or "nothing to send") + return report diff --git a/devplacepy/services/community/runner.py b/devplacepy/services/community/runner.py new file mode 100644 index 00000000..835831d3 --- /dev/null +++ b/devplacepy/services/community/runner.py @@ -0,0 +1,232 @@ +# retoor + +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass, field + +from devplacepy.services.sync.aliaser import AliasRefused, Aliaser +from devplacepy.services.sync.options import SyncOptions, alias_allowed +from devplacepy.services.sync.selection import Selector +from devplacepy.services.community.discoverer import CommunityDiscoverer, Discovery +from devplacepy.services.community.fetcher import Fetcher +from devplacepy.services.community.files import CommunityFiles, files as default_files +from devplacepy.services.community.logbook import LogBook +from devplacepy.services.community.pusher import SyncPusher +from devplacepy.services.community.runtime import LoginRequired +from devplacepy.services.community.spec import ( + LIMITS, + RUN_DRY, + RUN_FAILED, + RUN_OK, + RUN_PARTIAL, + STATUS_LOGIN, + STATUS_READY, +) +from devplacepy.services.community.store import CommunityStore, iso, now, store as default_store +from devplacepy.services.sync.gate import refusal + +logger = logging.getLogger("community.run") + +TRIGGER_CLI = "cli" +TRIGGER_ADMIN = "admin" +TRIGGER_SERVICE = "service" +KINDS = ("users", "posts", "comments") + + +@dataclass +class RunReport: + status: str + dry_run: bool + collected: dict = field(default_factory=dict) + pushed: dict = field(default_factory=dict) + errors: list = field(default_factory=list) + log: str = "" + run_uid: str = "" + watermark: dict = field(default_factory=dict) + + +class CommunityRunner: + def __init__(self, fetcher_factory=Fetcher, store: CommunityStore | None = None, files: CommunityFiles | None = None, sleep=None): + self.fetcher_factory = fetcher_factory + self.sleep = sleep + self.store = store or default_store + self.files = files or default_files + + async def discover( + self, + query: str, + created_by: str, + *, + dry_run: bool = False, + relearn_slug: str | None = None, + credentials: dict | None = None, + ) -> tuple[Discovery, dict | None, str]: + with LogBook() as book: + logger.info("discover%s: %r", " (dry run, nothing is saved)" if dry_run else "", query) + existing = self.store.get(relearn_slug) if relearn_slug else None + if existing and not dry_run: + self.store.claim(existing["uid"]) + secrets = credentials or (self.store.credentials(existing) if existing else {}) + try: + async with self.fetcher_factory() as fetcher: + result = await asyncio.wait_for( + CommunityDiscoverer(fetcher, credentials=secrets).discover(query), timeout=LIMITS.run_timeout_seconds + ) + except asyncio.TimeoutError: + result = Discovery(query=query, error=f"discovery exceeded {LIMITS.run_timeout_seconds}s") + logger.error("discover: %s", result.error) + except Exception as exc: + logger.exception("discover: unexpected failure") + result = Discovery(query=query, error=f"{type(exc).__name__}: {exc}") + row = None + if not dry_run and (result.slug or relearn_slug): + row = self.persist(result, query, created_by, relearn_slug, credentials) + logger.info("discover: saved community %s with status %s", result.slug, result.status) + elif dry_run: + logger.info("discover: dry run, would save %s with status %s (%s feed(s))", result.slug, result.status, len(result.schema.get("feeds", []))) + return result, row, book.text() + + def persist(self, result: Discovery, query: str, created_by: str, relearn_slug: str | None, credentials: dict | None = None) -> dict: + slug = relearn_slug or result.slug + existing = self.store.get(slug) + learned = result.status in (STATUS_READY, STATUS_LOGIN) + if existing and existing["status"] == STATUS_READY and not learned: + logger.warning("discover: relearn failed, keeping the working schema of %s: %s", slug, result.error) + return self.store.update(existing["uid"], last_error=result.error[:1000]) + fields = { + "name": result.name or existing and existing.get("name") or slug, + "origin": result.origin, + "url": result.url, + "status": result.status, + "description": result.metadata.get("description", ""), + "rules": result.metadata.get("rules", ""), + "language": result.metadata.get("language", ""), + "favicon": result.metadata.get("favicon", ""), + "user_count": int(result.metadata.get("user_count") or 0), + "schema": result.schema, + "learned_at": iso(now()) if learned else "", + "last_error": result.error, + "failure_count": 0, + } + if credentials and any(credentials.values()): + fields.update( + login_username=credentials.get("username", ""), + login_email=credentials.get("email", ""), + login_password=credentials.get("password", ""), + ) + if existing: + row = self.store.update(existing["uid"], **fields, enabled=1 if result.status == STATUS_READY else 0) + else: + row = self.store.create(slug, query, created_by, **fields) + if result.schema.get("feeds"): + self.files.write(slug, result.schema) + return row + + async def run(self, row: dict, *, dry_run: bool, trigger: str) -> RunReport: + run_uid = self.store.start_run(row["uid"], trigger, dry_run) + report = RunReport(status=RUN_FAILED, dry_run=dry_run, run_uid=run_uid) + with LogBook() as book: + mode = "DRY RUN" if dry_run else "LIVE" + logger.info("run[%s]: %s (%s) trigger=%s", mode, row["name"], row["origin"], trigger) + try: + await asyncio.wait_for(self.execute(row, report), timeout=LIMITS.run_timeout_seconds) + except asyncio.TimeoutError: + report.errors.append(f"run exceeded {LIMITS.run_timeout_seconds}s") + except LoginRequired as exc: + report.errors.append(str(exc)) + if not dry_run: + self.store.update(row["uid"], status=STATUS_LOGIN, enabled=0, last_error=str(exc)) + except Exception as exc: + logger.exception("run[%s]: unexpected failure", mode) + report.errors.append(f"{type(exc).__name__}: {exc}") + for error in report.errors: + logger.error("run[%s]: %s", mode, error) + logger.info("run[%s]: finished with status %s", mode, report.status) + report.log = book.text() + if not dry_run: + self.settle(row, report) + self.store.finish_run(run_uid, report.status, collected=report.collected, pushed=report.pushed, errors=report.errors, log=report.log) + self.store.prune_runs(row["uid"]) + return report + + async def execute(self, row: dict, report: RunReport) -> None: + refused = refusal(row["origin"]) + if refused and not report.dry_run: + report.errors.append(refused) + return + if refused: + logger.warning("run[DRY RUN]: %s; continuing because nothing is written", refused) + module = self.files.load(row["slug"], row["schema"]) + async with self.fetcher_factory() as fetcher: + result = await module.collect_all(fetcher, row.get("watermark") or None, sleep=self.sleep, credentials=self.store.credentials(row)) + records = result["records"] + report.collected = {kind: len(records[kind]) for kind in KINDS} + logger.info("run: collected %s", report.collected) + report.errors.extend(result.get("errors", [])) + if not result["ok"]: + report.errors.append("no feed could be collected") + return + options = SyncOptions.from_row(row) + prepared = await self.prepare(row, options, records, report) + if prepared is None: + return + records, skipped, held = prepared + push = await asyncio.to_thread(SyncPusher(row["origin"], dry_run=report.dry_run).push, records) + report.pushed = push.counts | ({"skipped": dict(skipped)} if skipped else {}) + report.errors.extend(push.problems[:20]) + report.watermark = {} if held else result["state"] + if held: + report.errors.append(f"{held} item(s) held back for a transient reason and will be retried") + logger.warning("run: %d item(s) were skipped for a transient reason, the watermark is held so the next run retries them", held) + failed = push.total("failed") + report.status = RUN_DRY if report.dry_run else (RUN_PARTIAL if report.errors or failed else RUN_OK) + + async def prepare(self, row: dict, options: SyncOptions, records: dict, report: RunReport): + records, skipped = Selector(row["origin"], options).select(records) + held = 0 + if options.alias or options.translate_to: + if options.alias and not alias_allowed(): + report.errors.append("identity aliasing is switched off: enable community_alias_enabled on the Community sync service") + return None + try: + outcome = await Aliaser(row["origin"], options, language=row.get("language") or "", dry_run=report.dry_run).apply(records) + except AliasRefused as exc: + report.errors.append(str(exc)) + return None + records, held = outcome.records, outcome.transient() + skipped.update(outcome.skipped) + logger.info("alias: %d substitution(s), %d translated, %d new identities", outcome.substitutions, outcome.translated, outcome.identities_created) + if skipped: + logger.info("select: skipped %s", dict(skipped)) + return records, skipped, held + + def settle(self, row: dict, report: RunReport) -> None: + current = self.store.get(row["uid"]) + if current is None: + return + if report.status in (RUN_OK, RUN_PARTIAL): + totals = dict(current.get("totals") or {}) + for kind, counts in report.pushed.items(): + for status, number in counts.items(): + key = f"{kind}_{status}" + totals[key] = int(totals.get(key, 0)) + number + self.store.update( + row["uid"], + watermark=report.watermark or current.get("watermark") or {}, + totals=totals, + run_count=int(current.get("run_count") or 0) + 1, + failure_count=0, + last_status=report.status, + last_error="; ".join(report.errors)[:1000], + last_run_at=iso(now()), + ) + self.store.schedule_next(row["uid"], int(current.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(current).jitter_percent) + elif current["status"] != STATUS_LOGIN: + failed = self.store.fail(current, "; ".join(report.errors) or "run failed") + self.store.schedule_next(row["uid"], int(failed.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(failed).jitter_percent) + logger.error( + "run: failure %d of %d for %s. Structure drift is likely: re-run discovery with relearn.", + failed["failure_count"], LIMITS.max_failures, row["slug"], + ) diff --git a/devplacepy/services/community/runtime.py b/devplacepy/services/community/runtime.py new file mode 100644 index 00000000..86d529ed --- /dev/null +++ b/devplacepy/services/community/runtime.py @@ -0,0 +1,487 @@ +# retoor + +from __future__ import annotations + +import asyncio +import hashlib +import json +import re +from datetime import datetime, timezone +from urllib.parse import urljoin + +from devplacepy.services.community.extract import ( + Node, + field_text, + find_login_form, + html_field, + json_or_none, + normalize_instant, + parse_feed, + parse_html, + select, + select_nodes, + strip_tags, + text_at, +) + +UID_PATTERN = re.compile(r"^[A-Za-z0-9._:@-]+$") +UID_MAX_LENGTH = 64 +MIN_POST_LENGTH = 10 +MIN_COMMENT_LENGTH = 3 +MAX_REPLY_DEPTH = 5 +RETRY_STATUSES = frozenset({408, 425, 429, 500, 502, 503, 504}) +RENDER_SETTLE_MS = 800 +DEFAULT_LIMITS = { + "max_pages": 3, + "max_posts": 150, + "max_comment_posts": 25, + "comment_recheck": 10, + "delay_seconds": 1.0, + "retries": 3, + "backoff_seconds": 2.0, + "timeout_seconds": 30, + "max_bytes": 5_000_000, +} + + +class CollectError(Exception): + pass + + +class LoginRequired(CollectError): + pass + + +class LoginFailed(LoginRequired): + pass + + +class PageMissing(CollectError): + pass + + +def uid_for(value) -> str: + text = str(value if value is not None else "").strip() + if text and len(text) <= UID_MAX_LENGTH and UID_PATTERN.match(text): + return text + return "h:" + hashlib.sha256(text.encode()).hexdigest()[:32] + + +def iso_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def absolute_id(value: str, base: str) -> str: + return urljoin(base, value) if value.startswith("/") else value + + +def flatten_matches(matches: list) -> list: + if len(matches) == 1 and isinstance(matches[0], list): + return matches[0] + return matches + + +class Collector: + def __init__(self, stealth, schema: dict, state: dict | None = None, sleep=None, credentials: dict | None = None): + self.stealth = stealth + self.schema = schema + self.state = {"feeds": dict((state or {}).get("feeds", {}))} + self.sleep = sleep or asyncio.sleep + self.credentials = credentials or {} + self.limits = {**DEFAULT_LIMITS, **schema.get("limits", {})} + self.session = f"community:{schema.get('origin') or 'default'}" + self.requests = 0 + self.users: dict[str, dict] = {} + self.posts: dict[str, dict] = {} + self.comments: dict[str, dict] = {} + self.pages: dict[str, Node] = {} + self.errors: list[str] = [] + + async def attempt(self, label: str, action) -> str: + last = "no attempt" + for attempt in range(int(self.limits["retries"])): + if self.requests: + await self.sleep(float(self.limits["delay_seconds"])) + self.requests += 1 + try: + status, text = await action() + if status in (401, 407): + raise LoginRequired(f"{label} answered {status}: login required") + if status in RETRY_STATUSES: + last = f"status {status}" + elif status in (404, 410): + raise PageMissing(f"{label} answered {status}") + elif status >= 400: + raise CollectError(f"{label} answered {status}") + else: + return text[: int(self.limits["max_bytes"])] + except CollectError: + raise + except Exception as exc: + last = f"{type(exc).__name__}: {exc}" + await self.sleep(float(self.limits["backoff_seconds"]) * (2**attempt)) + raise CollectError(f"{label} failed after {self.limits['retries']} attempts ({last})") + + async def fetch(self, url: str, params: dict | None = None, render: bool = False) -> str: + if render: + return await self.attempt(url, lambda: self.render_html(url)) + + async def action(): + response = await self.stealth.get(url, params=params, timeout=self.limits["timeout_seconds"], session=self.session) + return int(response.status), await response.text() + + return await self.attempt(url, action) + + async def post(self, url: str, data: dict) -> str: + async def action(): + response = await self.stealth.post(url, data=data, timeout=self.limits["timeout_seconds"], session=self.session) + return int(response.status), await response.text() + + return await self.attempt(url, action) + + async def render_html(self, url: str): + async with self.stealth.page(session=self.session) as page: + response = await page.goto(url, wait_until="domcontentloaded") + await page.wait_for_timeout(RENDER_SETTLE_MS) + return (response.status if response else 200), await page.content() + + async def page_root(self, url: str, render: bool) -> Node: + key = f"{int(render)}|{url}" + if key not in self.pages: + self.pages[key] = parse_html(await self.fetch(url, render=render)) + return self.pages[key] + + def has_credentials(self) -> bool: + return bool(self.credentials.get("password") and (self.credentials.get("username") or self.credentials.get("email"))) + + async def login(self) -> bool: + spec = self.schema.get("login") + if not spec or not self.has_credentials(): + return False + identity = self.credentials.get("email") if spec.get("identity") == "email" else self.credentials.get("username") + identity = identity or self.credentials.get("username") or self.credentials.get("email") + render = bool(spec.get("render")) + if spec.get("mode") == "form": + root = parse_html(await self.fetch(spec["url"], render=render)) + form = find_login_form(root, spec["url"]) or spec + data = {**form.get("fields", {}), form["username_field"]: identity, form["password_field"]: self.credentials["password"]} + if str(form.get("method", "POST")).upper() == "GET": + await self.fetch(form["action"], params=data) + else: + await self.post(form["action"], data) + else: + async with self.stealth.page(session=self.session) as page: + await page.goto(spec["url"], wait_until="domcontentloaded") + await page.fill(spec["username_selector"], identity) + await page.fill(spec["password_selector"], self.credentials["password"]) + await page.click(spec["submit_selector"]) + await page.wait_for_timeout(RENDER_SETTLE_MS * 2) + after = parse_html(await self.fetch(spec.get("check_url") or self.schema.get("base_url") or spec["url"], render=render)) + if select_nodes(after, "input[type=password]"): + raise LoginFailed(f"{self.schema.get('community', 'community')} rejected the stored credentials") + return True + + def parse(self, feed: dict, text: str): + kind = feed["type"] + if kind == "json_api": + document = json_or_none(text) + if document is None: + raise CollectError(f"{feed['url']} did not return JSON") + return document, flatten_matches(select(document, feed.get("list_path") or "$")) + if kind in ("rss", "atom"): + _, meta, items = parse_feed(text) + return {"meta": meta}, items + root = parse_html(text) + return root, select_nodes(root, feed["item_selector"]) + + def next_request(self, feed: dict, index: int, document, current: str): + pagination = feed.get("pagination") or {"style": "none"} + style = pagination.get("style", "none") + if style == "page": + return feed["url"], {pagination["param"]: int(pagination.get("start", 1)) + index} + if style == "offset": + return feed["url"], {pagination["param"]: index * int(pagination["per_page"])} + if style == "cursor": + cursor = text_at(document, pagination["cursor_path"]) if isinstance(document, (dict, list)) else "" + return (feed["url"], {pagination["param"]: cursor}) if cursor else None + if style == "next_link": + target = html_field(document, pagination["next"]) if isinstance(document, Node) else text_at(document, pagination["next"]) + return (urljoin(current, target), None) if target else None + return None + + def stamp_of(self, feed: dict, item) -> str | None: + fields = feed["post"]["fields"] + return normalize_instant(field_text(item, fields.get("updated_at")) or field_text(item, fields.get("created_at"))) + + def post_link(self, feed: dict, item, base: str) -> str: + link = field_text(item, feed["post"]["fields"].get("url")) + return urljoin(base, link) if link else "" + + def post_identifier(self, feed: dict, item, base: str) -> str: + raw = field_text(item, feed["post"]["fields"].get("id")) + return absolute_id(raw, base) if raw else self.post_link(feed, item, base) + + async def collect_feed(self, feed: dict) -> dict: + feed_state = self.state["feeds"].get(feed["id"], {}) + since = feed_state.get("since") + newest = since + limit = int(self.limits["max_posts"]) + render = bool(feed.get("render")) + fresh: list[tuple] = [] + recent: list[tuple] = [] + url, params = feed["url"], None + seen: set[str] = set() + stats = {"pages": 0, "items": 0, "fresh": 0} + for index in range(int(self.limits["max_pages"])): + try: + text = await self.fetch(url, params, render=render) + except PageMissing: + if index == 0: + raise + stats["ended"] = "no further pages" + break + document, items = self.parse(feed, text) + stats["pages"] += 1 + if not items: + break + page_old = bool(since) + added = 0 + for item in items: + stats["items"] += 1 + stamp = self.stamp_of(feed, item) + key = self.post_identifier(feed, item, url) or json.dumps(item, sort_keys=True, default=str)[:200] + if key in seen: + continue + seen.add(key) + added += 1 + if len(recent) < int(self.limits["comment_recheck"]): + recent.append((item, url)) + if stamp is None or since is None or stamp > since: + page_old = False + fresh.append((item, url)) + if stamp and (newest is None or stamp > newest): + newest = stamp + if len(fresh) >= limit: + break + if len(fresh) >= limit or page_old or not added: + break + following = self.next_request(feed, index + 1, document, url) + if following is None: + break + url, params = following + stats["fresh"] = len(fresh) + details = await self.fetch_details(feed, fresh) + for item, base in fresh: + self.add_post(feed, item, base, details.get(self.post_link(feed, item, base))) + await self.collect_comments(feed, fresh, recent) + self.state["feeds"][feed["id"]] = {"since": newest or since or iso_now()} + return stats + + async def fetch_details(self, feed: dict, fresh: list) -> dict[str, Node]: + detail = feed["post"].get("detail") + found: dict[str, Node] = {} + if not detail: + return found + for item, base in fresh[: int(self.limits["max_comment_posts"])]: + link = self.post_link(feed, item, base) + if not link: + continue + try: + found[link] = await self.page_root(link, bool(feed.get("render") or detail.get("render"))) + except PageMissing: + continue + except LoginRequired: + raise + except CollectError as exc: + self.errors.append(f"detail {link}: {exc}") + return found + + def add_post(self, feed: dict, item, base: str, detail: Node | None = None) -> str | None: + fields = feed["post"]["fields"] + link = self.post_link(feed, item, base) + identifier = self.post_identifier(feed, item, base) + if not identifier: + return None + title = strip_tags(field_text(item, fields.get("title"))) + content = strip_tags(field_text(item, fields.get("content"))) + if detail is not None: + full = strip_tags(field_text(detail, feed["post"]["detail"]["content"])) + if len(full) > len(content): + content = full + if feed["post"].get("source_footer", True) and link: + content = f"{content}\n\nSource: {link}".strip() + if len(content) < MIN_POST_LENGTH: + content = f"{title}\n\n{content}".strip() + if len(content) < MIN_POST_LENGTH: + return None + created = normalize_instant(field_text(item, fields.get("created_at"))) or iso_now() + updated = normalize_instant(field_text(item, fields.get("updated_at"))) + post_uid = uid_for(identifier) + record = { + "origin_uid": post_uid, + "author_uid": self.add_author(feed.get("author") or {}, item, created), + "title": title, + "content": content, + "created_at": created, + } + if updated: + record["updated_at"] = updated + self.posts[post_uid] = record + return post_uid + + def add_author(self, spec: dict, item, moment: str) -> str: + fields = spec.get("fields", {}) + username = strip_tags(field_text(item, fields.get("username"))) + identifier = field_text(item, fields.get("id")) + if not username and not identifier: + username, identifier = "anonymous", "anonymous" + user_uid = uid_for(identifier or f"name:{username}") + joined = normalize_instant(field_text(item, fields.get("created_at"))) or moment + existing = self.users.get(user_uid) + if existing: + existing["created_at"] = min(existing["created_at"], joined) + return user_uid + self.users[user_uid] = {"origin_uid": user_uid, "username": username or identifier, "created_at": joined} + return user_uid + + async def collect_comments(self, feed: dict, fresh: list, recent: list) -> None: + spec = feed.get("comments") or {"mode": "none"} + mode = spec.get("mode", "none") + if mode == "none": + return + targets: dict[str, tuple] = {} + for item, base in [*fresh, *recent]: + identifier = self.post_identifier(feed, item, base) + if identifier: + targets.setdefault(uid_for(identifier), (item, base, identifier)) + for post_uid, (item, base, identifier) in list(targets.items())[: int(self.limits["max_comment_posts"])]: + try: + if mode == "inline": + self.add_comments(spec, flatten_matches(select(item, spec["list_path"])), post_uid, None, 0) + elif mode == "url": + target = spec["url_template"].replace("{id}", identifier).replace("{uid}", post_uid) + document = json_or_none(await self.fetch(urljoin(base, target), render=bool(spec.get("render")))) + entries = flatten_matches(select(document, spec.get("list_path") or "$")) if document is not None else [] + self.add_comments(spec, entries, post_uid, None, 0) + elif mode == "page": + link = self.post_link(feed, item, base) + if link: + root = await self.page_root(link, bool(feed.get("render") or spec.get("render"))) + self.add_page_comments(spec, root, post_uid) + except PageMissing: + continue + except LoginRequired: + raise + except CollectError as exc: + self.errors.append(f"comments {identifier}: {exc}") + + def add_comments(self, spec: dict, entries: list, post_uid: str, parent_uid: str | None, depth: int) -> None: + fields = spec["fields"] + for entry in entries: + if not isinstance(entry, dict): + continue + content = strip_tags(field_text(entry, fields.get("content"))) + identifier = field_text(entry, fields.get("id")) + if not identifier or len(content) < MIN_COMMENT_LENGTH: + continue + created = normalize_instant(field_text(entry, fields.get("created_at"))) or iso_now() + declared = field_text(entry, fields.get("parent_id")) + parent = parent_uid or (uid_for(declared) if declared else None) + self.store_comment(spec, entry, uid_for(identifier), post_uid, parent, content, created) + replies = spec.get("replies_path") + if replies and depth < MAX_REPLY_DEPTH: + self.add_comments(spec, flatten_matches(select(entry, replies)), post_uid, uid_for(identifier), depth + 1) + + def comment_identifier(self, spec: dict, node: Node, post_uid: str, content: str) -> str: + declared = field_text(node, spec["fields"].get("id")) + if declared: + return declared + author = strip_tags(field_text(node, (spec.get("author") or {}).get("fields", {}).get("username"))) + return hashlib.sha256(f"{post_uid}|{author}|{content[:120]}".encode()).hexdigest()[:24] + + def add_page_comments(self, spec: dict, root: Node, post_uid: str) -> None: + fields = spec["fields"] + nodes = select_nodes(root, spec["item_selector"]) + known = {id(node) for node in nodes} + uids: dict[int, str] = {} + for node in nodes: + content = strip_tags(field_text(node, fields.get("content"))) + if len(content) < MIN_COMMENT_LENGTH: + continue + uids[id(node)] = uid_for(self.comment_identifier(spec, node, post_uid, content)) + for node in nodes: + if id(node) not in uids: + continue + ancestor = node.parent + while ancestor is not None and id(ancestor) not in known: + ancestor = ancestor.parent + parent = uids.get(id(ancestor)) if ancestor is not None else None + content = strip_tags(field_text(node, fields.get("content"))) + created = normalize_instant(field_text(node, fields.get("created_at"))) or iso_now() + self.store_comment(spec, node, uids[id(node)], post_uid, parent, content, created) + + def store_comment(self, spec: dict, entry, comment_uid: str, post_uid: str, parent: str | None, content: str, created: str) -> None: + record = { + "origin_uid": comment_uid, + "author_uid": self.add_author(spec.get("author", {}), entry, created), + "post_uid": post_uid, + "content": content, + "created_at": created, + } + if parent and parent != comment_uid: + record["parent_uid"] = parent + updated = normalize_instant(field_text(entry, spec["fields"].get("updated_at"))) + if updated: + record["updated_at"] = updated + self.comments[comment_uid] = record + + def ordered_comments(self) -> list[dict]: + ordered: list[dict] = [] + placed: set[str] = set() + pending = list(self.comments.values()) + while pending: + progress = [record for record in pending if record.get("parent_uid") not in self.comments or record.get("parent_uid") in placed] + if not progress: + progress = pending + for record in progress: + if record.get("parent_uid") not in self.comments: + record = {key: value for key, value in record.items() if key != "parent_uid"} + ordered.append(record) + placed.add(record["origin_uid"]) + done = {record["origin_uid"] for record in progress} + pending = [record for record in pending if record["origin_uid"] not in done] + return ordered + + async def run(self, only: str | None = None) -> dict: + community = self.schema.get("community", "community") + spec = self.schema.get("login") + if self.schema.get("login_required") and not (spec and self.has_credentials()): + raise LoginRequired(f"{community} requires a login; add the account's username or email and password to the community") + if spec and self.has_credentials(): + await self.login() + reports = {} + for feed in self.schema["feeds"]: + if only and feed["id"] != only: + continue + try: + reports[feed["id"]] = await self.collect_feed(feed) + except LoginRequired: + raise + except CollectError as exc: + self.errors.append(f"{feed['id']}: {exc}") + reports[feed["id"]] = {"error": str(exc)} + failed = [name for name, report in reports.items() if "error" in report] + return { + "community": community, + "origin": self.schema.get("origin"), + "collected_at": iso_now(), + "records": { + "users": list(self.users.values()), + "posts": list(self.posts.values()), + "comments": self.ordered_comments(), + }, + "state": self.state, + "feeds": reports, + "errors": self.errors, + "ok": bool(reports) and len(failed) < len(reports), + } diff --git a/devplacepy/services/community/search.py b/devplacepy/services/community/search.py new file mode 100644 index 00000000..e1059d69 --- /dev/null +++ b/devplacepy/services/community/search.py @@ -0,0 +1,99 @@ +# retoor + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass +from urllib.parse import urlsplit + +from devplacepy.services.community.extract import json_or_none +from devplacepy.services.community.spec import LIMITS, RSEARCH_URL + +logger = logging.getLogger("community.search") + +NOISE_HOSTS = ( + "linkedin.com", + "facebook.com", + "twitter.com", + "x.com", + "apps.apple.com", + "play.google.com", + "wikipedia.org", + "youtube.com", + "instagram.com", + "tiktok.com", + "pinterest.com", +) + + +@dataclass +class Candidate: + url: str + title: str + description: str + score: float = 0.0 + + @property + def host(self) -> str: + return urlsplit(self.url).hostname or "" + + +def words(text: str) -> list[str]: + return [word for word in re.split(r"[^a-z0-9]+", text.lower()) if len(word) > 1] + + +def direct_url(query: str) -> str | None: + text = query.strip() + if re.match(r"^https?://\S+$", text): + return text + if re.match(r"^[a-z0-9-]+(\.[a-z0-9-]+)+(/\S*)?$", text.lower()): + return f"https://{text}" + return None + + +def rank(query: str, results: list[dict]) -> list[Candidate]: + terms = words(query) + ranked = [] + for entry in results: + url = str(entry.get("url") or "") + host = (urlsplit(url).hostname or "").lower() + if not host or any(host == noise or host.endswith("." + noise) for noise in NOISE_HOSTS): + continue + title = str(entry.get("title") or "") + description = str(entry.get("description") or "") + score = 0.0 + for term in terms: + if term in host: + score += 3.0 + if term in title.lower(): + score += 1.5 + if term in description.lower(): + score += 0.5 + if urlsplit(url).path in ("", "/"): + score += 1.0 + ranked.append(Candidate(url, title, description, score)) + ranked.sort(key=lambda candidate: candidate.score, reverse=True) + return ranked + + +class SearchPhase: + def __init__(self, fetcher): + self.fetcher = fetcher + + async def run(self, query: str) -> list[Candidate]: + direct = direct_url(query) + if direct: + logger.info("search: %r is a direct address", query) + return [Candidate(direct, direct, "", 100.0)] + response = await self.fetcher.get( + RSEARCH_URL, + params={"query": query, "count": LIMITS.search_results, "content": "false"}, + ) + body = json_or_none(await response.text()) + if response.status >= 400 or not isinstance(body, dict) or not body.get("success", True): + raise RuntimeError(f"rsearch failed for {query!r}: status {response.status}") + candidates = rank(query, body.get("results") or [])[: LIMITS.max_candidates] + for candidate in candidates: + logger.info("search: candidate %.1f %s (%s)", candidate.score, candidate.url, candidate.title) + return candidates diff --git a/devplacepy/services/community/service.py b/devplacepy/services/community/service.py new file mode 100644 index 00000000..616f242d --- /dev/null +++ b/devplacepy/services/community/service.py @@ -0,0 +1,113 @@ +# retoor + +from __future__ import annotations + +import logging + +from devplacepy.services.base import BaseService, ConfigField +from devplacepy.services.community.runner import TRIGGER_SERVICE, CommunityRunner +from devplacepy.services.community.spec import LIMITS, STATUS_FAILING, STATUS_LOGIN, STATUS_READY +from devplacepy.services.community.store import store +from devplacepy.services.sync.options import ALIAS_KEY + +DRY_RUN_KEY = "community_dry_run" +MAX_PER_RUN_KEY = "community_max_per_run" +DISCOVERY_ACTOR = "system" + + +class CommunityService(BaseService): + title = "Community sync" + description = ( + "Discovers and learns external communities queued by an administrator, then runs their " + "generated collectors on a schedule and feeds users, posts and comments through the sync API." + ) + details = ( + "Opt-in and dry run by default: a dry run fetches, validates and logs exactly what would " + "be created, updated or rejected, and writes nothing. Switch dry run off only after the " + "log lists what you expect. Live runs also require the sync_enabled site setting and honor " + "sync_origins. A community that fails repeatedly is disabled and flagged for a human." + ) + default_enabled = False + min_interval = 30 + METRICS_SECONDS = 60 + config_fields = [ + ConfigField( + DRY_RUN_KEY, + "Dry run", + type="bool", + default=True, + help="Collect and validate everything, log the outcome, write nothing and advance no watermark.", + group="Safety", + ), + ConfigField( + MAX_PER_RUN_KEY, + "Communities per tick", + type="int", + default=LIMITS.max_communities_per_run, + minimum=1, + maximum=20, + help="Upper bound on communities processed in one tick. The remainder waits for the next tick.", + group="Safety", + ), + ConfigField( + ALIAS_KEY, + "Identity aliasing", + type="bool", + default=False, + help="Master kill switch. While off, no community with aliasing enabled is run and no new identity is created.", + group="Safety", + ), + ] + + def __init__(self, runner: CommunityRunner | None = None) -> None: + super().__init__("community", interval_seconds=60) + self.runner = runner or CommunityRunner() + + async def on_disable(self) -> None: + from devplacepy.services.community.fetcher import SharedBrowser + + await SharedBrowser.close() + + async def run_once(self) -> None: + config = self.get_config() + dry = config[DRY_RUN_KEY] + budget = config[MAX_PER_RUN_KEY] + mode = "DRY RUN" if dry else "LIVE" + for row in store.pending(budget): + budget -= 1 + self.log(f"[{mode}] discovering pending community {row['query']!r}") + result, saved, text = await self.runner.discover(row["query"], DISCOVERY_ACTOR, dry_run=dry, relearn_slug=row["slug"]) + if dry: + self.log(f"[DRY RUN] would learn {result.slug}: {result.status}{' - ' + result.error if result.error else ''}") + continue + self.log(f"discovery of {row['slug']}: {result.status}{' - ' + result.error if result.error else ''}") + if budget <= 0: + return + for row in store.due(budget): + self.log(f"[{mode}] running {row['slug']}") + report = await self.runner.run(row, dry_run=dry, trigger=TRIGGER_SERVICE) + counts = {kind: sum(per.values()) for kind, per in report.pushed.items()} + self.log(f"[{mode}] {row['slug']}: {report.status}, collected {report.collected}, pushed {counts}") + for line in report.log.splitlines()[-12:]: + self.log(f" {line}", level=logging.DEBUG) + + def collect_metrics(self) -> dict: + rows = store.list() + tally = {status: 0 for status in (STATUS_READY, STATUS_LOGIN, STATUS_FAILING)} + for row in rows: + tally[row["status"]] = tally.get(row["status"], 0) + 1 + return { + "stats": [ + {"label": "Communities", "value": len(rows)}, + {"label": "Ready", "value": tally.get(STATUS_READY, 0)}, + {"label": "Need a human", "value": tally.get(STATUS_LOGIN, 0) + tally.get(STATUS_FAILING, 0)}, + {"label": "Dry run", "value": 1 if self.get_config()[DRY_RUN_KEY] else 0}, + ], + "table": { + "columns": ["Community", "Status", "Runs", "Last run", "Next run"], + "rows": [ + [row["name"], row["status"], row.get("run_count") or 0, row.get("last_run_at") or "-", row.get("next_run_at") or "-"] + for row in rows + ], + }, + } diff --git a/devplacepy/services/community/spec.py b/devplacepy/services/community/spec.py new file mode 100644 index 00000000..27e07115 --- /dev/null +++ b/devplacepy/services/community/spec.py @@ -0,0 +1,91 @@ +# retoor + +from __future__ import annotations + +from dataclasses import dataclass + +FEED_RSS = "rss" +FEED_ATOM = "atom" +FEED_JSON = "json_api" +FEED_HTML = "html_list" +FEED_TYPES = (FEED_RSS, FEED_ATOM, FEED_JSON, FEED_HTML) + +STATUS_PENDING = "pending" +STATUS_LEARNING = "learning" +STATUS_READY = "ready" +STATUS_LOGIN = "login_required" +STATUS_FAILED = "failed" +STATUS_FAILING = "failing" +STATUSES = ( + STATUS_PENDING, + STATUS_LEARNING, + STATUS_READY, + STATUS_LOGIN, + STATUS_FAILED, + STATUS_FAILING, +) + +RUN_OK = "ok" +RUN_PARTIAL = "partial" +RUN_FAILED = "failed" +RUN_DRY = "dry_run" + +SCHEMA_VERSION = 1 +RSEARCH_URL = "https://rsearch.app.molodetz.nl/search" +USER_AGENT_NOTE = "devplace-community" + + +@dataclass(frozen=True) +class CommunityLimits: + search_results: int = 15 + max_candidates: int = 5 + max_probe_paths: int = 12 + sample_posts: int = 12 + sample_comment_posts: int = 3 + max_pages: int = 3 + max_posts: int = 150 + max_comment_posts: int = 25 + comment_recheck: int = 10 + delay_seconds: float = 1.0 + retries: int = 3 + backoff_seconds: float = 2.0 + timeout_seconds: int = 30 + max_bytes: int = 5_000_000 + min_interval_seconds: int = 900 + default_interval_seconds: int = 86_400 + run_timeout_seconds: int = 900 + max_failures: int = 5 + max_communities_per_run: int = 3 + runs_kept: int = 30 + batch_records: int = 50 + min_field_coverage: float = 0.6 + + def runtime(self) -> dict: + return { + "max_pages": self.max_pages, + "max_posts": self.max_posts, + "max_comment_posts": self.max_comment_posts, + "comment_recheck": self.comment_recheck, + "delay_seconds": self.delay_seconds, + "retries": self.retries, + "backoff_seconds": self.backoff_seconds, + "timeout_seconds": self.timeout_seconds, + "max_bytes": self.max_bytes, + } + + +LIMITS = CommunityLimits() +FEED_PROBE_PATHS = ( + "/feed", + "/feed.xml", + "/rss", + "/rss.xml", + "/atom.xml", + "/index.xml", + "/feeds/posts", + "/posts.json", + "/latest.json", + "/api/posts", + "/api/v1/posts", + "/json", +) diff --git a/devplacepy/services/community/store.py b/devplacepy/services/community/store.py new file mode 100644 index 00000000..dc575811 --- /dev/null +++ b/devplacepy/services/community/store.py @@ -0,0 +1,200 @@ +# retoor + +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone + +from devplacepy.database import get_table +from devplacepy.database.soft_delete import soft_delete +from devplacepy.services.community.spec import ( + LIMITS, + STATUS_FAILING, + STATUS_LEARNING, + STATUS_PENDING, + STATUS_READY, +) +from devplacepy.services.sync.selection import jittered +from devplacepy.utils.text import generate_uid + +JSON_COLUMNS = ("schema", "watermark", "totals", "sync_options") +RUN_JSON_COLUMNS = ("collected", "pushed", "errors") +EDITABLE = ( + "name", + "description", + "rules", + "language", + "favicon", + "user_count", + "status", + "schema", + "watermark", + "enabled", + "interval_seconds", + "next_run_at", + "last_run_at", + "last_status", + "last_error", + "run_count", + "failure_count", + "totals", + "learned_at", + "url", + "origin", + "login_username", + "login_email", + "login_password", + "sync_options", + "claimed_at", +) + + +def now() -> datetime: + return datetime.now(timezone.utc) + + +def iso(moment: datetime) -> str: + return moment.isoformat() + + +def decode(row: dict | None, columns: tuple[str, ...]) -> dict | None: + if row is None: + return None + view = dict(row) + for column in columns: + raw = view.get(column) + try: + view[column] = json.loads(raw) if raw else ({} if column != "errors" else []) + except ValueError: + view[column] = {} if column != "errors" else [] + return view + + +def encode(fields: dict, columns: tuple[str, ...]) -> dict: + return { + key: json.dumps(value, ensure_ascii=False) if key in columns and not isinstance(value, str) else value + for key, value in fields.items() + } + + +class CommunityStore: + def get(self, ref: str) -> dict | None: + table = get_table("communities") + row = table.find_one(slug=ref, deleted_at=None) or table.find_one(uid=ref, deleted_at=None) + return decode(row, JSON_COLUMNS) + + def list(self) -> list[dict]: + rows = get_table("communities").find(deleted_at=None, order_by=["name"]) + return [decode(row, JSON_COLUMNS) for row in rows] + + def pending(self, limit: int) -> list[dict]: + horizon = iso(now() - timedelta(seconds=LIMITS.run_timeout_seconds)) + table = get_table("communities") + rows = list(table.find(status=STATUS_PENDING, deleted_at=None, order_by=["created_at"])) + rows += [row for row in table.find(status=STATUS_LEARNING, deleted_at=None) if (row.get("claimed_at") or "") < horizon] + return [decode(row, JSON_COLUMNS) for row in rows[:limit]] + + def claim(self, uid: str) -> dict: + return self.update(uid, status=STATUS_LEARNING, claimed_at=iso(now())) + + @staticmethod + def credentials(row: dict) -> dict: + return { + "username": row.get("login_username") or "", + "email": row.get("login_email") or "", + "password": row.get("login_password") or "", + } + + def due(self, limit: int) -> list[dict]: + moment = iso(now()) + rows = get_table("communities").find(status=STATUS_READY, enabled=1, deleted_at=None, order_by=["next_run_at"]) + due = [row for row in rows if not row.get("next_run_at") or row["next_run_at"] <= moment] + return [decode(row, JSON_COLUMNS) for row in due[:limit]] + + def create(self, slug: str, query: str, created_by: str, **fields) -> dict: + row = { + "uid": generate_uid(), + "slug": slug, + "query": query, + "status": STATUS_PENDING, + "enabled": 1, + "interval_seconds": LIMITS.default_interval_seconds, + "run_count": 0, + "failure_count": 0, + "created_at": iso(now()), + "created_by": created_by, + "deleted_at": None, + "deleted_by": None, + **fields, + } + get_table("communities").insert(encode(row, JSON_COLUMNS)) + return self.get(slug) + + def update(self, uid: str, **fields) -> dict: + unknown = set(fields) - set(EDITABLE) + if unknown: + raise KeyError(f"not editable: {sorted(unknown)}") + get_table("communities").update({"uid": uid, **encode(fields, JSON_COLUMNS)}, ["uid"]) + return self.get(uid) + + def delete(self, uid: str, actor: str) -> int: + return soft_delete("communities", actor, uid=uid) + + def schedule_next(self, uid: str, interval_seconds: int, jitter_percent: int = 0) -> str: + seconds = max(jittered(interval_seconds, jitter_percent), LIMITS.min_interval_seconds) + moment = iso(now() + timedelta(seconds=seconds)) + get_table("communities").update({"uid": uid, "next_run_at": moment}, ["uid"]) + return moment + + def start_run(self, community_uid: str, trigger: str, dry_run: bool) -> str: + uid = generate_uid() + get_table("community_runs").insert( + { + "uid": uid, + "community_uid": community_uid, + "started_at": iso(now()), + "finished_at": "", + "status": "running", + "dry_run": int(dry_run), + "trigger": trigger, + "collected": "", + "pushed": "", + "errors": "", + "log": "", + } + ) + return uid + + def finish_run(self, uid: str, status: str, **fields) -> None: + get_table("community_runs").update( + {"uid": uid, "status": status, "finished_at": iso(now()), **encode(fields, RUN_JSON_COLUMNS)}, ["uid"] + ) + + def runs(self, community_uid: str, limit: int = 20) -> list[dict]: + rows = get_table("community_runs").find(community_uid=community_uid, order_by=["-started_at"], _limit=limit) + return [decode(row, RUN_JSON_COLUMNS) for row in rows] + + def busy(self, community_uid: str) -> bool: + horizon = iso(now() - timedelta(seconds=LIMITS.run_timeout_seconds)) + rows = get_table("community_runs").find(community_uid=community_uid, status="running") + if any(row["started_at"] >= horizon for row in rows): + return True + row = get_table("communities").find_one(uid=community_uid) + return bool(row and row.get("status") == STATUS_LEARNING and (row.get("claimed_at") or "") >= horizon) + + def prune_runs(self, community_uid: str) -> int: + table = get_table("community_runs") + stale = list(table.find(community_uid=community_uid, order_by=["-started_at"]))[LIMITS.runs_kept :] + for row in stale: + table.delete(uid=row["uid"]) + return len(stale) + + def fail(self, row: dict, error: str) -> dict: + failures = int(row.get("failure_count") or 0) + 1 + fields = {"failure_count": failures, "last_error": error[:1000], "last_status": "failed", "last_run_at": iso(now())} + if failures >= LIMITS.max_failures: + fields.update(status=STATUS_FAILING, enabled=0) + return self.update(row["uid"], **fields) + + +store = CommunityStore() diff --git a/devplacepy/services/devii/actions/catalog/__init__.py b/devplacepy/services/devii/actions/catalog/__init__.py index 284e3dc7..87fa90db 100644 --- a/devplacepy/services/devii/actions/catalog/__init__.py +++ b/devplacepy/services/devii/actions/catalog/__init__.py @@ -17,6 +17,7 @@ from .jobs import JOB_ACTIONS from .messages import MESSAGE_ACTIONS from .moderation import MODERATION_ACTIONS from .news import NEWS_ACTIONS +from .nodes import NODE_ACTIONS from .notifications import NOTIFICATION_ACTIONS from .posts import POSTS_ACTIONS from .profile import PROFILE_ACTIONS @@ -24,6 +25,8 @@ from .project_files import PROJECT_FILE_ACTIONS from .projects import PROJECTS_ACTIONS from .quizzes import QUIZ_ACTIONS from .skills import SKILL_ACTIONS +from .community import COMMUNITY_ACTIONS +from .sync import SYNC_ACTIONS from .social import SOCIAL_ACTIONS from .tools import TOOLS_ACTIONS from .uploads import UPLOAD_ACTIONS @@ -53,6 +56,9 @@ CATALOG_GROUPS: dict[str, tuple[Action, ...]] = { "skills": SKILL_ACTIONS, "battles": BATTLE_ACTIONS, "moderation": MODERATION_ACTIONS, + "sync": SYNC_ACTIONS, + "community": COMMUNITY_ACTIONS, + "nodes": NODE_ACTIONS, } ACTIONS: tuple[Action, ...] = tuple( diff --git a/devplacepy/services/devii/actions/catalog/admin.py b/devplacepy/services/devii/actions/catalog/admin.py index 63b89427..4c4a6d56 100644 --- a/devplacepy/services/devii/actions/catalog/admin.py +++ b/devplacepy/services/devii/actions/catalog/admin.py @@ -189,6 +189,8 @@ ADMIN_ACTIONS: tuple[Action, ...] = ( body("maintenance_mode", "Whether maintenance mode is on."), body("maintenance_message", "Maintenance message."), body("happy_404_enabled", "Whether Happy 404 (rendering a random post instead of a 404 page) is on."), + body("sync_enabled", "Whether community sync is on (1) or off (0)."), + body("sync_origins", "Comma separated origins allowed to sync; empty allows any."), ), requires_admin=True, ), diff --git a/devplacepy/services/devii/actions/catalog/community.py b/devplacepy/services/devii/actions/catalog/community.py new file mode 100644 index 00000000..a630431d --- /dev/null +++ b/devplacepy/services/devii/actions/catalog/community.py @@ -0,0 +1,113 @@ +# retoor + +from __future__ import annotations + +from ..spec import Action +from ._shared import body, confirm, path + +SLUG = path("slug", "Community slug, as returned by community_list.") +ACCOUNT = ( + body("login_username", "Username of an account on that community, for a login-protected community."), + body("login_email", "Email of that account, when the community logs in by email."), + body("login_password", "Password of that account. Stored for the scheduled collectors, never shown again."), +) + +COMMUNITY_ACTIONS: tuple[Action, ...] = ( + Action( + name="community_list", + method="GET", + path="/admin/communities", + summary="List the external communities that are synchronised (admin only)", + description=( + "Every community with its status (pending, learning, ready, login_required, failed, failing), " + "schedule, last run and failure count, plus whether community sync and the Community sync " + "service are switched on and whether the service is in dry run." + ), + requires_admin=True, + ), + Action( + name="community_get", + method="GET", + path="/admin/communities/{slug}", + summary="Show one community: metadata, learned feeds, totals and recent runs (admin only)", + description=( + "Returns the community's description, rules, language and member count, the feeds the system " + "learned (type, address, field mapping, comments mode, pagination), the totals synced so far " + "and the log of recent runs including dry runs. Use it to explain why a run failed." + ), + params=(SLUG,), + requires_admin=True, + ), + Action( + name="community_add", + method="POST", + path="/admin/communities", + summary="Add a community to synchronise by name or address (admin only)", + description=( + "Starts discovery: the system searches for the community, detects its feeds (RSS, Atom, JSON or " + "HTML, with JavaScript rendering when needed), learns the structure once, generates collectors " + "and records the community's metadata. Discovery runs in the background; poll community_get until " + "the status is ready. Pass an account for a community that requires a login. Members never see " + "these communities." + ), + params=(body("query", "Community name or address, for example https://forum.example.com", required=True), *ACCOUNT), + requires_admin=True, + ajax=True, + ), + Action( + name="community_run", + method="POST", + path="/admin/communities/{slug}/run", + summary="Run a community's collectors now, optionally as a dry run (admin only)", + description=( + "Collects and pushes users, posts and comments through the sync API in the background. With " + "dry_run=true everything is validated and logged but nothing is written and the watermark does " + "not move. A live run needs the sync_enabled setting. Read the result with community_get." + ), + params=(SLUG, body("dry_run", "Validate and report without writing.", type="boolean")), + requires_admin=True, + ajax=True, + ), + Action( + name="community_relearn", + method="POST", + path="/admin/communities/{slug}/relearn", + summary="Learn a community's structure again, for example after its site changed (admin only)", + description="Runs discovery again with the stored account. A failed relearn keeps the working schema.", + params=(SLUG,), + requires_admin=True, + ajax=True, + ), + Action( + name="community_update", + method="POST", + path="/admin/communities/{slug}/update", + summary="Change a community's interval, enabled flag or login account (admin only)", + description=( + "interval_seconds is between 900 and 2592000. An empty password keeps the stored one; " + "clear_login=true removes the stored account." + ), + params=( + SLUG, + body("interval_seconds", "Seconds between runs, 900 to 2592000.", required=True, type="integer"), + body("enabled", "Whether the scheduled service runs this community.", type="boolean"), + *ACCOUNT, + body("clear_login", "Remove the stored account.", type="boolean"), + ), + requires_admin=True, + ajax=True, + ), + Action( + name="community_delete", + method="POST", + path="/admin/communities/{slug}/delete", + summary="Delete a community from the synchronisation list (admin only)", + description=( + "Moves the community to the trash and stops its collectors. Users, posts and comments that were " + "already synchronised stay." + ), + params=(SLUG, confirm()), + requires_admin=True, + ajax=True, + ), +) diff --git a/devplacepy/services/devii/actions/catalog/nodes.py b/devplacepy/services/devii/actions/catalog/nodes.py new file mode 100644 index 00000000..90182809 --- /dev/null +++ b/devplacepy/services/devii/actions/catalog/nodes.py @@ -0,0 +1,149 @@ +# retoor + +from __future__ import annotations + +from ..spec import Action +from ._shared import body, confirm, path, query + +DEVICE = path("device_uid", "Device uid, as returned by node_device_list.") +ADMIN_DEVICE = path("device_uid", "Device uid, as listed by node_admin_status.") + +NODE_ACTIONS: tuple[Action, ...] = ( + Action( + name="social_search", + method="GET", + path="/nodes/social/search", + summary="Search public developer discussion outside this site (Hacker News, Stack Overflow, GitHub, dev.to, Mastodon, Reddit)", + description=( + "Returns merged hits with platform, title, url, snippet, author, score and comment count. " + "Platforms that could not be reached are listed under 'unavailable' with a reason code; report " + "that honestly and never invent results for them. Reddit is only reachable when community " + "donors are online. Use it only when the user asks what people elsewhere say about something; " + "platform tools come first." + ), + params=( + query("q", "What to search for.", required=True), + query("platforms", "Comma separated subset of hackernews, stackexchange, github, devto, mastodon, reddit. Empty means all."), + query("limit", "Hits per platform, 1 to 20."), + ), + read_only=True, + ), + Action( + name="node_fetch", + method="POST", + path="/nodes/internal/fetch", + summary="Fetch one public page from an allowlisted site through the community donor network", + description=( + "Only hosts on the published allowlist are accepted (today reddit.com and redd.it); anything else is " + "refused with a reason. Returns status, text and whether the body was truncated. It fails with " + "no_nodes when no donor is online; say so instead of guessing the page content." + ), + params=( + body("url", "Absolute https URL on an allowlisted host.", required=True), + body("cache", "Allow a cached copy.", type="boolean"), + ), + read_only=True, + ), + Action( + name="node_status", + method="GET", + path="/nodes/me", + summary="Show your community donation status: consent, settings, devices and what you donated so far", + description="Reports whether you may donate right now and, if not, the reason.", + read_only=True, + ), + Action( + name="node_settings_set", + method="POST", + path="/nodes/settings", + summary="Change your community donation settings (bandwidth, hours, platforms)", + description=( + "Donation can only be switched ON by the member in their own profile, never through this tool; " + "'enabled': true is refused. Switching it off, and every other setting, is allowed." + ), + params=( + body("enabled", "Set false to stop donating.", type="boolean"), + body("bandwidth_mb_day", "Daily bandwidth cap in megabytes.", type="integer"), + body("max_concurrent", "Simultaneous requests.", type="integer"), + body("active_from", "Start of the active window, HH:MM."), + body("active_to", "End of the active window, HH:MM."), + body("timezone", "IANA timezone for the active window, for example Europe/Amsterdam."), + body("pause_on_battery", "Stop donating on battery power.", type="boolean"), + body("public_display", "Show your name on the donor leaderboard.", type="boolean"), + ), + ajax=True, + ), + Action( + name="node_device_list", + method="GET", + path="/nodes/devices", + summary="List your donor devices", + description="Shows each device with its status and when it last connected. Secrets are never shown.", + read_only=True, + ), + Action( + name="node_device_revoke", + method="POST", + path="/nodes/devices/{device_uid}/delete", + summary="Remove one of your donor devices", + description="A connected device is disconnected within seconds and its secret stops working for good.", + params=(DEVICE, confirm()), + ajax=True, + ), + Action( + name="node_admin_status", + method="GET", + path="/admin/nodes", + summary="Show the community node network: connected pool, queue, devices, open flags, settings and the kill switch", + description=( + "Administrator overview. Lists the 100 most recently seen devices with their owner and status, " + "the open flags, the queue depth and the caps in force. Use the device uids it returns for " + "node_admin_suspend and node_admin_unsuspend." + ), + requires_admin=True, + read_only=True, + ), + Action( + name="node_admin_suspend", + method="POST", + path="/admin/nodes/devices/{device_uid}/suspend", + summary="Suspend one donor device for a fixed number of minutes with a stated reason", + description=( + "A suspended device is disconnected within seconds and cannot reconnect until the period ends or " + "it is unsuspended. The account itself is untouched. Show the administrator the exact device and " + "owner, get explicit confirmation, then call again with confirm=true." + ), + requires_admin=True, + params=( + ADMIN_DEVICE, + body("minutes", "Suspension length in minutes, 1 to 10080.", required=True, type="integer"), + body("reason", "Reason recorded with the suspension."), + confirm(), + ), + ), + Action( + name="node_admin_unsuspend", + method="POST", + path="/admin/nodes/devices/{device_uid}/unsuspend", + summary="Lift the suspension of one donor device", + description="The device may reconnect immediately. Reversible, so no confirmation is needed.", + requires_admin=True, + params=(ADMIN_DEVICE,), + ), + Action( + name="node_admin_killswitch", + method="POST", + path="/admin/nodes/killswitch", + summary="Switch community donation on or off for the whole platform", + description=( + "Off disconnects every donor node within seconds and stops new jobs; on lets consenting donors " + "connect again. Omit 'enabled' to flip the current state. Show the administrator the current state " + "and the effect, get explicit confirmation, then call again with confirm=true." + ), + requires_admin=True, + params=( + body("enabled", "True to switch donation on, false to switch it off. Omit to flip it.", type="boolean"), + confirm(), + ), + ), +) diff --git a/devplacepy/services/devii/actions/catalog/sync.py b/devplacepy/services/devii/actions/catalog/sync.py new file mode 100644 index 00000000..b40bf264 --- /dev/null +++ b/devplacepy/services/devii/actions/catalog/sync.py @@ -0,0 +1,44 @@ +# retoor + +from __future__ import annotations + +from ..spec import Action +from ._shared import body + +ENVELOPE = ( + body("origin", "Stable id of the sending community, for example other.example.org.", required=True), + body("records", "JSON array string of at most 50 records, for example [{\"origin_uid\": \"1\", ...}]. Field names are listed at /sync/spec.json.", required=True), + body("dry_run", "Validate and report what would happen without writing.", type="boolean"), +) + + +def sync_action(kind: str, summary: str, description: str) -> Action: + return Action( + name=f"sync_{kind}", + method="POST", + path=f"/sync/{kind}", + summary=summary, + description=description, + ajax=True, + params=ENVELOPE, + requires_admin=True, + ) + + +SYNC_ACTIONS: tuple[Action, ...] = ( + sync_action( + "users", + "Upsert users copied from another community", + "Creates or updates synced accounts keyed on each record's origin_uid. Counters, email and role are ignored. Send users before posts.", + ), + sync_action( + "posts", + "Upsert posts copied from another community", + "Creates or updates posts; author_uid is the author's origin_uid and must already be synced.", + ), + sync_action( + "comments", + "Upsert comments copied from another community", + "Creates or updates comments; author_uid, post_uid and any parent_uid are origin ids that must already be synced.", + ), +) diff --git a/devplacepy/services/devii/actions/dispatcher.py b/devplacepy/services/devii/actions/dispatcher.py index 57c91d5c..346f7a7d 100644 --- a/devplacepy/services/devii/actions/dispatcher.py +++ b/devplacepy/services/devii/actions/dispatcher.py @@ -33,6 +33,10 @@ from .spec import Action, Catalog MUTATING_METHODS = ("POST", "DELETE", "PUT", "PATCH") CONFIRM_REQUIRED = { + "community_delete", + "node_device_revoke", + "node_admin_suspend", + "node_admin_killswitch", "join_battle", "fight_battle", "delete_my_account", @@ -241,6 +245,31 @@ def confirmation_error(name: str, arguments: dict[str, Any]) -> ToolInputError | f"Deleting comment '{uid}' is permanent and cannot be undone. " "Ask the user to confirm explicitly, then call again with confirm=true." ) + if name == "community_delete": + slug = str(arguments.get("slug", "")).strip() or "(unspecified)" + return ToolInputError( + f"Deleting community '{slug}' stops its synchronisation and moves it to the trash; content already " + "synchronised stays. Show the user the community, get explicit confirmation, then call again with confirm=true." + ) + if name == "node_device_revoke": + uid = str(arguments.get("device_uid", "")).strip() or "(unspecified)" + return ToolInputError( + f"Removing donor device '{uid}' disconnects it and permanently invalidates its secret. " + "Show the user the device, get explicit confirmation, then call again with confirm=true." + ) + if name == "node_admin_suspend": + uid = str(arguments.get("device_uid", "")).strip() or "(unspecified)" + return ToolInputError( + f"Suspending donor device '{uid}' disconnects it within seconds and blocks it from reconnecting for " + "the whole period. Show the administrator the exact device and owner, get explicit confirmation, " + "then call again with confirm=true." + ) + if name == "node_admin_killswitch": + return ToolInputError( + "The community donation switch is global: turning it off disconnects every donor node within " + "seconds and stops all node jobs. Show the administrator the current state and the effect, get " + "explicit confirmation, then call again with confirm=true." + ) if name == "delete_gist": uid = str(arguments.get("uid", "")).strip() or "(unspecified)" return ToolInputError( diff --git a/devplacepy/services/devii/agent.py b/devplacepy/services/devii/agent.py index 572fc562..74a3a3b7 100644 --- a/devplacepy/services/devii/agent.py +++ b/devplacepy/services/devii/agent.py @@ -197,6 +197,15 @@ SYSTEM_PROMPT = ( "information the platform cannot provide and the user wants it. Never use them to answer questions " "about this DevPlace instance, its users, posts, settings, or metrics - those have dedicated " "platform tools. When platform tools can serve the request, do not call rsearch.\n\n" + "SOCIAL SEARCH AND THE DONOR NETWORK\n" + "social_search looks up public developer discussion on Hacker News, Stack Overflow, GitHub, dev.to, " + "Mastodon and, when community donors are online, Reddit. Use it only when the user asks what people " + "outside this site say about a topic; platform tools and data come first. Read its 'unavailable' map " + "and tell the user which platforms could not be reached; never claim results from a platform that is " + "listed there and never invent a platform it does not cover (X, Instagram, LinkedIn and TikTok are not " + "reachable). node_fetch reads one page from an allowlisted site through donors. node_status, " + "node_settings_set, node_device_list and node_device_revoke manage the member's own donation; donation " + "can only be switched on by the member in their profile, never by you.\n\n" "EMAIL TOOLS (email_*)\n" "The email_* tools connect to the user's OWN external mailbox over IMAP/SMTP, not this " "platform, and are only available to signed-in users. The connection must be configured first " diff --git a/devplacepy/services/devii/registry.py b/devplacepy/services/devii/registry.py index 171b019d..6936df40 100644 --- a/devplacepy/services/devii/registry.py +++ b/devplacepy/services/devii/registry.py @@ -74,6 +74,9 @@ GROUP_LABELS: dict[str, str] = { "skills": "Skills", "battles": "Opinion Wars", "moderation": "Moderation", + "sync": "Community Sync", + "community": "Community Discovery (admin)", + "nodes": "Community Donation and Social Search", "tasks": "Scheduled Tasks and Reminders", "agentic": "Self-Evaluation (delegate, eval)", "avatar": "Browser Avatar Tutorials", diff --git a/devplacepy/services/jobs/deepsearch/crawl.py b/devplacepy/services/jobs/deepsearch/crawl.py index 65d32cbe..b7da1fea 100644 --- a/devplacepy/services/jobs/deepsearch/crawl.py +++ b/devplacepy/services/jobs/deepsearch/crawl.py @@ -15,6 +15,7 @@ from urllib.parse import urlparse import httpx from devplacepy import stealth +from devplacepy.config import INTERNAL_BASE_URL from devplacepy.net_guard import BlockedAddressError, guard_public_url, guarded_async_client from .extract import extract_html, relevant_links @@ -155,6 +156,51 @@ async def search_queries( return _interleave(buckets) +SOCIAL_QUERIES = 2 +SOCIAL_HITS = 4 +SOCIAL_TIMEOUT_SECONDS = 40.0 + + +async def social_candidates( + queries: list[str], api_key: str, emit: Callable[[dict], None] = lambda frame: None +) -> list[dict]: + if not api_key: + return [] + found: list[dict] = [] + timeout = httpx.Timeout(SOCIAL_TIMEOUT_SECONDS, connect=10.0) + async with stealth.stealth_async_client( + base_url=INTERNAL_BASE_URL, + headers={"Authorization": f"Bearer {api_key}", "Accept": "application/json"}, + timeout=timeout, + ) as client: + for query in queries[:SOCIAL_QUERIES]: + try: + response = await client.get( + "/nodes/social/search", params={"q": query, "limit": SOCIAL_HITS} + ) + emit({"type": "social", "success": response.status_code < 400}) + if response.status_code >= 400: + continue + data = response.json() + except (httpx.HTTPError, ValueError) as exc: + emit({"type": "social", "success": False}) + logger.info("deepsearch social search failed for %r: %s", query, exc) + continue + for hit in data.get("hits") or []: + url = (hit.get("url") or "").strip() + if url: + found.append( + { + "url": url, + "title": hit.get("title") or "", + "description": hit.get("snippet") or "", + "content": hit.get("snippet") or "", + "query": query, + } + ) + return found + + def _snippet_page(candidate: dict, depth: int) -> CrawledPage | None: snippet = _clean_snippet(candidate.get("content") or candidate.get("description") or "") if len(snippet) < SNIPPET_MIN_CHARS: diff --git a/devplacepy/services/jobs/deepsearch/service.py b/devplacepy/services/jobs/deepsearch/service.py index cfea2e68..8ba56b7c 100644 --- a/devplacepy/services/jobs/deepsearch/service.py +++ b/devplacepy/services/jobs/deepsearch/service.py @@ -43,6 +43,14 @@ class DeepsearchService(JobService): help="Model sent to the internal AI gateway for query planning and report synthesis. Blank uses the gateway default model.", group="DeepSearch", ), + ConfigField( + "deepsearch_social", + "Include social search", + type="bool", + default=False, + help="Add public discussion from Hacker News, Stack Overflow, GitHub, dev.to, Mastodon and (when donors are online) Reddit to the sources of a signed-in member's research. Off by default.", + group="DeepSearch", + ), ] def session_dir(self, uid: str) -> Path: @@ -68,6 +76,9 @@ class DeepsearchService(JobService): payload["collection"] = self.collection_name(uid) payload["model"] = self.get_config()["deepsearch_model"] or INTERNAL_MODEL payload["cached_hashes"] = self._cached_hashes(database) + payload["social"] = bool( + job.get("owner_kind") == "user" and self.get_config()["deepsearch_social"] + ) payload_path = output_dir / "payload.json" await asyncio.to_thread(payload_path.write_text, json.dumps(payload), encoding="utf-8") diff --git a/devplacepy/services/jobs/deepsearch/worker.py b/devplacepy/services/jobs/deepsearch/worker.py index a2dd8816..0b965319 100644 --- a/devplacepy/services/jobs/deepsearch/worker.py +++ b/devplacepy/services/jobs/deepsearch/worker.py @@ -14,7 +14,7 @@ from devplacepy.services.deepsearch.store import Chunk, VectorStore from devplacepy.utils import generate_uid from .chunking import chunk_text -from .crawl import content_hash, crawl, search_queries, url_hash +from .crawl import content_hash, crawl, search_queries, social_candidates, url_hash from .enhance import plan_followup_queries, plan_queries from .orchestrate import orchestrate from .phases import ( @@ -174,6 +174,10 @@ async def _run(payload: dict, output_dir: Path) -> dict: _stage("searching", "Searching the web", PHASE_SEARCHING) candidates = await search_queries(queries, _emit) + if payload.get("social"): + known = {item["url"] for item in candidates} + extra = await social_candidates(queries, api_key, _emit) + candidates += [item for item in extra if item["url"] not in known] _emit({"type": "candidates", "count": len(candidates)}) _stage("crawling", "Crawling sources", PHASE_CRAWLING) diff --git a/devplacepy/services/moderation/deletion.py b/devplacepy/services/moderation/deletion.py index 806d2a98..4050fe27 100644 --- a/devplacepy/services/moderation/deletion.py +++ b/devplacepy/services/moderation/deletion.py @@ -64,6 +64,7 @@ OWNED_TABLES: tuple[tuple[str, str], ...] = ( ("node_settings", "user_uid"), ("node_donors", "user_uid"), ("node_usage_daily", "user_uid"), + ("node_caller_daily", "caller_id"), ) CHILD_TABLES: tuple[tuple[str, str, str, str], ...] = ( diff --git a/devplacepy/services/nodes/CLAUDE.md b/devplacepy/services/nodes/CLAUDE.md index db7d1304..38b3ab22 100644 --- a/devplacepy/services/nodes/CLAUDE.md +++ b/devplacepy/services/nodes/CLAUDE.md @@ -2,7 +2,7 @@ This file documents the community node network (`devplacepy/services/nodes/`, `d ## What exists today -Phase 1 (the specification), Phase 2 (the data layer and usage ledger) and Phase 3 (registry, session, consent gate, kill switch, device and ticket routes, the lock-owner socket) are built. No job traffic flows yet. Everything else is planned and listed under "Gaps" next to the phase that closes it. Nothing here is enabled for any account. +The specification, data layer, registry, session, consent gate, kill switch, destination policy, queue, scheduler, dispatcher, circuit breakers, cache, the fabric facade, tunnel mode, social search adapters, the Go client, the browser extension, the Donate tab, the admin page, the Devii tools, the CLI and the docs are built and tested. The gaps below list what is unmeasured or missing. Nothing here is enabled for any account. | Piece | File | State | |-------|------|-------| @@ -10,7 +10,7 @@ Phase 1 (the specification), Phase 2 (the data layer and usage ledger) and Phase | `GET /nodes/spec.json` | `routers/nodes/spec.py` | built, tested, public | | Generated protocol docs page | `templates/docs/nodes-protocol.html` | built, reads the `nodes_spec` Jinja global | | API docs group | `docs_api/groups/nodes.py` | built, one endpoint | -| Tables and helpers | `database/nodes.py`, `NODES_COLUMNS` in `database/schema.py` | built, tested, no route uses them yet | +| Tables and helpers | `database/nodes.py`, `NODES_COLUMNS` in `database/schema.py` | built, tested | | Usage ledger | `ledger.py` (`UsageLedger`, `DonationSummary`) | built, tested | | `bandwidth_sharing` consent kind | `CONSENT_KINDS` in `database/moderation.py` | registered, never granted by default, enforced by `ConsentGate` | | Frame codec | `frames.py` (`decode`, `encode`, `FrameError`) | built, tested, validates every frame against `spec.py` | @@ -20,6 +20,22 @@ Phase 1 (the specification), Phase 2 (the data layer and usage ledger) and Phase | Supervisor | `service.py` (`NodesService`) | built, registered in `main.py`, default enabled | | Device, ticket and socket routes | `routers/nodes/devices.py`, `ticket.py`, `ws.py` | built, tested | | nginx upgrade block for `/nodes/ws` | `nginx/nginx.conf.template` | added | +| Destination policy | `policy.py` (`DestinationPolicy`, `Target`, `is_public_address`) | built, tested | +| Result cache | `cache.py` (`ResultCache`) | built, tested, per process | +| Circuit breaker | `circuit.py` (`CircuitBreaker`) | built, tested | +| Scheduler | `scheduler.py` (`NodeScheduler`, `in_active_hours`) | built, tested | +| Queue and broker | `broker.py` (`JobBroker`), `database/nodes.py` request helpers | built, tested | +| Dispatcher | `dispatcher.py` (`NodeDispatcher`) | built, tested, runs in the lock owner | +| Fabric facade | `fabric.py` (`NodeFabric`, `FabricResult`, `FabricError`) | built, tested, used by `social.py` and `POST /nodes/internal/fetch` | +| Admin page and actions | `routers/admin/nodes.py`, `templates/admin_nodes.html`, `static/js/AdminNodes.js` | built | + +## Administration (`/admin/nodes`) + +`GET /admin/nodes` (HTML and JSON `AdminNodesOut`) is the administrator view: pool counters read from the `nodes` row in `service_state` (so any worker answers, at most one persist interval stale; `pool.online` is false when the heartbeat is stale), queue depth and counts by status, device counts by status, kind, address class and country, the current UTC day's totals, open flags and the 100 most recently seen devices (both capped), the kill switch and the six settings with defaults and ceilings. **Per-destination circuit state is not shown:** it lives in memory on the lock-owner worker and only the number of open circuits is persisted, so a per-destination view would need a new persisted snapshot. Device counts use the stored status, so a suspension that has expired still counts as `suspended` until the next write; the recent-devices list uses `effective_status`. + +POST routes, all `require_admin`, all audited: `/killswitch` (`node.killswitch`, flips or sets `nodes_enabled`), `/settings` (`node.admin.settings`, only submitted fields change, values above the ceiling are clamped and reported under `clamped`), `/devices/{uid}/suspend` (`node.suspend`, 1 to 10080 minutes) and `/unsuspend` (`node.unsuspend`), `/flags/{uid}/resolve` (`node.flag.resolve`). The per-device and per-flag routes are gated by `is_senior_admin` against the device or flag owner and answer through `deny_senior` (audited `denied`). Suspension and the kill switch take effect within one `reconcile` tick on whichever worker holds the sockets. Ceilings: `nodes_max_total` and `nodes_cache_ttl_seconds` use `LimitSpec`; the quota ceilings are constants in the router because `LimitSpec` only carries defaults for them. + +The page refreshes every 10 seconds by fetching its own HTML and swapping the `[data-live]` sections (`AdminNodes.js`, `app.adminNodes`); the settings form and the suspend modal are never swapped, so typing is not interrupted. Devii: `node_admin_status`, `node_admin_suspend` and `node_admin_killswitch` (the last two in `CONFIRM_REQUIRED`), `node_admin_unsuspend`, all `requires_admin`. ## The specification is the contract @@ -65,6 +81,22 @@ Seven tables, all ensured with their full column set in `init_db` (`_ensure_node 5. `hello` must arrive within `hello_timeout_seconds`, with a supported `protocol`, a `kind` equal to the registered device kind and a non-empty list of known `modes`. The server answers `welcome` with the full limits and allowlist from the spec. A second connection of the same device kicks the first with `replaced`. 6. The session pings every `ping_seconds`; any frame from the client resets the missed counter and `missed_pings_before_evict` unanswered pings close it with `heartbeat`. `usage` frames are advisory and only summed on the session. Every other client frame is a protocol error until Phase 4 assigns jobs. +## Jobs (Phase 4) + +A job is a row in `node_requests`, so any worker can ask and only the lock owner serves. Life cycle: `queued` -> `assigned` -> `done`, or `failed` / `expired`; every transition is one conditional `UPDATE`. + +1. `NodeFabric.fetch(url, caller_kind=, caller_id=)` (any worker): policy check, result cache, kill switch and "any node online" (read from the `nodes` row in `service_state`, so it is at most one persist interval stale), then the **caller's daily quota**, then enqueue, then poll the row every `poll_interval_ms` until it is terminal or the caller's wait ends. One retry on `node_disconnected`, `timeout`, `send_failed`, `concurrency_exceeded`, `outside_hours`, `bandwidth_exhausted`; the retry does not spend a second quota unit. Bounded: queue depth (`max_queue_depth`, a conditional `INSERT`), `max_wait_seconds`, `queue_max_wait_seconds`. +2. `NodeDispatcher.tick` (lock owner, every `dispatcher_tick_ms`, started by `ensure_dispatching()` from the socket route or `on_enable`): expire, sweep timed-out in-flight jobs, then for each queued row re-check the policy, ask the scheduler for a node, claim the row atomically, send the `job` frame. +3. The node answers with `job_result` or `job_error`. `Session._handle` only accepts an answer for a job it has in flight; anything else is a protocol error and closes the socket. + +**Scheduler eligibility** (every one must hold): socket open and not paused, speaks a transport the job allows (`tunnel` is preferred over `fetch` for `any`; a node that speaks neither gets nothing), is not the caller's own account, below `min(server, donor setting, hello limits)` concurrency, the destination's adapter is in the donor's `allowed_platforms`, inside the donor's active hours in the donor's timezone (windows may wrap midnight), below the donor's daily byte cap, at least `min_seconds_per_node_domain` since this node last started a job for the destination suffix, node circuit closed. Globally the destination needs `1/max_rps` seconds since its last job and a closed destination circuit. Choice: fewest in-flight, then least recently assigned, then random. Per-destination state is keyed by the allowlist suffix, so `a.reddit.com` ... `z.reddit.com` cannot grow it. + +**Trust.** `via` is always `fetch` for node results. A result is checked before it is stored: status 100-599, strict base64, body capped at `max_body_bytes` (truncated and flagged, never silent), headers reduced to at most 64 short strings with `set-cookie`, `set-cookie2`, `authorization` and `proxy-authenticate` dropped. A malformed result is an **integrity failure**: the request fails, `integrity_failures` is counted on the device and the process, a `node_flags` row is raised for a human, and the node is NOT auto-banned. A donor refusing a destination the server allowed raises a `refusal_mismatch` flag. Donor limit codes (`bandwidth_exhausted`, `outside_hours`, `concurrency_exceeded`) pause the node instead of blaming it. HTTP 429 and 5xx count against the destination circuit (they are the signal that stops hammering a rate-limiting site); the result is still returned to the caller. Sampled redundancy between two donors is not built (Phase 6B). + +**Accounting.** A successful job calls `record_job` (daily bucket and lifetime totals in one transaction). If the donor's cap would be crossed the result is still delivered but the node is stopped for the day. Failures call `record_failure`. Bytes up are estimated (`len(url) + 256`) because the server never sees the donor's real request. + +**Measured** (scratch SQLite on this machine, 2000 synthetic sessions, 500 queued requests, batch 50): an idle tick costs 0.1 ms; a tick where no node is eligible because every node is outside its hours cost 225 ms before two fixes (the active-hours window is now cached per minute and a destination found saturated is scanned once per tick, not once per queued request) and 8.4 ms after; a destination held back by its rate ceiling costs 4.7 ms. Races with 16 to 60 real processes on fresh rows: the queue cap admitted exactly the remaining slots, one request was claimed by exactly one assignment, a caller quota of 5 admitted exactly 5. These are one machine and synthetic sessions, not a soak test. + ## The consent choke point `ConsentGate` is the only place that decides whether a member's node may exist. `allows_many(user_uids)` decides a whole set with a fixed number of queries and `allows(uid)` is the one-element case of it, so the single and batch answers cannot diverge (a test compares them). Order of reasons: `kill_switch`, `no_account`, `deletion_pending`, `account_inactive`, `suspended`, `not_adult`, `no_consent`, `not_enabled`. It is called at device registration, ticket issue, socket accept and every supervisor tick. An account whose `age_band` is empty (created before age bands existed) is refused as not adult, loudly, rather than assumed adult. @@ -77,13 +109,32 @@ Withdrawal is member-only: `MEMBER_ONLY_CONSENTS` in `database/moderation.py` li `tunnel` (native client) relays raw TCP so TLS runs end to end between DevPlace and the destination: the donor cannot read or forge content. `fetch` (browser extension, which has no raw sockets) lets the donor read and forge the response, so every `fetch` result is verified and labelled. A plain web page or a service worker cannot do either (CORS, no sockets) and is out of scope for good. -## Gaps (what the next phases close) +## Donate tab (member UI) -- Phase 2 leftovers: the `bandwidth_sharing` consent can currently be granted through `POST /profile/{username}/consent` and by Devii's `set_consent` (Devii calls that route as the user). Nothing reads it yet, so it is inert, but Phase 3 must make Devii unable to grant it (Devii may read and withdraw) at the same time as `ConsentGate` lands. No XP or badge is awarded yet (`UsageLedger.claim_xp` only reserves the daily budget); the privacy page and app-store declarations are updated with the first phase that lets a device connect, because until then no personal data is collected. -- Phase 3 leftovers: `country`, `asn` and `asn_class` on a device are never filled (they stay `unknown`), so the datacenter exclusion (D6) is not enforced yet; it needs the offline ASN database from Phase 4. `last_seen_at` is written on connect only, not per heartbeat, so a device list cannot yet show who is online right now. The socket counters are per process and are shown in the service metrics only. The `devplace.net` front proxy idle timeout for a long-lived socket is unmeasured. -- Phase 4: broker, scheduler, dispatcher, destination policy, circuit breakers, result cache. -- Phase 5 onward: the reference client, adapters, the Go client, the extension, the profile UI, Devii tools, the CLI, the privacy and donor-terms pages, the legal gate. -- Not measured yet: Reddit and X success rates per address class, WebSocket idle timeouts through `pravda.education` and `devplace.net`, dispatcher tick cost against SQLite, socket capacity of one worker. The caps in `LimitSpec` are starting ceilings, not measurements. +The owner-only `Donate` tab on the profile (`?tab=donate`, `templates/_node_donation.html`, `static/js/NodeDonation.js` as `app.nodeDonation`, `static/css/nodes.css`) is a thin client of the routes above: it renders `/nodes/me` server-side (context key `node_me`, only for the owner on that tab, `noindex,follow`), then polls `GET /nodes/me` every 30 seconds with the shared `Poller`. Consent goes through `POST /profile/{username}/consent`, never a second consent route. The device secret exists only in the browser memory and DOM until "I have saved it" is pressed; the install lines are built client-side from `GET /nodes/client-version`. Prose for members is `templates/docs/nodes.html`; the data categories are in `templates/docs/privacy.html`. + +## Tunnel mode (`tunnel.py`) + +For a `tunnel` or `any` job the scheduler prefers a node whose `hello.modes` include `tunnel`. The dispatcher marks the transport on the row (`node_requests.mode` becomes the transport actually used, which is how `FabricResult.via` says `tunnel` or `fetch`), then runs the request itself in the lock owner: `TunnelBridge.fetch` opens a one-shot listener on `127.0.0.1:`, points a stealth (curl_cffi, Chrome fingerprint) client at it as an HTTP proxy, and the `Relay` answers exactly one `CONNECT host:port` for the job's own host and port (anything else is `403`, a second `CONNECT` is `405`, a donor that refuses is `502`). Bytes are relayed as `tunnel_data` frames (48 KiB chunks, base64, both directions, `RELAY_CAP_BYTES` total, `queue_full` and `bad_base64` end the tunnel with a code). TLS runs between DevPlace and the destination, so the donor sees the host and ciphertext and cannot read or forge the page; the ledger counts the real bytes relayed. Late `tunnel_*` frames for a closed relay are ignored (the donor legitimately sends `tunnel_close` after the server did). Redirects are not followed; the caller gets the 3xx. In `fetch` mode the `job` frame now carries `JOB_HEADERS` (a descriptive bot user agent), because a Go or browser client would otherwise send its own. + +## Social search (`social.py`, `adapters/`) + +`SocialSearch.search` fans one query out to adapters in parallel (`asyncio.gather`, 20 s each) and merges hits round-robin. Direct adapters (`adapters/direct.py`: Hacker News, Stack Overflow, GitHub issues, dev.to by tag, Mastodon by hashtag) need no donors. `adapters/reddit.py` is node-backed and only used when `nodes_social_via_nodes` is `1`; otherwise it is reported as `nodes_not_used`. Every adapter parses a response structurally and raises `AdapterError` on a forged or wrong shape; platforms that fail are listed under `unavailable` with a code and never filled with invented results. **Measured on this machine (2026-10-15, datacenter address):** Hacker News Algolia, Stack Exchange, GitHub search, dev.to tag API and Mastodon hashtag timelines answer; Bluesky `searchPosts` returns 403 without auth, Mastodon `/api/v2/search` returns nothing without login, Lobsters search sits behind a bot wall, dev.to has no usable text search, and Reddit returns 403 to this address. Those are why Bluesky and Lobsters are not adapters. Surfaces: `GET /nodes/social/search` (daily per-member quota `nodes_daily_social_user`, default 300), Devii `social_search`, the DeepSearch `deepsearch_social` option (off by default, signed-in owners only, goes through the HTTP route with the owner's key). + +## Clients + +`nodes/client/` is the Go donor client (standard library only, `build.sh` cross-compiles five targets and writes `manifest.json`). `nodes/extension/` is the MV3 extension (Chrome and Firefox, fetch mode only, 49 `node --test` tests, one of which reads `spec.py` and fails on drift). `make node-client` builds both into `data/node_clients/`, which `GET /nodes/client-version` and `GET /nodes/download/{artifact}` serve (names must match a strict pattern, symlinks and the manifest are never served, the SHA-256 is computed from the file). **Verified end to end on 2026-10-15** against a scratch server on this machine: the Go client logged in, connected, answered a `tunnel` job for `reddit.com` (`via: tunnel`, status 403 because the donor was this same datacenter address) and stayed connected across repeated jobs. Not verified: a residential donor, the extension in a real browser, Windows and macOS builds running. + +## Gaps (what is not built or not measured) + +- **Reddit viability is unmeasured from a residential line.** The only real run used a datacenter donor and got 403, which proves the path, not the success rate. X, Instagram, LinkedIn, Facebook and TikTok stay unreachable (login walls); nothing here claims them. +- `country`, `asn` and `asn_class` on a device are never filled, so the datacenter exclusion (D6) is not enforced: the scheduler treats every node as eligible. It needs an offline ASN database file. +- `last_seen_at` is written on connect only. Socket counters are per process. The fabric cache is per process. Circuit breaker state is in memory. The same-process fast path is not built, callers poll at `poll_interval_ms`. +- Fetch-mode results are not cross-checked (sampled redundancy between two donors is not built). They are structurally checked and labelled `via: fetch`, and `tunnel` is preferred whenever a node can do it. +- `DeepSearch` still skips hostile domains with a static list; Reddit URLs from social search are used through their snippet. Routing a hostile URL fetch through `/nodes/internal/fetch` is not built. +- The Go client has no battery-aware pause (`pause_on_battery` is stored and sent but ignored by it), and treats a timed suspension as permanent until restarted. The extension enforces its own local hours in the browser clock, not the server timezone. +- Not measured: Reddit and X success rates per address class, WebSocket idle timeouts through `pravda.education` and `devplace.net`, socket capacity of one worker, a 24 hour soak. The caps in `LimitSpec` are starting ceilings, not measurements. +- **Launch is blocked on a recorded legal decision** (platform terms, donor terms, ISP terms, jurisdiction). Nothing here is enabled for any account: `nodes_enabled` is `0` until an administrator switches it on. ## Rules for work in this directory diff --git a/devplacepy/services/nodes/adapters/__init__.py b/devplacepy/services/nodes/adapters/__init__.py new file mode 100644 index 00000000..95ee7c3e --- /dev/null +++ b/devplacepy/services/nodes/adapters/__init__.py @@ -0,0 +1 @@ +# retoor diff --git a/devplacepy/services/nodes/adapters/base.py b/devplacepy/services/nodes/adapters/base.py new file mode 100644 index 00000000..c96d788e --- /dev/null +++ b/devplacepy/services/nodes/adapters/base.py @@ -0,0 +1,101 @@ +# retoor + +from __future__ import annotations + +import html +import json +import re +from dataclasses import dataclass +from typing import Any + +TITLE_MAX = 300 +SNIPPET_MAX = 500 +NAME_MAX = 80 +MAX_BODY_BYTES = 2 * 1024 * 1024 +TAG = re.compile(r"<[^>]+>") +WS = re.compile(r"\s+") +TAG_WORD = re.compile(r"[a-z0-9]{2,30}") + + +class AdapterError(Exception): + def __init__(self, code: str, message: str = ""): + super().__init__(message or code) + self.code = code + self.message = message or code + + +@dataclass(frozen=True) +class SocialHit: + platform: str + title: str + url: str + snippet: str = "" + author: str = "" + score: int = 0 + comments: int = 0 + created_at: str = "" + via: str = "direct" + + def as_dict(self) -> dict[str, Any]: + return dict(self.__dict__) + + +def clean(text: Any, limit: int) -> str: + if not isinstance(text, str): + return "" + plain = html.unescape(TAG.sub(" ", text)) + return WS.sub(" ", plain).strip()[:limit] + + +def number(value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0 + return max(0, int(value)) + + +def tag_for(query: str) -> str: + words = TAG_WORD.findall(query.lower()) + return max(words, key=len) if words else "" + + +def load_json(body: bytes) -> Any: + if len(body) > MAX_BODY_BYTES: + raise AdapterError("body_too_large") + try: + return json.loads(body) + except ValueError: + raise AdapterError("not_json", "the response is not JSON") + + +def expect(value: Any, kind: type, what: str) -> Any: + if not isinstance(value, kind): + raise AdapterError("bad_structure", f"{what} has an unexpected shape") + return value + + +class SearchAdapter: + name = "" + direct = True + + def request(self, query: str, limit: int) -> str: + raise NotImplementedError + + def parse(self, body: bytes, limit: int) -> list[SocialHit]: + raise NotImplementedError + + def hit(self, **fields: Any) -> SocialHit | None: + url = fields.get("url") or "" + title = clean(fields.get("title"), TITLE_MAX) + if not title or not isinstance(url, str) or not url.startswith(("http://", "https://")): + return None + return SocialHit( + platform=self.name, + title=title, + url=url[:2048], + snippet=clean(fields.get("snippet"), SNIPPET_MAX), + author=clean(fields.get("author"), NAME_MAX), + score=number(fields.get("score")), + comments=number(fields.get("comments")), + created_at=str(fields.get("created_at") or "")[:40], + via=fields.get("via") or "direct", + ) diff --git a/devplacepy/services/nodes/adapters/direct.py b/devplacepy/services/nodes/adapters/direct.py new file mode 100644 index 00000000..ef47f04b --- /dev/null +++ b/devplacepy/services/nodes/adapters/direct.py @@ -0,0 +1,197 @@ +# retoor + +from __future__ import annotations + +from urllib.parse import quote, urlencode + +import httpx + +from devplacepy import stealth +from devplacepy.services.nodes.adapters.base import ( + MAX_BODY_BYTES, + AdapterError, + SearchAdapter, + SocialHit, + clean, + expect, + load_json, + number, + tag_for, +) + +TIMEOUT_SECONDS = 12.0 +ACCEPT = {"Accept": "application/json"} + + +async def get(url: str) -> bytes: + timeout = httpx.Timeout(TIMEOUT_SECONDS, connect=6.0) + try: + async with stealth.stealth_async_client( + headers=ACCEPT, follow_redirects=True, timeout=timeout + ) as client: + response = await client.get(url) + except httpx.HTTPError as exc: + raise AdapterError("network", type(exc).__name__) + if response.status_code == 429: + raise AdapterError("rate_limited") + if response.status_code >= 400: + raise AdapterError("upstream", f"HTTP {response.status_code}") + body = response.content + if len(body) > MAX_BODY_BYTES: + raise AdapterError("body_too_large") + return body + + +class HackerNews(SearchAdapter): + name = "hackernews" + + def request(self, query: str, limit: int) -> str: + params = urlencode({"query": query, "tags": "story", "hitsPerPage": limit}) + return f"https://hn.algolia.com/api/v1/search?{params}" + + def parse(self, body: bytes, limit: int) -> list[SocialHit]: + data = expect(load_json(body), dict, "response") + hits = [] + for item in expect(data.get("hits"), list, "hits")[:limit]: + if not isinstance(item, dict): + continue + thread = f"https://news.ycombinator.com/item?id={quote(str(item.get('objectID') or ''))}" + found = self.hit( + title=item.get("title") or item.get("story_title"), + url=thread, + snippet=item.get("story_text") or item.get("url") or "", + author=item.get("author"), + score=item.get("points"), + comments=item.get("num_comments"), + created_at=item.get("created_at"), + ) + if found and item.get("objectID"): + hits.append(found) + return hits + + +class StackExchange(SearchAdapter): + name = "stackexchange" + + def request(self, query: str, limit: int) -> str: + params = urlencode( + { + "q": query, + "site": "stackoverflow", + "pagesize": limit, + "order": "desc", + "sort": "relevance", + } + ) + return f"https://api.stackexchange.com/2.3/search/advanced?{params}" + + def parse(self, body: bytes, limit: int) -> list[SocialHit]: + data = expect(load_json(body), dict, "response") + hits = [] + for item in expect(data.get("items"), list, "items")[:limit]: + if not isinstance(item, dict): + continue + owner = item.get("owner") if isinstance(item.get("owner"), dict) else {} + tags = item.get("tags") if isinstance(item.get("tags"), list) else [] + found = self.hit( + title=item.get("title"), + url=item.get("link"), + snippet=", ".join(str(tag) for tag in tags[:8]), + author=owner.get("display_name"), + score=item.get("score"), + comments=item.get("answer_count"), + created_at=item.get("creation_date"), + ) + if found: + hits.append(found) + return hits + + +class GitHub(SearchAdapter): + name = "github" + + def request(self, query: str, limit: int) -> str: + params = urlencode({"q": query, "per_page": limit}) + return f"https://api.github.com/search/issues?{params}" + + def parse(self, body: bytes, limit: int) -> list[SocialHit]: + data = expect(load_json(body), dict, "response") + hits = [] + for item in expect(data.get("items"), list, "items")[:limit]: + if not isinstance(item, dict): + continue + user = item.get("user") if isinstance(item.get("user"), dict) else {} + reactions = item.get("reactions") if isinstance(item.get("reactions"), dict) else {} + found = self.hit( + title=item.get("title"), + url=item.get("html_url"), + snippet=item.get("body"), + author=user.get("login"), + score=reactions.get("total_count"), + comments=item.get("comments"), + created_at=item.get("created_at"), + ) + if found: + hits.append(found) + return hits + + +class DevTo(SearchAdapter): + name = "devto" + + def request(self, query: str, limit: int) -> str: + tag = tag_for(query) + if not tag: + raise AdapterError("no_tag", "dev.to is searched by tag and the query has none") + return f"https://dev.to/api/articles?{urlencode({'tag': tag, 'per_page': limit})}" + + def parse(self, body: bytes, limit: int) -> list[SocialHit]: + data = expect(load_json(body), list, "response") + hits = [] + for item in data[:limit]: + if not isinstance(item, dict): + continue + user = item.get("user") if isinstance(item.get("user"), dict) else {} + found = self.hit( + title=item.get("title"), + url=item.get("url"), + snippet=item.get("description"), + author=user.get("username"), + score=item.get("public_reactions_count"), + comments=item.get("comments_count"), + created_at=item.get("published_at"), + ) + if found: + hits.append(found) + return hits + + +class Mastodon(SearchAdapter): + name = "mastodon" + + def request(self, query: str, limit: int) -> str: + tag = tag_for(query) + if not tag: + raise AdapterError("no_tag", "Mastodon is searched by hashtag and the query has none") + return f"https://mastodon.social/api/v1/timelines/tag/{quote(tag)}?{urlencode({'limit': min(limit, 40)})}" + + def parse(self, body: bytes, limit: int) -> list[SocialHit]: + data = expect(load_json(body), list, "response") + hits = [] + for item in data[:limit]: + if not isinstance(item, dict): + continue + account = item.get("account") if isinstance(item.get("account"), dict) else {} + content = clean(item.get("content"), 500) + found = self.hit( + title=content[:120], + url=item.get("url"), + snippet=content, + author=account.get("acct"), + score=number(item.get("favourites_count")) + number(item.get("reblogs_count")), + comments=item.get("replies_count"), + created_at=item.get("created_at"), + ) + if found: + hits.append(found) + return hits diff --git a/devplacepy/services/nodes/adapters/reddit.py b/devplacepy/services/nodes/adapters/reddit.py new file mode 100644 index 00000000..08399f79 --- /dev/null +++ b/devplacepy/services/nodes/adapters/reddit.py @@ -0,0 +1,43 @@ +# retoor + +from __future__ import annotations + +from urllib.parse import urlencode + +from devplacepy.services.nodes.adapters.base import SearchAdapter, SocialHit, expect, load_json + +BASE = "https://www.reddit.com" + + +class Reddit(SearchAdapter): + name = "reddit" + direct = False + + def request(self, query: str, limit: int) -> str: + params = urlencode({"q": query, "limit": limit, "raw_json": 1, "sort": "relevance"}) + return f"{BASE}/search.json?{params}" + + def parse(self, body: bytes, limit: int) -> list[SocialHit]: + data = expect(load_json(body), dict, "response") + listing = expect(data.get("data"), dict, "listing") + hits = [] + for child in expect(listing.get("children"), list, "children")[:limit]: + item = child.get("data") if isinstance(child, dict) else None + if not isinstance(item, dict): + continue + permalink = item.get("permalink") + if not isinstance(permalink, str) or not permalink.startswith("/r/"): + continue + found = self.hit( + title=item.get("title"), + url=BASE + permalink, + snippet=item.get("selftext"), + author=item.get("author"), + score=item.get("score"), + comments=item.get("num_comments"), + created_at=item.get("created_utc"), + via="fetch", + ) + if found: + hits.append(found) + return hits diff --git a/devplacepy/services/nodes/adapters/registry.py b/devplacepy/services/nodes/adapters/registry.py new file mode 100644 index 00000000..67408173 --- /dev/null +++ b/devplacepy/services/nodes/adapters/registry.py @@ -0,0 +1,12 @@ +# retoor + +from __future__ import annotations + +from devplacepy.services.nodes.adapters.base import SearchAdapter +from devplacepy.services.nodes.adapters.direct import DevTo, GitHub, HackerNews, Mastodon, StackExchange +from devplacepy.services.nodes.adapters.reddit import Reddit + +ADAPTERS: dict[str, SearchAdapter] = { + item.name: item for item in (HackerNews(), StackExchange(), GitHub(), DevTo(), Mastodon(), Reddit()) +} +DEFAULT_PLATFORMS = ("hackernews", "stackexchange", "github", "devto", "mastodon", "reddit") diff --git a/devplacepy/services/nodes/broker.py b/devplacepy/services/nodes/broker.py new file mode 100644 index 00000000..35f3bf40 --- /dev/null +++ b/devplacepy/services/nodes/broker.py @@ -0,0 +1,127 @@ +# retoor + +from __future__ import annotations + +import asyncio +import base64 +import binascii +import json +import time +from dataclasses import dataclass + +from devplacepy.database import nodes as store +from devplacepy.services.nodes.policy import Target +from devplacepy.services.nodes.spec import SPEC + +LIMITS = SPEC.limits + +DONE = "done" +FAILED = "failed" +EXPIRED = "expired" +TERMINAL = (DONE, FAILED, EXPIRED) + +MAX_HEADERS = 64 +MAX_HEADER_CHARS = 1024 +DROPPED_HEADERS = frozenset({"set-cookie", "set-cookie2", "authorization", "proxy-authenticate"}) + + +@dataclass(frozen=True) +class BrokerResult: + uid: str + status: str + http_status: int + headers: dict + body: bytes + truncated: bool + error: str + via: str = "fetch" + + @property + def ok(self) -> bool: + return self.status == DONE + + +class BodyError(ValueError): + pass + + +def clean_headers(raw) -> dict[str, str]: + if not isinstance(raw, dict): + return {} + kept: dict[str, str] = {} + for key, value in raw.items(): + if len(kept) >= MAX_HEADERS: + break + if not isinstance(key, str) or not isinstance(value, str): + continue + name = key.strip().lower() + if not name or name in DROPPED_HEADERS: + continue + kept[name[:MAX_HEADER_CHARS]] = value[:MAX_HEADER_CHARS] + return kept + + +def decode_body(body_b64: str) -> tuple[bytes, bool]: + try: + body = base64.b64decode(body_b64.encode(), validate=True) + except (binascii.Error, ValueError): + raise BodyError("bad_base64") + if len(body) > LIMITS.max_body_bytes: + return body[: LIMITS.max_body_bytes], True + return body, False + + +class JobBroker: + def enqueue(self, caller_kind: str, caller_id: str, target: Target, mode: str) -> str: + return store.enqueue_request( + caller_kind, caller_id, target.url, target.method, target.adapter, mode + ) + + def queued(self, limit: int = LIMITS.dispatch_batch) -> list[dict]: + return store.list_queued(limit) + + def claim(self, uid: str, device_uid: str) -> bool: + return store.claim_request(uid, device_uid) + + def complete(self, uid: str, http_status: int, headers: dict, body: bytes, truncated: bool) -> bool: + return store.finish_request( + uid, + result_status=http_status, + result_headers=json.dumps(headers), + result_body=base64.b64encode(body).decode(), + result_bytes=len(body), + truncated=truncated, + ) + + def fail(self, uid: str, error: str, *, expired: bool = False) -> bool: + return store.fail_request(uid, error, expired=expired) + + def expire(self) -> dict[str, int]: + return store.expire_requests() + + def read(self, uid: str) -> BrokerResult | None: + row = store.get_request(uid) + if row is None: + return None + return BrokerResult( + uid=row["uid"], + status=row["status"], + http_status=int(row.get("result_status") or 0), + headers=json.loads(row.get("result_headers") or "{}") if row.get("result_headers") else {}, + body=base64.b64decode(row["result_body"]) if row.get("result_body") else b"", + truncated=bool(row.get("truncated")), + error=row.get("error") or "", + via=row.get("mode") if row.get("mode") in ("tunnel", "fetch") else "fetch", + ) + + async def wait(self, uid: str, timeout: float) -> BrokerResult: + deadline = time.monotonic() + timeout + pause = LIMITS.poll_interval_ms / 1000 + while True: + row = store.get_request(uid) + if row is not None and row["status"] in TERMINAL: + return self.read(uid) + if time.monotonic() >= deadline: + self.fail(uid, "caller_timeout") + return self.read(uid) + await asyncio.sleep(pause) diff --git a/devplacepy/services/nodes/cache.py b/devplacepy/services/nodes/cache.py new file mode 100644 index 00000000..e0c24649 --- /dev/null +++ b/devplacepy/services/nodes/cache.py @@ -0,0 +1,90 @@ +# retoor + +from __future__ import annotations + +import hashlib +import time +from collections import OrderedDict +from dataclasses import dataclass +from typing import Callable + +from devplacepy.services.nodes.spec import SPEC + +LIMITS = SPEC.limits + + +@dataclass(frozen=True) +class CachedResult: + status: int + headers: dict + body: bytes + truncated: bool + stored_at: float + + +def cache_key(method: str, url: str) -> str: + return hashlib.sha256(f"{method.upper()} {url}".encode()).hexdigest() + + +class ResultCache: + def __init__( + self, + ttl_seconds: int = LIMITS.cache_ttl_seconds, + max_bytes: int = LIMITS.cache_max_bytes, + entry_max_bytes: int = LIMITS.cache_entry_max_bytes, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self.ttl_seconds = max(0, min(ttl_seconds, LIMITS.cache_ttl_seconds)) + self.max_bytes = max_bytes + self.entry_max_bytes = min(entry_max_bytes, LIMITS.cache_entry_max_bytes) + self._clock = clock + self._entries: OrderedDict[str, CachedResult] = OrderedDict() + self._bytes = 0 + self.hits = 0 + self.misses = 0 + + @property + def size_bytes(self) -> int: + return self._bytes + + def __len__(self) -> int: + return len(self._entries) + + def _drop(self, key: str) -> None: + entry = self._entries.pop(key, None) + if entry is not None: + self._bytes -= len(entry.body) + + def get(self, method: str, url: str) -> CachedResult | None: + key = cache_key(method, url) + entry = self._entries.get(key) + if entry is None: + self.misses += 1 + return None + if self._clock() - entry.stored_at >= self.ttl_seconds: + self._drop(key) + self.misses += 1 + return None + self._entries.move_to_end(key) + self.hits += 1 + return entry + + def put( + self, method: str, url: str, status: int, headers: dict, body: bytes, truncated: bool + ) -> bool: + if self.ttl_seconds <= 0 or status >= 500 or truncated: + return False + if len(body) > self.entry_max_bytes or len(body) > self.max_bytes: + return False + key = cache_key(method, url) + self._drop(key) + self._entries[key] = CachedResult(status, dict(headers), body, truncated, self._clock()) + self._bytes += len(body) + while self._bytes > self.max_bytes and self._entries: + oldest = next(iter(self._entries)) + self._drop(oldest) + return True + + def clear(self) -> None: + self._entries.clear() + self._bytes = 0 diff --git a/devplacepy/services/nodes/circuit.py b/devplacepy/services/nodes/circuit.py new file mode 100644 index 00000000..44019ea3 --- /dev/null +++ b/devplacepy/services/nodes/circuit.py @@ -0,0 +1,71 @@ +# retoor + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Callable, Hashable + +from devplacepy.services.nodes.spec import SPEC + +LIMITS = SPEC.limits + +CLOSED = "closed" +OPEN = "open" +HALF_OPEN = "half_open" + + +@dataclass +class _Circuit: + failures: int = 0 + opened_at: float = 0.0 + probing: bool = False + + +class CircuitBreaker: + def __init__( + self, + failures: int = LIMITS.breaker_failures, + open_seconds: float = LIMITS.breaker_open_seconds, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self.failures = max(1, failures) + self.open_seconds = open_seconds + self._clock = clock + self._circuits: dict[Hashable, _Circuit] = {} + + def state(self, key: Hashable) -> str: + circuit = self._circuits.get(key) + if circuit is None or circuit.failures < self.failures: + return CLOSED + if self._clock() - circuit.opened_at < self.open_seconds: + return OPEN + return HALF_OPEN + + def allows(self, key: Hashable) -> bool: + state = self.state(key) + if state == CLOSED: + return True + if state == OPEN: + return False + return not self._circuits[key].probing + + def begin(self, key: Hashable) -> None: + if self.state(key) == HALF_OPEN: + self._circuits[key].probing = True + + def success(self, key: Hashable) -> None: + self._circuits.pop(key, None) + + def failure(self, key: Hashable) -> None: + circuit = self._circuits.setdefault(key, _Circuit()) + circuit.probing = False + circuit.failures += 1 + if circuit.failures >= self.failures: + circuit.opened_at = self._clock() + + def forget(self, key: Hashable) -> None: + self._circuits.pop(key, None) + + def open_count(self) -> int: + return sum(1 for key in list(self._circuits) if self.state(key) != CLOSED) diff --git a/devplacepy/services/nodes/dispatcher.py b/devplacepy/services/nodes/dispatcher.py new file mode 100644 index 00000000..3187c886 --- /dev/null +++ b/devplacepy/services/nodes/dispatcher.py @@ -0,0 +1,273 @@ +# retoor + +from __future__ import annotations + +import asyncio +import logging +import time +from typing import Callable + +from devplacepy.database import nodes as store +from devplacepy.services.nodes.broker import BodyError, JobBroker, clean_headers, decode_body +from devplacepy.services.nodes.circuit import CircuitBreaker +from devplacepy.services.nodes.ledger import UsageLedger +from devplacepy.services.nodes.policy import DestinationPolicy, PolicyError, Target +from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry +from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler +from devplacepy.services.nodes.session import Inflight, NodeSession +from devplacepy.services.nodes.spec import MODE_FETCH, MODE_TUNNEL, SPEC +from devplacepy.services.nodes.tunnel import TunnelBridge, TunnelError + +logger = logging.getLogger(__name__) + +LIMITS = SPEC.limits + +JOB_HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; DevPlaceBot/1.0; +https://devplace.net)", + "Accept": "application/json, text/html;q=0.9, */*;q=0.5", +} + +REQUEST_OVERHEAD_BYTES = 256 +DONOR_LIMIT_PAUSE_SECONDS = 300 +CONCURRENCY_PAUSE_SECONDS = 5 +THROTTLED_STATUSES = (429,) +DONOR_LIMIT_CODES = { + "bandwidth_exhausted": None, + "outside_hours": DONOR_LIMIT_PAUSE_SECONDS, + "concurrency_exceeded": CONCURRENCY_PAUSE_SECONDS, +} +REFUSAL_CODES = frozenset(item.code for item in SPEC.refusals) - set(DONOR_LIMIT_CODES) - { + "body_too_large", + "timeout", +} + + +class NodeDispatcher: + def __init__( + self, + registry: NodeRegistry, + scheduler: NodeScheduler, + broker: JobBroker, + breaker: CircuitBreaker, + ledger: UsageLedger, + counters: NodeCounters, + policy: DestinationPolicy | None = None, + clock: Callable[[], float] = time.monotonic, + tunnels: TunnelBridge | None = None, + ) -> None: + self.tunnels = tunnels or TunnelBridge() + self._tunnel_tasks: dict[str, asyncio.Task] = {} + self.registry = registry + self.scheduler = scheduler + self.broker = broker + self.breaker = breaker + self.ledger = ledger + self.counters = counters + self.policy = policy or DestinationPolicy() + self._clock = clock + + def attach(self, session: NodeSession) -> None: + session.job_sink = self.on_job_frame + session.tunnel_sink = self.tunnels.on_frame + + async def tick(self) -> int: + if store.count_open_requests() == 0: + return 0 + self.broker.expire() + await self.sweep_inflight() + if self.registry.count() == 0: + return 0 + assigned = 0 + saturated: set[tuple[str, str]] = set() + for row in self.broker.queued(): + assigned += await self._assign(row, saturated) + return assigned + + async def _assign(self, row: dict, saturated: set) -> int: + uid = row["uid"] + try: + target = self.policy.check(row["url"], row["method"], row["mode"]) + except PolicyError as exc: + self.broker.fail(uid, exc.code) + self.counters.inc("jobs_refused") + return 0 + key = (target.destination.suffix, row["mode"]) + if key in saturated: + return 0 + owner = row["caller_id"] if row["caller_kind"] == "user" else "" + found = self.scheduler.candidates(target, row["mode"]) + if not found: + saturated.add(key) + return 0 + session = self.scheduler.choose(found, owner) + if session is None or not self.broker.claim(uid, session.device_uid): + return 0 + job = Inflight( + job_id=uid, + host=target.host, + destination=target.destination.suffix, + method=target.method, + url=target.url, + started=self._clock(), + bytes_up=len(target.url.encode()) + REQUEST_OVERHEAD_BYTES, + ) + session.inflight[uid] = job + self.scheduler.mark_started(session, target) + transport = self.scheduler.transport_of(session, target, row["mode"]) or MODE_FETCH + store.set_request_transport(uid, transport) + if transport == MODE_TUNNEL: + self._tunnel_tasks[uid] = asyncio.create_task(self._run_tunnel(session, job, target)) + self.counters.inc("jobs_assigned") + return 1 + try: + await session.send( + "job", + job_id=uid, + method=target.method, + url=target.url, + headers=JOB_HEADERS, + timeout=LIMITS.job_timeout_seconds, + ) + except Exception: + session.inflight.pop(uid, None) + self._failed(session, job, "send_failed") + return 0 + self.counters.inc("jobs_assigned") + return 1 + + def _failed(self, session: NodeSession, job: Inflight, code: str, node_fault: bool = True) -> None: + self.broker.fail(job.job_id, code) + self.counters.inc("jobs_failed") + if node_fault: + self.breaker.failure(("node", session.device_uid)) + store.add_event("job_failed", device_uid=session.device_uid, user_uid=session.user_uid, detail=code) + self.ledger.record_failure(session.device_uid, session.user_uid) + + async def on_job_frame(self, session: NodeSession, job: Inflight, frame: dict) -> None: + session.inflight.pop(job.job_id, None) + if frame["type"] == "job_error": + self._on_error(session, job, frame["code"]) + return + self._on_result(session, job, frame) + + def _integrity_failure(self, session: NodeSession, job: Inflight, code: str) -> None: + self.counters.inc("integrity_failures") + self._failed(session, job, code) + self.ledger.record_integrity_failure(session.device_uid, session.user_uid) + store.raise_flag( + "integrity", device_uid=session.device_uid, user_uid=session.user_uid, detail=code + ) + + def _on_error(self, session: NodeSession, job: Inflight, code: str) -> None: + if code in DONOR_LIMIT_CODES: + pause = DONOR_LIMIT_CODES[code] + if pause is None: + session.used_today = int(session.settings.get("bandwidth_mb_day") or 0) * MEGABYTE + else: + session.paused_until = self._clock() + pause + self._failed(session, job, code, node_fault=False) + return + if code in REFUSAL_CODES: + self.counters.inc("jobs_refused") + store.raise_flag( + "refusal_mismatch", + device_uid=session.device_uid, + user_uid=session.user_uid, + detail=f"{code} {job.destination}", + ) + self._failed(session, job, code, node_fault=False) + return + self._failed(session, job, code) + + def _on_result(self, session: NodeSession, job: Inflight, frame: dict) -> None: + status = frame["status"] + if not 100 <= status <= 599: + self._integrity_failure(session, job, "bad_status") + return + try: + body, over = decode_body(frame["body_b64"]) + except BodyError as exc: + self._integrity_failure(session, job, str(exc)) + return + truncated = bool(frame.get("truncated")) or over + if truncated: + self.counters.inc("results_truncated") + headers = clean_headers(frame.get("headers")) + if not self.broker.complete(job.job_id, status, headers, body, truncated): + return + self._account(session, job, status, job.bytes_up, len(body)) + + async def _run_tunnel(self, session: NodeSession, job: Inflight, target: Target) -> None: + try: + result = await self.tunnels.fetch(session, target, LIMITS.job_timeout_seconds) + except asyncio.CancelledError: + raise + except TunnelError as exc: + self._tunnel_ended(session, job, exc.code) + return + except Exception: + logger.exception("tunnel job %s failed", job.job_id) + self._tunnel_ended(session, job, "tunnel_failed") + return + finally: + self._tunnel_tasks.pop(job.job_id, None) + if session.inflight.pop(job.job_id, None) is None: + return + if result.truncated: + self.counters.inc("results_truncated") + if not self.broker.complete( + job.job_id, result.status, clean_headers(result.headers), result.body, result.truncated + ): + return + self._account(session, job, result.status, result.bytes_up, result.bytes_down) + + def _tunnel_ended(self, session: NodeSession, job: Inflight, code: str) -> None: + if session.inflight.pop(job.job_id, None) is not None: + self._failed(session, job, code) + + def _account(self, session: NodeSession, job: Inflight, status: int, up: int, down: int) -> None: + self.counters.inc("jobs_done") + spent = up + down + accepted = self.ledger.record_success( + session.device_uid, session.user_uid, bytes_up=up, bytes_down=down + ) + session.used_today += spent + if accepted: + self.ledger.reward(session.user_uid) + if not accepted: + session.used_today = int(session.settings.get("bandwidth_mb_day") or 0) * MEGABYTE + self.breaker.success(("node", session.device_uid)) + destination = ("destination", job.destination) + if status >= 500 or status in THROTTLED_STATUSES: + self.breaker.failure(destination) + else: + self.breaker.success(destination) + + async def sweep_inflight(self) -> int: + limit = LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds + now = self._clock() + swept = 0 + for session in self.registry.sessions(): + for job in list(session.inflight.values()): + if now - job.started > limit: + task = self._tunnel_tasks.pop(job.job_id, None) + if task is not None: + task.cancel() + session.inflight.pop(job.job_id, None) + self._failed(session, job, "timeout") + swept += 1 + return swept + + def release_session(self, session: NodeSession) -> int: + jobs = list(session.inflight.values()) + session.inflight.clear() + self.tunnels.release_session(session) + for job in jobs: + task = self._tunnel_tasks.pop(job.job_id, None) + if task is not None: + task.cancel() + self._failed(session, job, "node_disconnected") + return len(jobs) + + def target_for(self, url: str, method: str = "GET", mode: str = "any") -> Target: + return self.policy.check(url, method, mode) diff --git a/devplacepy/services/nodes/fabric.py b/devplacepy/services/nodes/fabric.py new file mode 100644 index 00000000..205c882a --- /dev/null +++ b/devplacepy/services/nodes/fabric.py @@ -0,0 +1,204 @@ +# retoor + +from __future__ import annotations + +import json +import time +from dataclasses import dataclass +from datetime import datetime, timezone + +from devplacepy.database import get_int_setting, get_table +from devplacepy.database import nodes as store +from devplacepy.services.base import BaseService +from devplacepy.services.nodes.broker import FAILED, JobBroker +from devplacepy.services.nodes.cache import ResultCache +from devplacepy.services.nodes.consent import ConsentGate +from devplacepy.services.nodes.policy import DestinationPolicy, PolicyError, Target +from devplacepy.services.nodes.spec import SPEC + +LIMITS = SPEC.limits + +VIA_FETCH = "fetch" +VIA_CACHE = "cache" +RETRYABLE = frozenset( + { + "node_disconnected", + "timeout", + "send_failed", + "concurrency_exceeded", + "outside_hours", + "bandwidth_exhausted", + } +) +MAX_ATTEMPTS = 2 +GUEST_KIND = "guest" +USER_KIND = "user" + + +class FabricError(Exception): + def __init__(self, code: str, message: str = ""): + super().__init__(message or code) + self.code = code + self.message = message or code + + +class FabricRefused(FabricError): + pass + + +class FabricUnavailable(FabricError): + pass + + +class FabricQuotaExceeded(FabricError): + pass + + +@dataclass(frozen=True) +class FabricResult: + url: str + status: int + headers: dict + body: bytes + via: str + truncated: bool + elapsed_ms: int + + +@dataclass(frozen=True) +class FabricStatus: + enabled: bool + nodes_online: int + + +class NodeFabric: + def __init__( + self, + policy: DestinationPolicy | None = None, + broker: JobBroker | None = None, + cache: ResultCache | None = None, + gate: ConsentGate | None = None, + ) -> None: + self.policy = policy or DestinationPolicy() + self.broker = broker or JobBroker() + self.cache = cache or ResultCache() + self.gate = gate or ConsentGate() + + def available(self) -> FabricStatus: + enabled = self.gate.switched_on() + return FabricStatus(enabled, self._online() if enabled else 0) + + @staticmethod + def _online() -> int: + row = get_table("service_state").find_one(name="nodes") + if not row or not row.get("heartbeat"): + return 0 + try: + beat = datetime.fromisoformat(row["heartbeat"]) + except ValueError: + return 0 + if beat.tzinfo is None: + beat = beat.replace(tzinfo=timezone.utc) + if (datetime.now(timezone.utc) - beat).total_seconds() > BaseService.STALE_SECONDS * 2: + return 0 + try: + stats = json.loads(row.get("metrics") or "{}").get("stats") or [] + except ValueError: + return 0 + for item in stats: + if item.get("label") == "connected": + return int(item.get("value") or 0) + return 0 + + def can_serve(self, url: str, method: str = "GET") -> bool: + try: + self.policy.check(url, method) + except PolicyError: + return False + status = self.available() + return status.enabled and status.nodes_online > 0 + + def _quota(self, caller_kind: str) -> int: + if caller_kind == USER_KIND: + return get_int_setting("nodes_daily_quota_user", LIMITS.default_daily_requests_user) + return get_int_setting("nodes_daily_quota_guest", LIMITS.default_daily_requests_guest) + + def _sync_cache_ttl(self) -> None: + configured = get_int_setting("nodes_cache_ttl_seconds", LIMITS.cache_ttl_seconds) + self.cache.ttl_seconds = max(0, min(configured, LIMITS.cache_ttl_seconds)) + + async def fetch( + self, + url: str, + *, + caller_kind: str, + caller_id: str, + method: str = "GET", + mode: str = "any", + use_cache: bool = True, + timeout: float | None = None, + ) -> FabricResult: + try: + target = self.policy.check(url, method, mode) + except PolicyError as exc: + raise FabricRefused(exc.code, exc.reason) + started = time.monotonic() + self._sync_cache_ttl() + if use_cache: + cached = self.cache.get(target.method, target.url) + if cached is not None: + return FabricResult( + target.url, cached.status, cached.headers, cached.body, VIA_CACHE, cached.truncated, 0 + ) + status = self.available() + if not status.enabled: + raise FabricUnavailable("kill_switch", "Community donation is switched off") + if status.nodes_online <= 0: + raise FabricUnavailable("no_nodes", "No donor node is connected") + if not store.reserve_caller_request(caller_kind, caller_id, self._quota(caller_kind)): + raise FabricQuotaExceeded("quota", "Daily node request quota reached") + wait = min(float(timeout or LIMITS.default_wait_seconds), float(LIMITS.max_wait_seconds)) + result = await self._run(target, caller_kind, caller_id, mode, wait) + if use_cache: + self.cache.put( + target.method, target.url, result.http_status, result.headers, result.body, result.truncated + ) + return FabricResult( + target.url, + result.http_status, + result.headers, + result.body, + result.via, + result.truncated, + int((time.monotonic() - started) * 1000), + ) + + async def _run(self, target: Target, caller_kind: str, caller_id: str, mode: str, wait: float): + deadline = time.monotonic() + wait + error = "" + for _ in range(MAX_ATTEMPTS): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + uid = self.broker.enqueue(caller_kind, caller_id, target, mode) + except store.QueueFull: + raise FabricUnavailable("queue_full", "The node queue is full") + result = await self.broker.wait(uid, remaining) + if result.ok: + return result + error = result.error or result.status + if result.status == FAILED and result.error in RETRYABLE: + continue + break + raise FabricUnavailable(str(error) or "failed", f"Node fetch failed: {error}") + + +_shared: NodeFabric | None = None + + +def shared_fabric() -> NodeFabric: + global _shared + if _shared is None: + _shared = NodeFabric() + return _shared diff --git a/devplacepy/services/nodes/ledger.py b/devplacepy/services/nodes/ledger.py index 475f444d..45978278 100644 --- a/devplacepy/services/nodes/ledger.py +++ b/devplacepy/services/nodes/ledger.py @@ -7,6 +7,17 @@ from datetime import datetime, timezone from devplacepy.database import nodes as store +XP_PER_JOB = 1 +GIGABYTE = 1024 * 1024 * 1024 +BYTE_BADGES = ( + (GIGABYTE, "Gigabyte Donor"), + (10 * GIGABYTE, "10 GB Donor"), + (100 * GIGABYTE, "100 GB Donor"), +) +FIRST_BADGE = "Donor" +STEADY_DAYS = 30 +STEADY_BADGE = "Steady Donor" + @dataclass(frozen=True) class DonationSummary: @@ -49,6 +60,20 @@ class UsageLedger: def record_integrity_failure(self, device_uid: str, user_uid: str) -> None: store.record_integrity_failure(device_uid, user_uid) + def reward(self, user_uid: str) -> None: + from devplacepy.utils import award_badge, award_rewards, notify_badge + + if self.claim_xp(user_uid, XP_PER_JOB): + award_rewards(user_uid, XP_PER_JOB) + donor = store.get_donor(user_uid) + earned = [FIRST_BADGE] if donor["requests_ok"] >= 1 else [] + earned += [name for size, name in BYTE_BADGES if donor["bytes_total"] >= size] + if donor["requests_ok"] >= STEADY_DAYS and store.donation_days(user_uid) >= STEADY_DAYS: + earned.append(STEADY_BADGE) + for name in earned: + if award_badge(user_uid, name): + notify_badge(user_uid, name) + def claim_xp(self, user_uid: str, amount: int) -> bool: return store.claim_xp(user_uid, amount) diff --git a/devplacepy/services/nodes/policy.py b/devplacepy/services/nodes/policy.py new file mode 100644 index 00000000..1ecfd19f --- /dev/null +++ b/devplacepy/services/nodes/policy.py @@ -0,0 +1,91 @@ +# retoor + +from __future__ import annotations + +import ipaddress +from dataclasses import dataclass +from urllib.parse import urlsplit + +from devplacepy.services.nodes.spec import ALLOWED_PORTS, MODES, SPEC, DestinationSpec + +LIMITS = SPEC.limits +DEFAULT_PORTS = {"http": 80, "https": 443} +SCHEMES = tuple(DEFAULT_PORTS) + +SHARED_ADDRESS_SPACE = ipaddress.ip_network("100.64.0.0/10") + + +class PolicyError(ValueError): + def __init__(self, code: str, reason: str = ""): + super().__init__(reason or code) + self.code = code + self.reason = reason or code + + +@dataclass(frozen=True) +class Target: + url: str + method: str + host: str + port: int + destination: DestinationSpec + + @property + def adapter(self) -> str: + return self.destination.adapter + + +def is_public_address(address: str) -> bool: + try: + ip = ipaddress.ip_address(address) + except ValueError: + return False + if ip.version == 4 and ip in SHARED_ADDRESS_SPACE: + return False + return ip.is_global and not ip.is_multicast + + +class DestinationPolicy: + def check(self, url: str, method: str = "GET", mode: str = "any") -> Target: + if mode not in MODES: + raise PolicyError("bad_mode", f"unknown mode {mode!r}") + if not isinstance(url, str) or not url or len(url) > LIMITS.url_max_chars: + raise PolicyError("bad_url", "url is missing or too long") + try: + parts = urlsplit(url.strip()) + port = parts.port + except ValueError: + raise PolicyError("bad_url", "url cannot be parsed") + if parts.scheme not in SCHEMES: + raise PolicyError("bad_url", "only http and https are allowed") + if parts.username is not None or parts.password is not None: + raise PolicyError("bad_url", "credentials in a url are not allowed") + host = (parts.hostname or "").lower().rstrip(".") + if not host: + raise PolicyError("bad_url", "url has no host") + if _is_address_literal(host): + raise PolicyError("host_not_allowed", "address literals are not allowed") + destination = SPEC.destination(host) + if destination is None: + raise PolicyError("host_not_allowed", "destination is not on the allowlist") + port = port or DEFAULT_PORTS[parts.scheme] + if port not in ALLOWED_PORTS or port not in destination.ports: + raise PolicyError("port_not_allowed", "port is not allowed") + verb = (method or "").upper() + if verb not in destination.methods: + raise PolicyError("method_not_allowed", "method is not allowed here") + return Target(parts.geturl(), verb, host, port, destination) + + def check_redirect(self, location: str, method: str = "GET") -> Target: + try: + return self.check(location, method) + except PolicyError as exc: + raise PolicyError("redirect_not_allowed", exc.reason) + + +def _is_address_literal(host: str) -> bool: + try: + ipaddress.ip_address(host.strip("[]")) + return True + except ValueError: + return False diff --git a/devplacepy/services/nodes/registry.py b/devplacepy/services/nodes/registry.py index 92e4c20f..f3ceb487 100644 --- a/devplacepy/services/nodes/registry.py +++ b/devplacepy/services/nodes/registry.py @@ -20,6 +20,12 @@ COUNTER_NAMES: tuple[str, ...] = ( "evicted_heartbeat", "kicked", "replaced", + "jobs_assigned", + "jobs_done", + "jobs_failed", + "jobs_refused", + "integrity_failures", + "results_truncated", ) diff --git a/devplacepy/services/nodes/scheduler.py b/devplacepy/services/nodes/scheduler.py new file mode 100644 index 00000000..3b1708a0 --- /dev/null +++ b/devplacepy/services/nodes/scheduler.py @@ -0,0 +1,163 @@ +# retoor + +from __future__ import annotations + +import random +import time +from functools import lru_cache +from datetime import datetime, timezone +from typing import Callable +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from devplacepy.services.nodes.circuit import CircuitBreaker +from devplacepy.services.nodes.policy import Target +from devplacepy.services.nodes.registry import NodeRegistry +from devplacepy.services.nodes.session import NodeSession +from devplacepy.services.nodes.spec import MODE_ANY, MODE_FETCH, MODE_TUNNEL, SPEC + +LIMITS = SPEC.limits +MEGABYTE = 1024 * 1024 +MINUTES_PER_DAY = 24 * 60 + +SUPPORTED_MODES = (MODE_TUNNEL, MODE_FETCH) + + +def _minutes(value: str) -> int | None: + try: + hours, minutes = value.split(":") + return int(hours) * 60 + int(minutes) + except (ValueError, AttributeError): + return None + + +@lru_cache(maxsize=256) +def _zone(name: str): + try: + return ZoneInfo(name or "UTC") + except (ZoneInfoNotFoundError, ValueError): + return timezone.utc + + +@lru_cache(maxsize=8192) +def _window_open(start: str, end: str, zone_name: str, minute_epoch: int) -> bool: + first, last = _minutes(start), _minutes(end) + if first is None or last is None or first == last: + return True + moment = datetime.fromtimestamp(minute_epoch * 60, timezone.utc) + local = moment.astimezone(_zone(zone_name)) + now = local.hour * 60 + local.minute + if first < last: + return first <= now < last + return now >= first or now < last + + +def in_active_hours(settings: dict, moment: datetime) -> bool: + start, end = settings.get("active_from") or "", settings.get("active_to") or "" + if not start and not end: + return True + return _window_open(start, end, settings.get("timezone") or "UTC", int(moment.timestamp() // 60)) + + +class NodeScheduler: + def __init__( + self, + registry: NodeRegistry, + breaker: CircuitBreaker, + clock: Callable[[], float] = time.monotonic, + wall: Callable[[], datetime] = lambda: datetime.now(timezone.utc), + rng: random.Random | None = None, + ) -> None: + self.registry = registry + self.breaker = breaker + self._clock = clock + self._wall = wall + self._rng = rng or random.Random() + self._destination_started: dict[str, float] = {} + + @staticmethod + def node_key(session: NodeSession) -> tuple[str, str]: + return ("node", session.device_uid) + + @staticmethod + def destination_key(target: Target) -> tuple[str, str]: + return ("destination", target.destination.suffix) + + def transports_for(self, target: Target, mode: str) -> tuple[str, ...]: + if mode not in (MODE_ANY, *SUPPORTED_MODES): + return () + return tuple( + item + for item in SUPPORTED_MODES + if item in target.destination.modes and mode in (MODE_ANY, item) + ) + + def transport_of(self, session: NodeSession, target: Target, mode: str) -> str | None: + for item in self.transports_for(target, mode): + if item in session.modes: + return item + return None + + def destination_ready(self, target: Target) -> bool: + interval = 1.0 / max(target.destination.max_rps, 0.001) + last = self._destination_started.get(target.destination.suffix) + return last is None or self._clock() - last >= interval + + def node_ready(self, session: NodeSession, target: Target, transport: str) -> bool: + if session.closed or self._clock() < session.paused_until: + return False + if transport not in session.modes: + return False + if len(session.inflight) >= session.max_concurrent: + return False + if target.adapter not in (session.settings.get("allowed_platforms") or []): + return False + if not in_active_hours(session.settings, self._wall()): + return False + session.roll_day() + cap = int(session.settings.get("bandwidth_mb_day") or 0) * MEGABYTE + if session.used_today >= cap: + return False + last = session.host_started.get(target.destination.suffix) + if last is not None and self._clock() - last < LIMITS.min_seconds_per_node_domain: + return False + return self.breaker.allows(self.node_key(session)) + + def candidates(self, target: Target, mode: str = MODE_ANY) -> list[NodeSession]: + if not self.destination_ready(target) or not self.breaker.allows(self.destination_key(target)): + return [] + for transport in self.transports_for(target, mode): + found = [ + session + for session in self.registry.sessions() + if self.node_ready(session, target, transport) + ] + if found: + return found + return [] + + def eligible(self, target: Target, mode: str = MODE_ANY, caller_owner: str = "") -> list[NodeSession]: + found = self.candidates(target, mode) + if not caller_owner: + return found + return [session for session in found if session.user_uid != caller_owner] + + def choose(self, found: list[NodeSession], caller_owner: str = "") -> NodeSession | None: + if caller_owner: + found = [session for session in found if session.user_uid != caller_owner] + if not found: + return None + return min( + found, + key=lambda item: (len(item.inflight), item.last_assigned, self._rng.random()), + ) + + def pick(self, target: Target, mode: str = MODE_ANY, caller_owner: str = "") -> NodeSession | None: + return self.choose(self.candidates(target, mode), caller_owner) + + def mark_started(self, session: NodeSession, target: Target) -> None: + now = self._clock() + session.last_assigned = now + session.host_started[target.destination.suffix] = now + self._destination_started[target.destination.suffix] = now + self.breaker.begin(self.node_key(session)) + self.breaker.begin(self.destination_key(target)) diff --git a/devplacepy/services/nodes/service.py b/devplacepy/services/nodes/service.py index fe35f4a3..2b3ba33e 100644 --- a/devplacepy/services/nodes/service.py +++ b/devplacepy/services/nodes/service.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import logging import time @@ -10,11 +11,17 @@ from devplacepy.database.nodes import ( add_event, effective_status, get_devices_by_uids, + get_settings_by_users, ) from devplacepy.services.base import BaseService, ConfigField +from devplacepy.services.nodes.broker import JobBroker +from devplacepy.services.nodes.circuit import CircuitBreaker from devplacepy.services.nodes.consent import ConsentGate +from devplacepy.services.nodes.dispatcher import NodeDispatcher +from devplacepy.services.nodes.fabric import NodeFabric from devplacepy.services.nodes.ledger import UsageLedger from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry +from devplacepy.services.nodes.scheduler import NodeScheduler from devplacepy.services.nodes.spec import SPEC logger = logging.getLogger(__name__) @@ -64,7 +71,52 @@ class NodesService(BaseService): self.registry = NodeRegistry(self.counters) self.gate = ConsentGate() self.ledger = UsageLedger() + self.breaker = CircuitBreaker() + self.broker = JobBroker() + self.scheduler = NodeScheduler(self.registry, self.breaker) + self.dispatcher = NodeDispatcher( + self.registry, + self.scheduler, + self.broker, + self.breaker, + self.ledger, + self.counters, + ) + self.fabric = NodeFabric(broker=self.broker, gate=self.gate) self._last_prune = 0.0 + self._dispatch_task: asyncio.Task | None = None + + def ensure_dispatching(self) -> None: + loop = asyncio.get_running_loop() + task = self._dispatch_task + if task is not None and not task.done() and task.get_loop() is loop: + return + self._dispatch_task = loop.create_task(self._dispatch_loop()) + + async def _dispatch_loop(self) -> None: + pause = LIMITS.dispatcher_tick_ms / 1000 + while True: + try: + await self.dispatcher.tick() + except asyncio.CancelledError: + raise + except Exception: + logger.exception("node dispatch tick failed") + await asyncio.sleep(pause) + + async def on_enable(self) -> None: + self.ensure_dispatching() + + async def _refresh_settings(self, sessions) -> None: + fresh = get_settings_by_users(item.user_uid for item in sessions) + for session in sessions: + current = fresh.get(session.user_uid) + if current is not None and current != session.settings: + session.settings = current + try: + await session.send("settings_update", settings=current) + except Exception: + pass async def reconcile(self) -> int: sessions = self.registry.sessions() @@ -87,6 +139,7 @@ class NodesService(BaseService): detail=reason, ) kicked += 1 + await self._refresh_settings(self.registry.sessions()) return kicked @staticmethod @@ -113,6 +166,9 @@ class NodesService(BaseService): self.log(f"pruned {pruned}") async def on_disable(self) -> None: + if self._dispatch_task is not None: + self._dispatch_task.cancel() + self._dispatch_task = None await self.registry.kick_all("service_stopped") def collect_metrics(self) -> dict: @@ -123,6 +179,9 @@ class NodesService(BaseService): {"label": "capacity", "value": self.registry.capacity()}, {"label": "accepted", "value": counts["connected_total"]}, {"label": "kicked", "value": counts["kicked"]}, + {"label": "jobs done", "value": counts["jobs_done"]}, + {"label": "jobs failed", "value": counts["jobs_failed"]}, + {"label": "open circuits", "value": self.breaker.open_count()}, ], "table": { "columns": ["counter", "value"], diff --git a/devplacepy/services/nodes/session.py b/devplacepy/services/nodes/session.py index 86266b57..5a33d462 100644 --- a/devplacepy/services/nodes/session.py +++ b/devplacepy/services/nodes/session.py @@ -4,8 +4,11 @@ from __future__ import annotations import asyncio import time +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Awaitable, Callable -from devplacepy.database.nodes import get_node_settings +from devplacepy.database.nodes import get_node_settings, usage_today from devplacepy.services.nodes.frames import FrameError, decode, encode from devplacepy.services.nodes.spec import ( CLIENT_TO_SERVER, @@ -20,6 +23,21 @@ LIMITS = SPEC.limits END_HEARTBEAT = "heartbeat" END_UNEXPECTED_FRAME = "unexpected_frame" +TUNNEL_FRAMES = ("tunnel_opened", "tunnel_data", "tunnel_close") + + +@dataclass +class Inflight: + job_id: str + host: str + destination: str + method: str + url: str + started: float + bytes_up: int + + +JobSink = Callable[["NodeSession", Inflight, dict], Awaitable[None]] class NodeSession: @@ -34,6 +52,37 @@ class NodeSession: self.reported_up = 0 self.reported_down = 0 self.closed = False + self.settings: dict = {} + self.inflight: dict[str, Inflight] = {} + self.host_started: dict[str, float] = {} + self.used_today = 0 + self.used_day = "" + self.paused_until = 0.0 + self.last_assigned = 0.0 + self.job_sink: JobSink | None = None + self.tunnel_sink: Callable[["NodeSession", dict], bool] | None = None + self.tunnels: dict = {} + + @property + def modes(self) -> list[str]: + return list(self.hello.get("modes") or []) + + @property + def max_concurrent(self) -> int: + limit = min( + LIMITS.max_concurrent_per_node, + int(self.settings.get("max_concurrent") or LIMITS.max_concurrent_per_node), + ) + declared = (self.hello.get("limits") or {}).get("max_concurrent") + if isinstance(declared, int) and not isinstance(declared, bool) and declared >= 1: + limit = min(limit, declared) + return max(1, limit) + + def roll_day(self) -> None: + today = datetime.now(timezone.utc).date().isoformat() + if self.used_day != today: + self.used_day = today + self.used_today = 0 async def send(self, frame_type: str, **fields) -> None: await self.websocket.send_text(encode(frame_type, SERVER_TO_CLIENT, **fields)) @@ -76,16 +125,20 @@ class NodeSession: raise FrameError("expected_hello", "the first frame must be hello") self._check_hello(frame) self.hello = frame + self.settings = get_node_settings(self.user_uid) + self.roll_day() + today = usage_today(self.device_uid) + self.used_today = today["bytes_up"] + today["bytes_down"] await self.send( "welcome", device_uid=self.device_uid, limits=LIMITS.describe(), destinations=[item.describe() for item in SPEC.destinations], - settings=get_node_settings(self.user_uid), + settings=self.settings, ) return frame - def _handle(self, frame: dict) -> None: + async def _handle(self, frame: dict) -> None: kind = frame["type"] if kind == "pong": return @@ -93,7 +146,18 @@ class NodeSession: self.reported_up += max(0, frame["bytes_up"]) self.reported_down += max(0, frame["bytes_down"]) return - raise FrameError(END_UNEXPECTED_FRAME, f"unexpected {kind} frame, no job is assigned") + if kind in TUNNEL_FRAMES: + if self.tunnel_sink is not None: + self.tunnel_sink(self, frame) + return + if kind in ("job_result", "job_error"): + job = self.inflight.get(frame["job_id"]) + if job is not None and self.job_sink is not None: + await self.job_sink(self, job, frame) + return + raise FrameError( + END_UNEXPECTED_FRAME, f"unexpected {kind} frame, no such job is assigned" + ) async def run(self) -> str: while not self.closed: @@ -109,5 +173,5 @@ class NodeSession: await self.send("ping") continue self.missed_pings = 0 - self._handle(decode(raw, CLIENT_TO_SERVER)) + await self._handle(decode(raw, CLIENT_TO_SERVER)) return "closed" diff --git a/devplacepy/services/nodes/social.py b/devplacepy/services/nodes/social.py new file mode 100644 index 00000000..471b896a --- /dev/null +++ b/devplacepy/services/nodes/social.py @@ -0,0 +1,96 @@ +# retoor + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from itertools import zip_longest + +from devplacepy.database import get_setting +from devplacepy.services.nodes.adapters.base import AdapterError, SearchAdapter, SocialHit +from devplacepy.services.nodes.adapters.direct import get as direct_get +from devplacepy.services.nodes.adapters.registry import ADAPTERS, DEFAULT_PLATFORMS +from devplacepy.services.nodes.fabric import FabricError, NodeFabric, shared_fabric + +MAX_LIMIT = 20 +DEFAULT_LIMIT = 8 +MAX_QUERY_CHARS = 200 +MAX_TOTAL_HITS = 60 +PLATFORM_TIMEOUT_SECONDS = 20.0 + + +@dataclass +class SocialOutcome: + hits: list[SocialHit] = field(default_factory=list) + unavailable: dict[str, str] = field(default_factory=dict) + + def as_dict(self) -> dict: + return { + "hits": [item.as_dict() for item in self.hits], + "unavailable": dict(self.unavailable), + } + + +class SocialSearch: + def __init__(self, fabric: NodeFabric | None = None) -> None: + self.fabric = fabric or shared_fabric() + + @staticmethod + def platforms_for(requested: list[str] | None) -> list[str]: + if not requested: + return list(DEFAULT_PLATFORMS) + return [name for name in dict.fromkeys(requested) if name in ADAPTERS] + + async def search( + self, + query: str, + *, + caller_kind: str, + caller_id: str, + platforms: list[str] | None = None, + limit: int = DEFAULT_LIMIT, + ) -> SocialOutcome: + query = " ".join(str(query or "").split())[:MAX_QUERY_CHARS] + outcome = SocialOutcome() + if not query: + return outcome + limit = max(1, min(int(limit or DEFAULT_LIMIT), MAX_LIMIT)) + names = self.platforms_for(platforms) + results = await asyncio.gather( + *(self._one(ADAPTERS[name], query, limit, caller_kind, caller_id) for name in names) + ) + buckets = [] + for name, (hits, code) in zip(names, results): + if code: + outcome.unavailable[name] = code + buckets.append(hits) + for tier in zip_longest(*buckets): + outcome.hits.extend(item for item in tier if item) + outcome.hits = outcome.hits[:MAX_TOTAL_HITS] + return outcome + + async def _one( + self, adapter: SearchAdapter, query: str, limit: int, caller_kind: str, caller_id: str + ) -> tuple[list[SocialHit], str]: + try: + return await asyncio.wait_for( + self._fetch(adapter, query, limit, caller_kind, caller_id), + PLATFORM_TIMEOUT_SECONDS, + ), "" + except asyncio.TimeoutError: + return [], "timeout" + except (AdapterError, FabricError) as exc: + return [], exc.code + + async def _fetch( + self, adapter: SearchAdapter, query: str, limit: int, caller_kind: str, caller_id: str + ) -> list[SocialHit]: + url = adapter.request(query, limit) + if adapter.direct: + return adapter.parse(await direct_get(url), limit) + if get_setting("nodes_social_via_nodes", "0") != "1": + raise AdapterError("nodes_not_used", "Routing through donor nodes is switched off") + result = await self.fabric.fetch(url, caller_kind=caller_kind, caller_id=caller_id) + if result.status >= 400: + raise AdapterError("upstream", f"HTTP {result.status}") + return adapter.parse(result.body, limit) diff --git a/devplacepy/services/nodes/spec.py b/devplacepy/services/nodes/spec.py index 402c5064..36929ae7 100644 --- a/devplacepy/services/nodes/spec.py +++ b/devplacepy/services/nodes/spec.py @@ -97,6 +97,13 @@ class LimitSpec: hello_timeout_seconds: int = 10 reconcile_seconds: int = 2 prune_interval_seconds: int = 3600 + breaker_failures: int = 5 + breaker_open_seconds: int = 60 + queue_max_wait_seconds: int = 30 + job_grace_seconds: int = 5 + dispatch_batch: int = 50 + url_max_chars: int = 2048 + poll_interval_ms: int = 100 def describe(self) -> dict[str, int]: return dict(self.__dict__) diff --git a/devplacepy/services/nodes/tunnel.py b/devplacepy/services/nodes/tunnel.py new file mode 100644 index 00000000..a60c7907 --- /dev/null +++ b/devplacepy/services/nodes/tunnel.py @@ -0,0 +1,220 @@ +# retoor + +from __future__ import annotations + +import asyncio +import base64 +import binascii +import logging +from dataclasses import dataclass + +import httpx + +from devplacepy import stealth +from devplacepy.services.nodes.policy import Target +from devplacepy.services.nodes.session import NodeSession +from devplacepy.services.nodes.spec import SPEC +from devplacepy.utils import generate_uid + +logger = logging.getLogger(__name__) + +LIMITS = SPEC.limits + +LOOPBACK = "127.0.0.1" +CHUNK_BYTES = 48 * 1024 +REQUEST_HEAD_MAX = 8 * 1024 +OPEN_TIMEOUT_SECONDS = 10.0 +POLL_SECONDS = 1.0 +TLS_OVERHEAD_BYTES = 512 * 1024 +RELAY_CAP_BYTES = LIMITS.max_body_bytes * 2 + TLS_OVERHEAD_BYTES +CLOSED = object() + + +class TunnelError(Exception): + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +@dataclass(frozen=True) +class TunnelResult: + status: int + headers: dict + body: bytes + truncated: bool + bytes_up: int + bytes_down: int + + +class Relay: + def __init__(self, session: NodeSession, target: Target) -> None: + self.session = session + self.target = target + self.conn_id = generate_uid() + self.frames: asyncio.Queue = asyncio.Queue(maxsize=256) + self.up = 0 + self.down = 0 + self.error = "" + self.used = False + + def on_frame(self, frame: dict) -> None: + try: + self.frames.put_nowait(frame) + except asyncio.QueueFull: + self.error = "queue_full" + + def drop(self) -> None: + try: + self.frames.put_nowait(CLOSED) + except asyncio.QueueFull: + pass + + async def handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + await self._serve(reader, writer) + except (ConnectionError, asyncio.IncompleteReadError, asyncio.LimitOverrunError): + pass + finally: + writer.close() + + async def _serve(self, reader, writer) -> None: + head = await reader.readuntil(b"\r\n\r\n") + if len(head) > REQUEST_HEAD_MAX: + return + parts = head.split(b"\r\n", 1)[0].decode("latin-1", "replace").split() + if len(parts) < 2 or parts[0].upper() != "CONNECT" or self.used: + await self._refuse(writer, 405) + return + host, _, port = parts[1].rpartition(":") + if host.lower() != self.target.host.lower() or port != str(self.target.port): + self.error = "host_not_allowed" + await self._refuse(writer, 403) + return + self.used = True + self.session.tunnels[self.conn_id] = self + await self.session.send("tunnel_open", conn_id=self.conn_id, host=self.target.host, port=self.target.port) + if not await self._opened(): + await self._refuse(writer, 502) + return + writer.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + await writer.drain() + uplink = asyncio.create_task(self._uplink(reader)) + try: + await self._downlink(writer) + finally: + uplink.cancel() + await self._close_donor() + + @staticmethod + async def _refuse(writer, status: int) -> None: + writer.write(f"HTTP/1.1 {status} Refused\r\nContent-Length: 0\r\n\r\n".encode()) + await writer.drain() + + async def _opened(self) -> bool: + try: + frame = await asyncio.wait_for(self.frames.get(), OPEN_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + self.error = "tunnel_open_timeout" + return False + if frame is CLOSED or frame["type"] != "tunnel_opened" or not frame["ok"]: + self.error = (frame.get("code") if isinstance(frame, dict) else "") or "tunnel_refused" + return False + return True + + async def _uplink(self, reader) -> None: + while True: + chunk = await reader.read(CHUNK_BYTES) + if not chunk: + return + self.up += len(chunk) + if self.up + self.down > RELAY_CAP_BYTES: + self.error = "body_too_large" + self.drop() + return + await self.session.send( + "tunnel_data", conn_id=self.conn_id, b64=base64.b64encode(chunk).decode() + ) + + async def _downlink(self, writer) -> None: + while not self.error: + try: + frame = await asyncio.wait_for(self.frames.get(), POLL_SECONDS) + except asyncio.TimeoutError: + continue + if frame is CLOSED or frame["type"] == "tunnel_close": + return + if frame["type"] != "tunnel_data": + self.error = "bad_frame" + return + try: + data = base64.b64decode(frame["b64"].encode(), validate=True) + except (binascii.Error, ValueError): + self.error = "bad_base64" + return + self.down += len(data) + if self.up + self.down > RELAY_CAP_BYTES: + self.error = "body_too_large" + return + writer.write(data) + await writer.drain() + + async def _close_donor(self) -> None: + self.session.tunnels.pop(self.conn_id, None) + try: + await self.session.send("tunnel_close", conn_id=self.conn_id) + except Exception: + pass + + +class TunnelBridge: + async def fetch(self, session: NodeSession, target: Target, timeout: float) -> TunnelResult: + relay = Relay(session, target) + server = await asyncio.start_server(relay.handle, LOOPBACK, 0) + port = server.sockets[0].getsockname()[1] + try: + return await asyncio.wait_for( + self._request(relay, target, f"http://{LOOPBACK}:{port}", timeout), timeout + ) + except asyncio.TimeoutError: + raise TunnelError("timeout") + finally: + server.close() + session.tunnels.pop(relay.conn_id, None) + + async def _request(self, relay: Relay, target: Target, proxy: str, timeout: float) -> TunnelResult: + limit = LIMITS.max_body_bytes + try: + async with stealth.stealth_async_client( + proxy=proxy, follow_redirects=False, timeout=httpx.Timeout(timeout, connect=OPEN_TIMEOUT_SECONDS + 2) + ) as client: + async with client.stream(target.method, target.url) as response: + chunks = bytearray() + truncated = False + async for chunk in response.aiter_bytes(): + chunks.extend(chunk) + if len(chunks) > limit: + truncated = True + del chunks[limit:] + break + return TunnelResult( + response.status_code, + dict(response.headers), + bytes(chunks), + truncated, + relay.up, + relay.down, + ) + except httpx.HTTPError as exc: + raise TunnelError(relay.error or f"tunnel_failed:{type(exc).__name__}") + + def on_frame(self, session: NodeSession, frame: dict) -> bool: + relay = session.tunnels.get(frame["conn_id"]) + if relay is None: + return False + relay.on_frame(frame) + return True + + def release_session(self, session: NodeSession) -> None: + for relay in list(session.tunnels.values()): + relay.drop() + session.tunnels.clear() diff --git a/devplacepy/services/sync/CLAUDE.md b/devplacepy/services/sync/CLAUDE.md new file mode 100644 index 00000000..905f5f00 --- /dev/null +++ b/devplacepy/services/sync/CLAUDE.md @@ -0,0 +1,87 @@ +This file documents community sync (`devplacepy/services/sync/`, `devplacepy/routers/sync/`, `devplacepy/database/sync.py`): copying users, posts and comments in from another DevPlace-like community. Claude Code auto-loads it whenever a file under `devplacepy/services/sync/` is read or edited. + +## What exists today + +| Piece | File | State | +|-------|------|-------| +| Specification (limits, kinds, statuses, reasons) | `spec.py` | built, `GET /sync/spec.json` adds the record JSON schemas | +| Normalizers | `normalize.py` | built, tested, pure functions | +| Pydantic record models and envelope | `records.py` | built, tested | +| Link table helpers | `database/sync.py`, `SYNC_COLUMNS` in `database/schema.py` | built, tested | +| Per-kind upsert logic | `processors.py` (`RecordProcessor`, `UserProcessor`, `PostProcessor`, `CommentProcessor`) | built, tested | +| Batch runner | `batch.py` (`BatchRunner`) | built | +| Routes | `routers/sync/` | built, tested | +| Devii tools `sync_users`, `sync_posts`, `sync_comments` | `services/devii/actions/catalog/sync.py` | built, `requires_admin`; Devii sends forms, so `records` is a JSON array string (`SyncBatchForm.decode_records`) | +| Admin settings fields `sync_enabled`, `sync_origins` | `AdminSettingsForm`, `admin_settings.html`, `admin_save_settings` | built, origins normalized on save | +| Scheduled sync service, export endpoint, vote and reaction sync | none | not built, see Gaps | + +## Model + +Every synced row has a `sync_links` row `(origin, kind, origin_uid) -> local_uid`, unique on the triple. `origin` is the sender's stable id (a hostname, lower-cased, 2-63 characters of `a-z0-9.-`). `kind` is `users`, `posts` or `comments`. The link also stores a `fingerprint` of the normalized record (sha256 prefix, excludes `updated_at`) and `origin_updated_at`. + +Per record, `RecordProcessor.apply` does: validate (Pydantic), resolve references (author, post, parent) through the links, content filter, then: + +| Situation | Result | +|-----------|--------| +| no link, not deleted | `created`, link claimed first, row inserted second | +| no link, `deleted: true` | `unchanged`, reason `tombstone_for_unknown`, nothing stored | +| link, local row missing | `rejected`, reason `purged_locally`, never recreated | +| incoming `updated_at` older than the link's | `stale` | +| local row soft-deleted by anyone but `sync` | `unchanged`, reason `locally_deleted` (moderation wins) | +| same fingerprint | `unchanged` | +| otherwise | `updated`, link advanced | + +A record that fails validation is `rejected` with reason `invalid` and up to `max_errors_reported` field messages. An unexpected exception is `failed` with reason `internal_error`, logged with the traceback. One bad record never fails the batch; the HTTP status is `200` unless the envelope itself is invalid (`422`), the caller is not an administrator (`401`/`403`), or the kill switch or allowlist refuses (`503`/`403`). + +## Rules that are deliberate + +- **Counters are not imported.** Votes, stars, XP and levels cannot be verified across communities. They are not fields on the models, so Pydantic drops them and the result lists them under `ignored`. Synced posts have `stars = 0`; users start at XP 0, level 1. Email, password, role and API key are never accepted either. +- **Synced accounts cannot sign in.** The password hash is a valid pbkdf2 hash of a random secret nobody knows (`unusable_password_hash`, computed once per process), the email is the placeholder `@sync.invalid`, role is always `Member`. `account_row` in `utils/accounts.py` is the shared row builder, so a new `users` column is added in one place. Synced accounts never go through `_create_account`, so an empty database cannot make a synced user the first administrator. +- **No side effects.** No notifications, no XP, no badges, no mention fan-out, no AI correction or modifier, no SEO metadata job. The content filter runs on every non-deleted record (`screen_fields`, `ContentRefused` becomes `rejected` / `content_refused`) and `record_screening` files reports the same way a native post does. +- **The origin is authoritative for a synced row, except where this community already acted.** Local soft deletes by moderators or owners are never undone and a locally purged row is never recreated. +- **Tombstones.** `deleted: true` soft-deletes posts and comments with `deleted_by = "sync"` (so only sync can revive them) and sets `users.is_active = False` for users. There is no cascade: the origin sends a tombstone per record. +- **Usernames** are normalized to `[A-Za-z0-9_-]`, 3-32 characters. A name taken by a different account gets `-` and then a number (`username_candidates`), so the stored name may differ from the sent one. An update renames only to a free candidate. +- **Timestamps** are parsed from ISO 8601 or epoch seconds, converted to UTC, clamped to `[2000-01-01, now + 5 minutes]`, and `updated_at` is never earlier than `created_at`. +- **Caps are static** (`SyncLimits`): 50 records per batch, field lengths mirror the native forms. `GET /sync/spec.json` is generated from the same dataclass. +- Off by default: `sync_enabled` (`0`) is the kill switch, `sync_origins` an optional comma separated allowlist. Both are site settings and apply live. + +## Concurrency (measured) + +The link is claimed with one `INSERT OR IGNORE` against the unique index and `rowcount` decides the winner; the row is inserted after. 16 separate OS processes racing one new record against a seeded database gave exactly 1 `created` and 15 `unchanged` in 5 of 5 runs, one link, one user. A loser can see the link before the winner's row exists, so `find_local` waits up to `CLAIM_WAIT_ATTEMPTS * CLAIM_WAIT_SECONDS` (2 s) on a link younger than that window, calling `refresh_snapshot()` before each read because a cached read transaction otherwise never sees the winner's commit. Without the wait, 3 of 4 runs returned a false `purged_locally` to one loser. A genuinely purged row is only waited for when its link is younger than 2 s. + +A stateful fuzz (3000 random upserts, tombstones and edits over two origins and all three kinds, invariants checked every 100 steps: no dangling links, no duplicate usernames, no orphan comments, parents on the same post, `updated_at >= created_at`) produced no `failed` result and no violation. + +## Gaps + +- **No export endpoint and no scheduled sync service.** The sender today is a script reading the source community's own API. The service with one job per community is the next step; it should keep a per-origin watermark on `updated_at` and call these endpoints in the order users, posts, comments. +- **Votes, reactions, follows, attachments, polls and project links are not synced.** Counters will not match the origin. +- **A comment's post and parent must be sent in an earlier batch or earlier in the same batch.** There is no deferred resolution queue; the result reason tells the sender what to send first. +- **Privacy disclosure.** Synced profile data (username, bio, location, links) comes from a third party. `/docs/privacy.html` has not been updated for it. +- A user's own deletion on this side is final, but nothing tells the origin; the next sync of that user is `rejected` / `purged_locally` forever. + +## Selection, identity aliasing and translation (community runs only) + +The design record is `community_sync_additional.md`. Everything is optional and off by default, stored per community in `communities.sync_options` (JSON, `options.py` `SyncOptions`), edited only on the administrator page `/admin/communities/{slug}`. Ordinary users never see it: no public route, schema field, docs page or Devii tool for non-administrators exposes the options, the identity table or any original name. The master kill switch is the `community_alias_enabled` config field of the Community sync service (default off); while it is off an aliased community does not run and no identity is created. + +Order inside `CommunityRunner.prepare`, before `SyncPusher`: `Selector` (`selection.py`) then `Aliaser` (`aliaser.py`). + +| Piece | File | Rule | +|-------|------|------| +| Selection | `selection.py` | Deterministic: `sample_percent` buckets `sha256(origin:origin_uid) % 100`, so the same post always gets the same answer. `min_comments` counts comments collected in this run. `topics` allow-list. A post that already has a link is never dropped, so edits keep flowing. Comments of a dropped post are dropped. | +| Jitter | `selection.jittered`, `CommunityStore.schedule_next` | `jitter_percent` (max 50) randomises the next run time; the 900 s floor still applies. | +| Identity map | `identity.py`, table `sync_identities` | Unique on `(origin, source_key)` and on `our_key` (global). Created once per source name, never changed. The AI sees only a style fingerprint (casing, length, digit, separator) and a forbidden list, never the original name. A candidate is rejected when it shares 4 characters with the source, contains a banned fragment, mismatches the style, or collides with an identity or a local user. After 3 AI attempts a deterministic syllable generator is used. Claimed with `INSERT OR IGNORE`, the loser re-reads the winner. Hard cap 100 new identities per run. | +| Mention rewriter | `rewriter.py` | One regex pass, longest name first, case insensitive, `@name` always, a bare name only when it is at least 4 characters (a 3 character handle such as `tef` is otherwise indistinguishable from a word). Fenced code, inline code and URLs are untouched. | +| Aliaser | `aliaser.py` | The local clone is a user record keyed by the lower-cased source name (`origin_uid = source_key`), with the alias as username and no bio, location, links or avatar seed. Titles, bodies and translations all pass the rewriter. A mentioned name without an alias skips the item, as does an author without one. | +| Translation | `translator.py` | Runs after the rewrite, then the rewriter runs again on the output. Up to 40 per run. Up to 3 samples of 200 characters per identity are stored (`style_samples`) and sent along to keep the voice. Skipped when the community language equals the target. | + +Fail closed: any exception, missing identity, missing author or failed translation drops that item and counts it under `report.pushed["skipped"]`; the original name is never a fallback. Transient skips (`identity_failed`, `identity_cap`, `translation_failed`, `rewrite_failed`) hold the watermark, so the next run retries them and the run is `partial`. Dry runs project identities in memory and write none. + +Enabling aliasing on an origin that already has links is refused (form and runner), because those rows hold original names. The ledger is `sync_links` (fingerprint is the content hash, so edits re-run the rewrite and update only that row); there is no second ledger. + +Deliberately not built: + +- **Outbound posting to the source community.** Only the reverse mapping exists (the identity map is bidirectional, and the rewriter works in both directions). Posting under invented identities into another community is not implemented. +- **Visibility delay.** Items become visible immediately; a delayed publish needs a watermark that tolerates deferred items. +- **Avatars** (explicitly out of scope). + +Gaps: `min_comments` only sees comments collected in the same run; a clone renamed by an administrator no longer matches its alias; a username taken by a registration between alias generation and clone creation gets a suffix and diverges from the rewritten mentions. diff --git a/devplacepy/services/sync/__init__.py b/devplacepy/services/sync/__init__.py new file mode 100644 index 00000000..95ee7c3e --- /dev/null +++ b/devplacepy/services/sync/__init__.py @@ -0,0 +1 @@ +# retoor diff --git a/devplacepy/services/sync/ai.py b/devplacepy/services/sync/ai.py new file mode 100644 index 00000000..69a0a534 --- /dev/null +++ b/devplacepy/services/sync/ai.py @@ -0,0 +1,26 @@ +# retoor + +from __future__ import annotations + +import asyncio +import logging + +from devplacepy.config import INTERNAL_MODEL +from devplacepy.database import get_setting +from devplacepy.database.settings import internal_gateway_key + +logger = logging.getLogger("sync.ai") + + + +async def complete(system: str, text: str, timeout: float) -> str | None: + from devplacepy.services.correction import gateway_complete + + model = get_setting("community_ai_model", "") or INTERNAL_MODEL + reply, usage = await asyncio.to_thread( + gateway_complete, internal_gateway_key(), system, text, timeout, None, model, True, False + ) + if usage is None: + logger.warning("the %s model is unreachable", model) + return None + return reply diff --git a/devplacepy/services/sync/aliaser.py b/devplacepy/services/sync/aliaser.py new file mode 100644 index 00000000..05ff68d4 --- /dev/null +++ b/devplacepy/services/sync/aliaser.py @@ -0,0 +1,213 @@ +# retoor + +from __future__ import annotations + +import logging +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timezone + +from devplacepy.database.sync import count_links +from devplacepy.services.sync.identity import IdentityResolver, source_key +from devplacepy.services.sync.options import SyncOptions +from devplacepy.services.sync.rewriter import MentionRewriter +from devplacepy.services.sync.spec import ( + ALIAS, + SKIP_IDENTITY, + SKIP_IDENTITY_CAP, + SKIP_NO_AUTHOR, + SKIP_PARENT, + SKIP_REWRITE, + SKIP_TRANSLATION, + TRANSIENT_SKIPS, +) +from devplacepy.services.sync.translator import Translator + +logger = logging.getLogger("sync.alias") + + +class AliasRefused(Exception): + pass + + +class ItemSkipped(Exception): + def __init__(self, reason: str): + self.reason = reason + super().__init__(reason) + + +@dataclass +class AliasOutcome: + records: dict[str, list[dict]] + skipped: Counter = field(default_factory=Counter) + substitutions: int = 0 + translated: int = 0 + identities_created: int = 0 + + def transient(self) -> int: + return sum(self.skipped[reason] for reason in TRANSIENT_SKIPS) + + +class Aliaser: + def __init__(self, origin: str, options: SyncOptions, *, language: str = "", dry_run: bool = False, resolver: IdentityResolver | None = None, translator: Translator | None = None): + self.origin = origin + self.options = options + self.language = language + self.resolver = resolver or IdentityResolver(origin, persist=not dry_run) + self.translator = translator or (Translator(options.translate_to) if options.translate_to else None) + self.rewriter = MentionRewriter({}) + self.outcome = AliasOutcome({}) + self.authors: dict[str, str] = {} + + def check_origin(self) -> None: + if self.options.alias and not self.resolver.rows and count_links(self.origin): + raise AliasRefused( + f"{self.origin} already has synced records stored under their original names; " + "aliasing cannot be switched on afterwards" + ) + + async def apply(self, records: dict[str, list[dict]]) -> AliasOutcome: + if self.options.alias: + self.check_origin() + self.outcome = AliasOutcome({"users": [], "posts": [], "comments": []}) + if self.options.alias: + await self.map_names(records) + self.rewriter = MentionRewriter(self.resolver.mapping()) + users = {str(user.get("origin_uid")): user for user in records.get("users", [])} + kept_posts = set() + for post in records.get("posts", []): + kept = await self.process(post, users, ("title", "content")) + if kept: + self.outcome.records["posts"].append(kept) + kept_posts.add(str(kept.get("origin_uid"))) + placed: set[str] = set() + for comment in records.get("comments", []): + parent = comment.get("parent_uid") + if str(comment.get("post_uid")) not in kept_posts or (parent and str(parent) not in placed): + self.outcome.skipped[SKIP_PARENT] += 1 + continue + kept = await self.process(comment, users, ("content",)) + if kept: + self.outcome.records["comments"].append(kept) + placed.add(str(comment.get("origin_uid"))) + self.outcome.records["users"] = self.users_for(records, users) + self.outcome.identities_created = self.resolver.created + return self.outcome + + def author_name(self, record: dict, users: dict[str, dict]) -> str | None: + user = users.get(str(record.get("author_uid"))) + return str(user.get("username")) if user and user.get("username") else None + + async def map_names(self, records: dict[str, list[dict]]) -> None: + users = {str(user.get("origin_uid")): user for user in records.get("users", [])} + scan = MentionRewriter({}) + names: dict[str, str] = {} + for kind, fields in (("posts", ("title", "content")), ("comments", ("content",))): + for record in records.get(kind, []): + author = self.author_name(record, users) + if author: + names.setdefault(author.lower(), author) + for name in fields: + for mention in scan.mentioned(str(record.get(name) or "")): + names.setdefault(mention.lower(), mention) + for name in names.values(): + await self.resolver.ensure(name) + + async def process(self, record: dict, users: dict[str, dict], fields: tuple[str, ...]) -> dict | None: + try: + return await self.transform(record, users, fields) + except ItemSkipped as skipped: + self.outcome.skipped[skipped.reason] += 1 + logger.info("skipped %s %s: %s", self.origin, record.get("origin_uid"), skipped.reason) + except Exception: + self.outcome.skipped[SKIP_REWRITE] += 1 + logger.exception("fail closed: %s %s skipped", self.origin, record.get("origin_uid")) + return None + + async def transform(self, record: dict, users: dict[str, dict], fields: tuple[str, ...]) -> dict: + author = self.author_name(record, users) + if not author: + raise ItemSkipped(SKIP_NO_AUTHOR) + result = dict(record) + if self.options.alias: + alias = self.resolver.known(author) + if not alias: + raise ItemSkipped(SKIP_IDENTITY_CAP if self.resolver.created >= self.resolver.cap else SKIP_IDENTITY) + key = source_key(author) + result["author_uid"] = key + for name in fields: + for mention in self.rewriter.mentioned(str(result.get(name) or "")): + if self.resolver.known(mention) is None: + raise ItemSkipped(SKIP_IDENTITY_CAP if self.resolver.created >= self.resolver.cap else SKIP_IDENTITY) + result[name] = self.rewritten(str(result.get(name) or "")) + if self.translator and self.translator.needed(self.language): + await self.translate(result, fields, author) + if self.options.alias: + self.authors[source_key(author)] = author + return result + + def rewritten(self, text: str) -> str: + rewrite = self.rewriter.rewrite(text) + self.outcome.substitutions += len(rewrite.substitutions) + return rewrite.text + + async def translate(self, result: dict, fields: tuple[str, ...], author: str) -> None: + if self.outcome.translated >= ALIAS.max_translations_per_run: + raise ItemSkipped(SKIP_TRANSLATION) + samples = self.resolver.samples(author) if self.options.alias else [] + for name in fields: + text = str(result.get(name) or "") + if not text: + continue + translated = await self.translator.translate(text, samples) + if translated is None: + raise ItemSkipped(SKIP_TRANSLATION) + result[name] = self.rewritten(translated) if self.options.alias else translated + self.outcome.translated += 1 + if self.options.alias: + self.resolver.remember(author, str(result.get(fields[-1]) or "")) + + def users_for(self, records: dict[str, list[dict]], users: dict[str, dict]) -> list[dict]: + if not self.options.alias: + needed = {str(item.get("author_uid")) for kind in ("posts", "comments") for item in self.outcome.records[kind]} + return [user for uid, user in users.items() if uid in needed] if self.options.selective() else list(users.values()) + now = datetime.now(timezone.utc).isoformat() + created: dict[str, str] = {} + removed: set[str] = set() + for user in users.values(): + if user.get("username"): + created.setdefault(str(user["username"]).lower(), str(user.get("created_at") or now)) + if user.get("deleted"): + removed.add(str(user["username"]).lower()) + built = [] + for key, author in self.authors.items(): + alias = self.resolver.known(author) + if alias: + built.append(self.clone(key, alias, created.get(author.lower(), now), author.lower() in removed)) + for key, alias in self.resolver.mapping().items(): + if key not in self.authors and self.mentioned_in_output(alias): + built.append(self.clone(key, alias, created.get(key) or self.first_mention(alias, now), key in removed)) + return built + + @staticmethod + def clone(key: str, alias: str, created_at: str, deleted: bool) -> dict: + return {"origin_uid": key, "username": alias, "created_at": created_at, "deleted": deleted} + + def first_mention(self, alias: str, fallback: str) -> str: + needle = f"@{alias}".lower() + stamps = [ + str(item.get("created_at")) + for kind in ("posts", "comments") + for item in self.outcome.records[kind] + if needle in f"{item.get('title') or ''} {item.get('content') or ''}".lower() + ] + return min(stamps) if stamps else fallback + + def mentioned_in_output(self, alias: str) -> bool: + needle = f"@{alias}".lower() + return any( + needle in str(item.get(name) or "").lower() + for kind, names in (("posts", ("title", "content")), ("comments", ("content",))) + for item in self.outcome.records[kind] + for name in names + ) diff --git a/devplacepy/services/sync/batch.py b/devplacepy/services/sync/batch.py new file mode 100644 index 00000000..f6b33817 --- /dev/null +++ b/devplacepy/services/sync/batch.py @@ -0,0 +1,34 @@ +# retoor + +from __future__ import annotations + +from collections import Counter + +from devplacepy.schemas.sync import SyncBatchOut, SyncRecordResultOut +from devplacepy.services.sync.processors import PROCESSORS +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import STATUSES + + +class BatchRunner: + def __init__(self, kind: str, *, request=None): + self.kind = kind + self.request = request + + def run(self, data: SyncBatchForm) -> SyncBatchOut: + processor = PROCESSORS[self.kind]( + data.origin, dry_run=data.dry_run, request=self.request + ) + results = [ + SyncRecordResultOut(**vars(processor.process(index, raw))) + for index, raw in enumerate(data.records) + ] + tally = Counter(result.status for result in results) + return SyncBatchOut( + origin=data.origin, + kind=self.kind, + dry_run=data.dry_run, + total=len(results), + counts={status: tally.get(status, 0) for status in STATUSES}, + results=results, + ) diff --git a/devplacepy/services/sync/gate.py b/devplacepy/services/sync/gate.py new file mode 100644 index 00000000..49f71ca0 --- /dev/null +++ b/devplacepy/services/sync/gate.py @@ -0,0 +1,23 @@ +# retoor + +from __future__ import annotations + +from devplacepy.database import get_setting + + +def allowed_origins() -> set[str]: + raw = get_setting("sync_origins", "") or "" + return {item.strip().lower() for item in raw.split(",") if item.strip()} + + +def sync_enabled() -> bool: + return (get_setting("sync_enabled", "0") or "0") == "1" + + +def refusal(origin: str) -> str | None: + if not sync_enabled(): + return "Community sync is disabled" + allowed = allowed_origins() + if allowed and origin not in allowed: + return f"origin {origin} is not allowed" + return None diff --git a/devplacepy/services/sync/identity.py b/devplacepy/services/sync/identity.py new file mode 100644 index 00000000..983092ae --- /dev/null +++ b/devplacepy/services/sync/identity.py @@ -0,0 +1,194 @@ +# retoor + +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import random +import re + +from devplacepy.database.sync import ( + claim_identity, + identity_by_source, + identity_map, + identity_name_taken, + save_style_samples, + username_taken, +) +from devplacepy.services.sync import ai +from devplacepy.services.sync.normalize import ORIGIN_UID_PATTERN +from devplacepy.services.sync.spec import ALIAS, LIMITS +from devplacepy.services.sync.style import Style, plausible + +logger = logging.getLogger("sync.identity") + +SYLLABLES = ( + "ka", "ro", "vi", "te", "mu", "sa", "lo", "ne", "di", "po", "ra", "ke", "zu", "mi", "to", "la", + "ve", "no", "ba", "ri", "so", "ta", "ni", "ku", "me", "fa", "li", "do", "re", "gu", +) +NAME_SYSTEM = ( + "You invent a replacement username for a member of a developer forum. Reply with JSON only: " + '{"username": ""}. Match the style notes exactly: the casing pattern, a length within 2 ' + "characters, whether it contains a digit, and the separator character. Use only letters, digits, " + "underscore and hyphen. The name must look like a real person's handle, must not contain words such " + "as user or bot, and must not be any name from the forbidden list." +) +FENCE = re.compile(r"^```(?:json)?\s*|\s*```$", re.I) + + +def source_key(name: str) -> str | None: + key = str(name or "").strip().lower() + if not key or len(key) > LIMITS.max_origin_uid_length or not ORIGIN_UID_PATTERN.match(key): + return None + return key + + +def fallback_name(origin: str, source: str, style: Style, attempt: int) -> str: + rng = random.Random(hashlib.sha256(f"{origin}:{source.lower()}:{attempt}".encode()).digest()) + target = max(ALIAS.min_name_length, style.length - (2 if style.has_digit else 0)) + separator = style.separator + parts: list[str] = [] + while len("".join(parts)) + len(separator) * max(len(parts) - 1, 0) < target: + parts.append("".join(rng.choice(SYLLABLES) for _ in range(rng.choice((1, 2))))) + if separator and len(parts) == 2: + break + name = separator.join(parts) + if style.has_digit: + name += str(rng.randrange(10, 100)) + name = name[: ALIAS.max_name_length] + if style.casing == "upper": + return name.upper() + if style.casing == "title": + return name.capitalize() + if style.casing == "mixed" and len(name) > 2: + return name[:2] + name[2:].capitalize() + return name + + +class NameGenerator: + async def generate(self, origin: str, source: str, style: Style, forbidden: list[str]) -> tuple[str, bool] | None: + for _ in range(ALIAS.name_attempts): + reply = await ai.complete( + NAME_SYSTEM, + json.dumps({"style": style.describe(), "forbidden": forbidden[-50:]}), + ALIAS.name_timeout_seconds, + ) + candidate = self.parse(reply) + if candidate and plausible(candidate, source, style) and candidate not in forbidden: + return candidate, True + if reply is None: + break + for attempt in range(ALIAS.fallback_attempts): + candidate = fallback_name(origin, source, style, attempt) + if plausible(candidate, source, style) and candidate not in forbidden: + return candidate, False + return None + + @staticmethod + def parse(reply: str | None) -> str | None: + if not reply: + return None + cleaned = FENCE.sub("", reply.strip()) + start, end = cleaned.find("{"), cleaned.rfind("}") + try: + value = json.loads(cleaned[start : end + 1])["username"] + except (ValueError, KeyError, TypeError): + return None + return value.strip() if isinstance(value, str) else None + + + +class IdentityResolver: + def __init__(self, origin: str, *, persist: bool = True, generator: NameGenerator | None = None, cap: int = ALIAS.max_new_identities_per_run): + self.origin = origin + self.persist = persist + self.generator = generator or NameGenerator() + self.cap = cap + self.created = 0 + self.rows: dict[str, dict] = identity_map(origin) + self.projected: dict[str, str] = {} + self.lock = asyncio.Lock() + + def mapping(self) -> dict[str, str]: + return {key: row["our_name"] for key, row in self.rows.items()} | self.projected + + def known(self, name: str) -> str | None: + key = source_key(name) + if key is None: + return None + if key in self.rows: + return self.rows[key]["our_name"] + return self.projected.get(key) + + def taken(self, candidate: str) -> bool: + lowered = candidate.lower() + return ( + identity_name_taken(lowered) + or username_taken(candidate) + or lowered in {name.lower() for name in self.projected.values()} + ) + + async def ensure(self, name: str) -> str | None: + key = source_key(name) + if key is None: + return None + existing = self.known(name) + if existing: + return existing + async with self.lock: + existing = self.known(name) + if existing: + return existing + if self.created >= self.cap: + logger.warning("identity cap of %d new identities reached", self.cap) + return None + return await self.create(name, key) + + async def create(self, name: str, key: str) -> str | None: + style = Style.of(name) + forbidden: list[str] = [] + for _ in range(ALIAS.name_attempts + 1): + generated = await self.generator.generate(self.origin, name, style, forbidden) + if generated is None: + return None + candidate, by_ai = generated + if self.taken(candidate): + forbidden.append(candidate) + continue + self.created += 1 + if not self.persist: + self.projected[key] = candidate + return candidate + if claim_identity(self.origin, name, key, candidate, style.dump(), by_ai): + self.rows[key] = identity_by_source(self.origin, key) + logger.info("identity created for %s (ai=%s)", self.origin, by_ai) + return candidate + winner = identity_by_source(self.origin, key) + if winner: + self.rows[key] = winner + return winner["our_name"] + forbidden.append(candidate) + return None + + def samples(self, name: str) -> list[str]: + key = source_key(name) + row = self.rows.get(key) if key else None + try: + return list(json.loads(row["style_samples"])) if row and row.get("style_samples") else [] + except ValueError: + return [] + + def remember(self, name: str, text: str) -> None: + key = source_key(name) + row = self.rows.get(key) if key else None + sample = " ".join((text or "").split())[: ALIAS.sample_chars] + if not self.persist or row is None or not sample: + return + kept = self.samples(name) + if len(kept) >= ALIAS.samples_kept or sample in kept: + return + kept.append(sample) + row["style_samples"] = json.dumps(kept, ensure_ascii=False) + save_style_samples(row["uid"], row["style_samples"]) diff --git a/devplacepy/services/sync/normalize.py b/devplacepy/services/sync/normalize.py new file mode 100644 index 00000000..f1d48946 --- /dev/null +++ b/devplacepy/services/sync/normalize.py @@ -0,0 +1,112 @@ +# retoor + +from __future__ import annotations + +import hashlib +import json +import re +import unicodedata +from datetime import datetime, timedelta, timezone + +from devplacepy.services.sync.spec import LIMITS + +ORIGIN_PATTERN = re.compile(r"^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$") +ORIGIN_UID_PATTERN = re.compile(r"^[A-Za-z0-9._:@-]+$") +USERNAME_STRIP = re.compile(r"[^A-Za-z0-9_-]+") +CONTROL_CHARS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") +FINGERPRINT_LENGTH = 40 +SUFFIX_LABEL_LENGTH = 8 +MAX_NUMERIC_SUFFIX = 9999 +EARLIEST = datetime(LIMITS.earliest_year, 1, 1, tzinfo=timezone.utc) + + +def normalize_origin(value) -> str: + text = str(value or "").strip().lower() + if not (LIMITS.min_origin_length <= len(text) <= LIMITS.max_origin_length): + raise ValueError( + f"origin must be {LIMITS.min_origin_length}-{LIMITS.max_origin_length} characters" + ) + if not ORIGIN_PATTERN.match(text) or ".." in text: + raise ValueError("origin may only contain a-z, 0-9, dots and hyphens") + return text + + +def normalize_origin_uid(value) -> str: + text = str(value if value is not None else "").strip() + if not text or len(text) > LIMITS.max_origin_uid_length: + raise ValueError(f"id must be 1-{LIMITS.max_origin_uid_length} characters") + if not ORIGIN_UID_PATTERN.match(text): + raise ValueError("id may only contain letters, digits and . _ : @ -") + return text + + +def clean_text(value, *, multiline: bool) -> str: + text = unicodedata.normalize("NFC", str(value if value is not None else "")) + text = text.replace("\r\n", "\n").replace("\r", "\n") + text = CONTROL_CHARS.sub("", text) + if not multiline: + text = " ".join(text.split()) + return text.strip() + + +def parse_instant(value) -> datetime: + if isinstance(value, datetime): + parsed = value + elif isinstance(value, bool): + raise ValueError("timestamp must be an ISO 8601 string or epoch seconds") + elif isinstance(value, (int, float)): + parsed = datetime.fromtimestamp(float(value), tz=timezone.utc) + else: + text = str(value or "").strip() + if not text: + raise ValueError("timestamp is required") + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + raise ValueError("timestamp must be an ISO 8601 string or epoch seconds") + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def clamp_instant(value: datetime, now: datetime | None = None) -> datetime: + now = now or datetime.now(timezone.utc) + ceiling = now + timedelta(seconds=LIMITS.future_skew_seconds) + if value > ceiling: + return now + if value < EARLIEST: + return EARLIEST + return value + + +def instant_iso(value) -> str: + return clamp_instant(parse_instant(value)).isoformat() + + +def normalize_username(raw, origin_uid: str) -> str: + text = unicodedata.normalize("NFKD", str(raw or "")) + text = text.encode("ascii", "ignore").decode("ascii") + text = USERNAME_STRIP.sub("_", text.strip()) + text = re.sub(r"_{2,}", "_", text).strip("_-") + if not text: + text = "user" + if len(text) < LIMITS.min_username_length: + tail = hashlib.sha256(origin_uid.encode()).hexdigest()[:4] + text = f"{text}_{tail}" + return text[: LIMITS.max_username_length] + + +def username_candidates(base: str, origin: str): + yield base + label = re.sub(r"[^a-z0-9]", "", origin.split(".")[0])[:SUFFIX_LABEL_LENGTH] or "sync" + tagged = f"-{label}" + stem = base[: LIMITS.max_username_length - len(tagged)] + yield f"{stem}{tagged}" + for number in range(2, MAX_NUMERIC_SUFFIX + 1): + counter = f"{tagged}{number}" + yield f"{base[: LIMITS.max_username_length - len(counter)]}{counter}" + + +def fingerprint(fields: dict) -> str: + canonical = json.dumps(fields, sort_keys=True, ensure_ascii=False, default=str) + return hashlib.sha256(canonical.encode()).hexdigest()[:FINGERPRINT_LENGTH] diff --git a/devplacepy/services/sync/options.py b/devplacepy/services/sync/options.py new file mode 100644 index 00000000..1a4a07ee --- /dev/null +++ b/devplacepy/services/sync/options.py @@ -0,0 +1,71 @@ +# retoor + +from __future__ import annotations + +import re +from dataclasses import asdict, dataclass + +from devplacepy.constants import TOPICS +from devplacepy.database import get_setting +from devplacepy.services.sync.spec import ALIAS + +ALIAS_KEY = "community_alias_enabled" +LANGUAGE_PATTERN = re.compile(r"^[a-z]{2,3}(-[a-z0-9]{2,8})?$") + + +def alias_allowed() -> bool: + return (get_setting(ALIAS_KEY, "0") or "0") == "1" + + +@dataclass(frozen=True) +class SyncOptions: + alias: bool = False + translate_to: str = "" + sample_percent: int = ALIAS.sample_percent_default + min_comments: int = 0 + topics: tuple[str, ...] = () + jitter_percent: int = 0 + + @classmethod + def from_row(cls, row: dict | None) -> "SyncOptions": + raw = (row or {}).get("sync_options") or {} + if not isinstance(raw, dict): + raw = {} + return cls.build(**{key: raw[key] for key in cls.__dataclass_fields__ if key in raw}) + + @classmethod + def build(cls, **values) -> "SyncOptions": + topics = values.get("topics") or () + if isinstance(topics, str): + topics = [item for item in re.split(r"[,\s]+", topics.lower()) if item] + unknown = [item for item in topics if item not in TOPICS] + if unknown: + raise ValueError(f"unknown topic: {', '.join(unknown)}") + translate_to = str(values.get("translate_to") or "").strip().lower() + if translate_to and not LANGUAGE_PATTERN.match(translate_to): + raise ValueError("translate_to must be a language code such as en or nl") + sample = int(values.get("sample_percent", ALIAS.sample_percent_default)) + if not 1 <= sample <= 100: + raise ValueError("sample_percent must be between 1 and 100") + minimum = int(values.get("min_comments") or 0) + if not 0 <= minimum <= ALIAS.max_min_comments: + raise ValueError(f"min_comments must be between 0 and {ALIAS.max_min_comments}") + jitter = int(values.get("jitter_percent") or 0) + if not 0 <= jitter <= ALIAS.max_jitter_percent: + raise ValueError(f"jitter_percent must be between 0 and {ALIAS.max_jitter_percent}") + return cls(bool(values.get("alias")), translate_to, sample, minimum, tuple(dict.fromkeys(topics)), jitter) + + def merged(self, **changes) -> "SyncOptions": + values = self.to_dict() | {key: value for key, value in changes.items() if value is not None} + return self.build(**values) + + def to_dict(self) -> dict: + data = asdict(self) + data["topics"] = list(self.topics) + return data + + def selective(self) -> bool: + return self.sample_percent < 100 or self.min_comments > 0 or bool(self.topics) + + def active(self) -> bool: + return self.alias or bool(self.translate_to) or self.selective() diff --git a/devplacepy/services/sync/processors.py b/devplacepy/services/sync/processors.py new file mode 100644 index 00000000..ad649d63 --- /dev/null +++ b/devplacepy/services/sync/processors.py @@ -0,0 +1,516 @@ +# retoor + +from __future__ import annotations + +import logging +import secrets +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone + +from pydantic import ValidationError + +from devplacepy.database import get_table, refresh_snapshot +from devplacepy.database.content import invalidate_user_recent_cache +from devplacepy.database.pagination import clear_user_post_count +from devplacepy.database.soft_delete import restore, soft_delete +from devplacepy.database.sync import ( + advance_link, + claim_link, + get_link, + release_link, + username_taken, +) +from devplacepy.services.moderation.screening import ( + ContentRefused, + record as record_screening, + refuse_if_blocked, + screen_fields, +) +from devplacepy.services.sync.normalize import fingerprint, username_candidates +from devplacepy.services.sync.records import ( + CommentRecord, + PostRecord, + SyncRecord, + UserRecord, +) +from devplacepy.services.sync.spec import ( + KIND_COMMENTS, + KIND_POSTS, + KIND_USERS, + LIMITS, + REASON_CONTENT_REFUSED, + REASON_INTERNAL, + REASON_INVALID, + REASON_LOCALLY_DELETED, + REASON_OLDER_THAN_SYNCED, + REASON_PARENT_MISMATCH, + REASON_PURGED_LOCALLY, + REASON_TOMBSTONE_UNKNOWN, + REASON_UNKNOWN_AUTHOR, + REASON_UNKNOWN_PARENT, + REASON_UNKNOWN_POST, + STATUS_CREATED, + STATUS_FAILED, + STATUS_REJECTED, + STATUS_STALE, + STATUS_UNCHANGED, + STATUS_UPDATED, + SYNC_ACTOR, +) +from devplacepy.utils.accounts import account_row +from devplacepy.utils.authcache import clear_user_cache +from devplacepy.utils.passwords import hash_password +from devplacepy.utils.text import generate_uid, make_combined_slug + +logger = logging.getLogger(__name__) + +PLACEHOLDER_EMAIL_DOMAIN = "sync.invalid" +SLUG_FALLBACK_LENGTH = 50 +UNUSABLE_SECRET_BYTES = 32 +CLAIM_WAIT_ATTEMPTS = 40 +CLAIM_WAIT_SECONDS = 0.05 + +_unusable_hash: list[str] = [] + + +def unusable_password_hash() -> str: + if not _unusable_hash: + _unusable_hash.append(hash_password(secrets.token_hex(UNUSABLE_SECRET_BYTES))) + return _unusable_hash[0] + + +class Refusal(Exception): + def __init__(self, reason: str, message: str): + self.reason = reason + self.message = message + super().__init__(message) + + +@dataclass +class SyncResult: + index: int + origin_uid: str + status: str + reason: str = "" + local_uid: str = "" + errors: list[str] = field(default_factory=list) + ignored: list[str] = field(default_factory=list) + + +class RecordProcessor: + kind: str = "" + table: str = "" + model: type[SyncRecord] = SyncRecord + + def __init__(self, origin: str, *, dry_run: bool = False, request=None): + self.origin = origin + self.dry_run = dry_run + self.request = request + + def process(self, index: int, raw: dict) -> SyncResult: + origin_uid = str(raw.get("origin_uid", "")) if isinstance(raw, dict) else "" + ignored = self.ignored_fields(raw) + try: + record = self.model.model_validate(raw) + except ValidationError as exc: + return SyncResult( + index, + origin_uid, + STATUS_REJECTED, + REASON_INVALID, + errors=self.describe_errors(exc), + ignored=ignored, + ) + try: + result = self.apply(index, record) + except Refusal as refusal: + result = SyncResult( + index, + record.origin_uid, + STATUS_REJECTED, + refusal.reason, + errors=[refusal.message], + ) + except ContentRefused as refused: + result = SyncResult( + index, + record.origin_uid, + STATUS_REJECTED, + REASON_CONTENT_REFUSED, + errors=[refused.message], + ) + except Exception: + logger.exception( + "sync %s %s/%s failed", self.kind, self.origin, record.origin_uid + ) + result = SyncResult( + index, record.origin_uid, STATUS_FAILED, REASON_INTERNAL + ) + result.ignored = ignored + return result + + def ignored_fields(self, raw) -> list[str]: + if not isinstance(raw, dict): + return [] + return sorted(set(map(str, raw)) - self.model.known_fields()) + + @staticmethod + def describe_errors(exc: ValidationError) -> list[str]: + messages = [] + for error in exc.errors()[: LIMITS.max_errors_reported]: + location = ".".join(str(part) for part in error["loc"]) or "record" + messages.append(f"{location}: {error['msg'].removeprefix('Value error, ')}") + return messages + + def apply(self, index: int, record: SyncRecord) -> SyncResult: + refs = self.resolve(record) + if not record.deleted: + self.screen(record) + digest = fingerprint(record.content_fields()) + link = get_link(self.origin, self.kind, record.origin_uid) + if link is None: + if record.deleted: + return SyncResult( + index, record.origin_uid, STATUS_UNCHANGED, REASON_TOMBSTONE_UNKNOWN + ) + return self.create(index, record, refs, digest) + return self.update(index, record, refs, digest, link) + + def create( + self, index: int, record: SyncRecord, refs: dict, digest: str + ) -> SyncResult: + local_uid = generate_uid() + if self.dry_run: + return SyncResult(index, record.origin_uid, STATUS_CREATED, local_uid="") + self.prepare() + claimed = claim_link( + self.origin, + self.kind, + record.origin_uid, + local_uid, + digest, + record.updated_at, + ) + if not claimed: + link = get_link(self.origin, self.kind, record.origin_uid) + return self.update(index, record, refs, digest, link) + try: + self.insert(local_uid, record, refs) + except Exception: + release_link(self.origin, self.kind, record.origin_uid, local_uid) + raise + self.after_write(local_uid, record, refs) + return SyncResult(index, record.origin_uid, STATUS_CREATED, local_uid=local_uid) + + def find_local(self, link: dict) -> dict | None: + table = get_table(self.table) + refresh_snapshot() + row = table.find_one(uid=link["local_uid"]) + if row is not None or not self.is_fresh(link): + return row + for _ in range(CLAIM_WAIT_ATTEMPTS): + time.sleep(CLAIM_WAIT_SECONDS) + refresh_snapshot() + row = table.find_one(uid=link["local_uid"]) + if row is not None: + return row + return None + + @staticmethod + def is_fresh(link: dict) -> bool: + try: + claimed = datetime.fromisoformat(link["first_synced_at"]) + except (KeyError, TypeError, ValueError): + return False + window = CLAIM_WAIT_ATTEMPTS * CLAIM_WAIT_SECONDS + return (datetime.now(timezone.utc) - claimed).total_seconds() < window + + def update( + self, index: int, record: SyncRecord, refs: dict, digest: str, link: dict + ) -> SyncResult: + local_uid = link["local_uid"] + row = self.find_local(link) + if row is None: + return SyncResult( + index, + record.origin_uid, + STATUS_REJECTED, + REASON_PURGED_LOCALLY, + local_uid=local_uid, + errors=["the local copy was permanently removed and is not recreated"], + ) + if record.updated_at < (link.get("origin_updated_at") or ""): + return SyncResult( + index, + record.origin_uid, + STATUS_STALE, + REASON_OLDER_THAN_SYNCED, + local_uid=local_uid, + ) + if row.get("deleted_at") and row.get("deleted_by") != SYNC_ACTOR: + return SyncResult( + index, + record.origin_uid, + STATUS_UNCHANGED, + REASON_LOCALLY_DELETED, + local_uid=local_uid, + ) + if digest == link.get("fingerprint"): + return SyncResult( + index, record.origin_uid, STATUS_UNCHANGED, local_uid=local_uid + ) + if self.dry_run: + return SyncResult( + index, record.origin_uid, STATUS_UPDATED, local_uid=local_uid + ) + self.write(row, record, refs) + advance_link(link["uid"], digest, record.updated_at) + self.after_write(local_uid, record, refs) + return SyncResult(index, record.origin_uid, STATUS_UPDATED, local_uid=local_uid) + + def prepare(self) -> None: + return None + + def resolve(self, record: SyncRecord) -> dict: + return {} + + def screen(self, record: SyncRecord) -> None: + return None + + def insert(self, local_uid: str, record: SyncRecord, refs: dict) -> None: + raise NotImplementedError + + def write(self, row: dict, record: SyncRecord, refs: dict) -> None: + raise NotImplementedError + + def after_write(self, local_uid: str, record: SyncRecord, refs: dict) -> None: + return None + + def author_uid(self, record: SyncRecord) -> str: + link = get_link(self.origin, KIND_USERS, record.author_uid) + if link is None: + raise Refusal( + REASON_UNKNOWN_AUTHOR, + f"author {record.author_uid} is not synced yet, send users first", + ) + return link["local_uid"] + + def apply_tombstone(self, row: dict, record: SyncRecord) -> None: + if record.deleted and not row.get("deleted_at"): + soft_delete(self.table, SYNC_ACTOR, uid=row["uid"]) + elif not record.deleted and row.get("deleted_at"): + restore(self.table, uid=row["uid"]) + + def screen_and_report(self, local_uid: str, target_type: str, fields: dict, actor: str): + screening = screen_fields(self.table, fields) + record_screening( + screening, + target_type=target_type, + target_uid=local_uid, + actor_uid=actor, + request=self.request, + ) + + +class UserProcessor(RecordProcessor): + kind = KIND_USERS + table = "users" + model = UserRecord + + def prepare(self) -> None: + unusable_password_hash() + + def screen(self, record: UserRecord) -> None: + refuse_if_blocked( + screen_fields( + "users", + { + "username": record.username, + "bio": record.bio, + "location": record.location, + }, + ) + ) + + def free_username(self, record: UserRecord, current: str = "") -> str: + for candidate in username_candidates(record.username, self.origin): + if candidate == current or not username_taken(candidate): + return candidate + return current or record.username + + def insert(self, local_uid: str, record: UserRecord, refs: dict) -> None: + now = datetime.now(timezone.utc).isoformat() + row = account_row( + local_uid, + self.free_username(record), + f"{local_uid}@{PLACEHOLDER_EMAIL_DOMAIN}", + unusable_password_hash(), + "Member", + now, + ) + row.update( + { + "bio": record.bio, + "location": record.location, + "git_link": record.git_link, + "website": record.website, + "avatar_seed": record.avatar_seed, + "is_active": not record.deleted, + "created_at": record.created_at, + } + ) + get_table("users").insert(row) + + def write(self, row: dict, record: UserRecord, refs: dict) -> None: + values = { + "uid": row["uid"], + "username": self.free_username(record, row["username"]), + "bio": record.bio, + "location": record.location, + "git_link": record.git_link, + "website": record.website, + "is_active": not record.deleted, + } + if record.avatar_seed: + values["avatar_seed"] = record.avatar_seed + get_table("users").update(values, ["uid"]) + + def after_write(self, local_uid: str, record: UserRecord, refs: dict) -> None: + clear_user_cache(local_uid) + + +class PostProcessor(RecordProcessor): + kind = KIND_POSTS + table = "posts" + model = PostRecord + + def resolve(self, record: PostRecord) -> dict: + return {"user_uid": self.author_uid(record)} + + def screen(self, record: PostRecord) -> None: + refuse_if_blocked( + screen_fields("posts", {"title": record.title, "content": record.content}) + ) + + def insert(self, local_uid: str, record: PostRecord, refs: dict) -> None: + get_table("posts").insert( + { + "uid": local_uid, + "user_uid": refs["user_uid"], + "slug": make_combined_slug( + record.title or record.content[:SLUG_FALLBACK_LENGTH], local_uid + ), + "title": record.title or None, + "content": record.content, + "topic": record.topic, + "project_uid": None, + "image": None, + "stars": 0, + "created_at": record.created_at, + "updated_at": record.updated_at, + "deleted_at": None, + "deleted_by": None, + } + ) + + def write(self, row: dict, record: PostRecord, refs: dict) -> None: + get_table("posts").update( + { + "uid": row["uid"], + "title": record.title or None, + "content": record.content, + "topic": record.topic, + "updated_at": record.updated_at, + }, + ["uid"], + ) + self.apply_tombstone(row, record) + + def after_write(self, local_uid: str, record: PostRecord, refs: dict) -> None: + clear_user_post_count(refs["user_uid"]) + invalidate_user_recent_cache("posts", refs["user_uid"]) + if not record.deleted: + self.screen_and_report( + local_uid, + "post", + {"title": record.title, "content": record.content}, + refs["user_uid"], + ) + + +class CommentProcessor(RecordProcessor): + kind = KIND_COMMENTS + table = "comments" + model = CommentRecord + + def resolve(self, record: CommentRecord) -> dict: + user_uid = self.author_uid(record) + post_link = get_link(self.origin, KIND_POSTS, record.post_uid) + if post_link is None: + raise Refusal( + REASON_UNKNOWN_POST, + f"post {record.post_uid} is not synced yet, send posts first", + ) + post_uid = post_link["local_uid"] + parent_uid = None + if record.parent_uid: + parent_link = get_link(self.origin, KIND_COMMENTS, record.parent_uid) + if parent_link is None: + raise Refusal( + REASON_UNKNOWN_PARENT, + f"parent comment {record.parent_uid} is not synced yet, " + "send parents before replies", + ) + parent = get_table("comments").find_one(uid=parent_link["local_uid"]) + if parent is None or parent.get("target_uid") != post_uid: + raise Refusal( + REASON_PARENT_MISMATCH, + "parent comment does not belong to the same post", + ) + parent_uid = parent["uid"] + return {"user_uid": user_uid, "post_uid": post_uid, "parent_uid": parent_uid} + + def screen(self, record: CommentRecord) -> None: + refuse_if_blocked(screen_fields("comments", {"content": record.content})) + + def insert(self, local_uid: str, record: CommentRecord, refs: dict) -> None: + get_table("comments").insert( + { + "uid": local_uid, + "target_uid": refs["post_uid"], + "target_type": "post", + "post_uid": refs["post_uid"], + "user_uid": refs["user_uid"], + "content": record.content, + "parent_uid": refs["parent_uid"], + "created_at": record.created_at, + "updated_at": record.updated_at, + "deleted_at": None, + "deleted_by": None, + } + ) + + def write(self, row: dict, record: CommentRecord, refs: dict) -> None: + get_table("comments").update( + { + "uid": row["uid"], + "content": record.content, + "updated_at": record.updated_at, + }, + ["uid"], + ) + self.apply_tombstone(row, record) + + def after_write(self, local_uid: str, record: CommentRecord, refs: dict) -> None: + if not record.deleted: + self.screen_and_report( + local_uid, "comment", {"content": record.content}, refs["user_uid"] + ) + + +PROCESSORS: dict[str, type[RecordProcessor]] = { + KIND_USERS: UserProcessor, + KIND_POSTS: PostProcessor, + KIND_COMMENTS: CommentProcessor, +} diff --git a/devplacepy/services/sync/records.py b/devplacepy/services/sync/records.py new file mode 100644 index 00000000..fb19d2a3 --- /dev/null +++ b/devplacepy/services/sync/records.py @@ -0,0 +1,215 @@ +# retoor + +from __future__ import annotations + +import json +from typing import ClassVar, Optional + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from devplacepy.constants import TOPICS +from devplacepy.models import normalize_website_url +from devplacepy.services.sync.normalize import ( + clean_text, + instant_iso, + normalize_origin, + normalize_origin_uid, + normalize_username, +) +from devplacepy.services.sync.spec import ( + KIND_COMMENTS, + KIND_POSTS, + KIND_USERS, + LIMITS, +) + + +class SyncBatchForm(BaseModel): + origin: str + records: list[dict] = Field(max_length=LIMITS.max_batch_records) + dry_run: bool = False + + @field_validator("origin") + @classmethod + def valid_origin(cls, value): + return normalize_origin(value) + + @field_validator("records", mode="before") + @classmethod + def decode_records(cls, value): + if isinstance(value, list) and value and all(isinstance(item, str) for item in value): + value = value[0] if len(value) == 1 else "[" + ",".join(value) + "]" + if isinstance(value, str): + try: + return json.loads(value) + except ValueError: + raise ValueError("records must be a JSON array") + return value + + @field_validator("records") + @classmethod + def not_empty(cls, value): + if not value: + raise ValueError("records must contain at least one record") + return value + + +class SyncRecord(BaseModel): + model_config = ConfigDict(extra="ignore") + + kind: ClassVar[str] = "" + reference_fields: ClassVar[tuple[str, ...]] = () + + origin_uid: str + created_at: str + updated_at: Optional[str] = None + deleted: bool = False + + @field_validator( + "origin_uid", "author_uid", "post_uid", mode="before", check_fields=False + ) + @classmethod + def valid_reference(cls, value): + return normalize_origin_uid(value) + + @field_validator("parent_uid", mode="before", check_fields=False) + @classmethod + def valid_optional_reference(cls, value): + if value is None or value == "": + return None + return normalize_origin_uid(value) + + @field_validator("created_at", "updated_at", mode="before") + @classmethod + def valid_instant(cls, value): + if value is None or value == "": + return None + return instant_iso(value) + + @model_validator(mode="after") + def settle_instants(self): + if not self.created_at: + raise ValueError("created_at is required") + if not self.updated_at or self.updated_at < self.created_at: + self.updated_at = self.created_at + return self + + @classmethod + def known_fields(cls) -> set[str]: + return set(cls.model_fields) + + def content_fields(self) -> dict: + return self.model_dump(exclude={"updated_at"}) + + +class UserRecord(SyncRecord): + kind: ClassVar[str] = KIND_USERS + + username: str + bio: str = "" + location: str = "" + git_link: str = "" + website: str = "" + avatar_seed: Optional[str] = None + + @field_validator("bio", mode="before") + @classmethod + def valid_bio(cls, value): + return cls._bounded(value, LIMITS.max_bio_length, "bio", multiline=True) + + @field_validator("location", mode="before") + @classmethod + def valid_location(cls, value): + return cls._bounded(value, LIMITS.max_location_length, "location", multiline=False) + + @field_validator("git_link", "website", mode="before") + @classmethod + def valid_url(cls, value): + text = clean_text(value, multiline=False) + if not text or len(text) > LIMITS.max_url_length: + return "" + try: + return normalize_website_url(text) + except ValueError: + return "" + + @field_validator("avatar_seed", mode="before") + @classmethod + def valid_seed(cls, value): + text = clean_text(value, multiline=False) + if not text or len(text) > LIMITS.max_avatar_seed_length: + return None + return text + + @model_validator(mode="after") + def settle_username(self): + self.username = normalize_username(self.username, self.origin_uid) + return self + + @staticmethod + def _bounded(value, limit: int, name: str, *, multiline: bool) -> str: + text = clean_text(value, multiline=multiline) + if len(text) > limit: + raise ValueError(f"{name} exceeds {limit} characters") + return text + + +class PostRecord(SyncRecord): + kind: ClassVar[str] = KIND_POSTS + reference_fields: ClassVar[tuple[str, ...]] = ("author_uid",) + + author_uid: str + title: str = "" + content: str + topic: str = "random" + + @field_validator("title", mode="before") + @classmethod + def valid_title(cls, value): + text = clean_text(value, multiline=False) + if len(text) > LIMITS.max_title_length: + raise ValueError(f"title exceeds {LIMITS.max_title_length} characters") + return text + + @field_validator("content", mode="before") + @classmethod + def valid_content(cls, value): + text = clean_text(value, multiline=True) + if not LIMITS.min_post_length <= len(text) <= LIMITS.max_post_length: + raise ValueError( + f"content must be {LIMITS.min_post_length}-{LIMITS.max_post_length} characters" + ) + return text + + @field_validator("topic", mode="before") + @classmethod + def valid_topic(cls, value): + text = clean_text(value, multiline=False).lower() + return text if text in TOPICS else "random" + + +class CommentRecord(SyncRecord): + kind: ClassVar[str] = KIND_COMMENTS + reference_fields: ClassVar[tuple[str, ...]] = ("author_uid", "post_uid", "parent_uid") + + author_uid: str + post_uid: str + parent_uid: Optional[str] = None + content: str + + @field_validator("content", mode="before") + @classmethod + def valid_content(cls, value): + text = clean_text(value, multiline=True) + if not LIMITS.min_comment_length <= len(text) <= LIMITS.max_comment_length: + raise ValueError( + f"content must be {LIMITS.min_comment_length}-{LIMITS.max_comment_length} characters" + ) + return text + + +RECORD_MODELS = { + KIND_USERS: UserRecord, + KIND_POSTS: PostRecord, + KIND_COMMENTS: CommentRecord, +} diff --git a/devplacepy/services/sync/rewriter.py b/devplacepy/services/sync/rewriter.py new file mode 100644 index 00000000..642a2d08 --- /dev/null +++ b/devplacepy/services/sync/rewriter.py @@ -0,0 +1,65 @@ +# retoor + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from devplacepy.services.sync.spec import ALIAS + +PROTECTED = re.compile(r"```.*?```|~~~.*?~~~|```.*\Z|`[^`\n]*`|(?:https?|ftp)://\S+|www\.\S+", re.S) +NAME_CHARS = "A-Za-z0-9_-" + + +@dataclass +class Rewrite: + text: str + substitutions: list[tuple[str, str]] = field(default_factory=list) + + +class MentionRewriter: + def __init__(self, mapping: dict[str, str], standalone_min_length: int = ALIAS.standalone_min_length): + self.mapping = {key.lower(): value for key, value in mapping.items() if key and value} + self.standalone_min_length = standalone_min_length + names = sorted(self.mapping, key=len, reverse=True) + self.pattern = ( + re.compile(rf"(? position: + yield False, text[position : match.start()] + yield True, match.group(0) + position = match.end() + if position < len(text): + yield False, text[position:] + + def rewrite(self, text: str) -> Rewrite: + if not text or self.pattern is None: + return Rewrite(text or "") + substitutions: list[tuple[str, str]] = [] + + def swap(match: re.Match) -> str: + prefix, name = match.group(1), match.group(2) + if not prefix and len(name) < self.standalone_min_length: + return match.group(0) + replacement = self.mapping[name.lower()] + substitutions.append((name, replacement)) + return f"{prefix}{replacement}" + + parts = [chunk if protected else self.pattern.sub(swap, chunk) for protected, chunk in self.segments(text)] + return Rewrite("".join(parts), substitutions) + + def mentioned(self, text: str, minimum: int = ALIAS.mention_min_length) -> list[str]: + found: dict[str, str] = {} + pattern = re.compile(rf"(? + +from __future__ import annotations + +import hashlib +import random +from collections import Counter + +from devplacepy.database.sync import get_links_by_origin_uids +from devplacepy.services.sync.options import SyncOptions +from devplacepy.services.sync.spec import ( + KIND_POSTS, + SKIP_MIN_COMMENTS, + SKIP_PARENT, + SKIP_SAMPLED_OUT, + SKIP_TOPIC, +) + +BUCKETS = 100 + + +def bucket(origin: str, origin_uid: str) -> int: + digest = hashlib.sha256(f"{origin}:{origin_uid}".encode()).digest() + return int.from_bytes(digest[:4], "big") % BUCKETS + + +def jittered(seconds: int, percent: int, rng: random.Random | None = None) -> int: + if percent <= 0: + return seconds + spread = (rng or random).uniform(-percent, percent) / 100 + return max(1, round(seconds * (1 + spread))) + + +class Selector: + def __init__(self, origin: str, options: SyncOptions): + self.origin = origin + self.options = options + + def select(self, records: dict[str, list[dict]]) -> tuple[dict[str, list[dict]], Counter]: + skipped: Counter = Counter() + if not self.options.selective(): + return records, skipped + posts = records.get("posts", []) + known = get_links_by_origin_uids(self.origin, KIND_POSTS, [str(post.get("origin_uid")) for post in posts]) + counts = Counter(str(comment.get("post_uid")) for comment in records.get("comments", [])) + kept, dropped = [], set() + for post in posts: + uid = str(post.get("origin_uid")) + reason = None if uid in known else self.refusal(post, counts[uid]) + if reason: + skipped[reason] += 1 + dropped.add(uid) + else: + kept.append(post) + comments = [] + for comment in records.get("comments", []): + if str(comment.get("post_uid")) in dropped: + skipped[SKIP_PARENT] += 1 + else: + comments.append(comment) + return {**records, "posts": kept, "comments": comments}, skipped + + def refusal(self, post: dict, comment_count: int) -> str | None: + if self.options.topics and str(post.get("topic") or "random").lower() not in self.options.topics: + return SKIP_TOPIC + if comment_count < self.options.min_comments: + return SKIP_MIN_COMMENTS + if bucket(self.origin, str(post.get("origin_uid"))) >= self.options.sample_percent: + return SKIP_SAMPLED_OUT + return None diff --git a/devplacepy/services/sync/spec.py b/devplacepy/services/sync/spec.py new file mode 100644 index 00000000..80081eac --- /dev/null +++ b/devplacepy/services/sync/spec.py @@ -0,0 +1,139 @@ +# retoor + +from __future__ import annotations + +from dataclasses import dataclass + +KIND_USERS = "users" +KIND_POSTS = "posts" +KIND_COMMENTS = "comments" +KINDS = (KIND_USERS, KIND_POSTS, KIND_COMMENTS) + +STATUS_CREATED = "created" +STATUS_UPDATED = "updated" +STATUS_UNCHANGED = "unchanged" +STATUS_STALE = "stale" +STATUS_REJECTED = "rejected" +STATUS_FAILED = "failed" +STATUSES = ( + STATUS_CREATED, + STATUS_UPDATED, + STATUS_UNCHANGED, + STATUS_STALE, + STATUS_REJECTED, + STATUS_FAILED, +) + +REASON_INVALID = "invalid" +REASON_UNKNOWN_AUTHOR = "unknown_author" +REASON_UNKNOWN_POST = "unknown_post" +REASON_UNKNOWN_PARENT = "unknown_parent" +REASON_PARENT_MISMATCH = "parent_mismatch" +REASON_CONTENT_REFUSED = "content_refused" +REASON_PURGED_LOCALLY = "purged_locally" +REASON_LOCALLY_DELETED = "locally_deleted" +REASON_OLDER_THAN_SYNCED = "older_than_synced" +REASON_TOMBSTONE_UNKNOWN = "tombstone_for_unknown" +REASON_INTERNAL = "internal_error" +REASONS = ( + REASON_INVALID, + REASON_UNKNOWN_AUTHOR, + REASON_UNKNOWN_POST, + REASON_UNKNOWN_PARENT, + REASON_PARENT_MISMATCH, + REASON_CONTENT_REFUSED, + REASON_PURGED_LOCALLY, + REASON_LOCALLY_DELETED, + REASON_OLDER_THAN_SYNCED, + REASON_TOMBSTONE_UNKNOWN, + REASON_INTERNAL, +) + +SYNC_ACTOR = "sync" + +SKIP_SAMPLED_OUT = "sampled_out" +SKIP_MIN_COMMENTS = "min_comments" +SKIP_TOPIC = "topic_not_allowed" +SKIP_NO_AUTHOR = "no_author" +SKIP_IDENTITY = "identity_failed" +SKIP_TRANSLATION = "translation_failed" +SKIP_REWRITE = "rewrite_failed" +SKIP_PARENT = "parent_skipped" +SKIP_IDENTITY_CAP = "identity_cap" +TRANSIENT_SKIPS = (SKIP_IDENTITY, SKIP_TRANSLATION, SKIP_REWRITE, SKIP_IDENTITY_CAP) +SKIP_REASONS = ( + SKIP_SAMPLED_OUT, + SKIP_MIN_COMMENTS, + SKIP_TOPIC, + SKIP_NO_AUTHOR, + SKIP_IDENTITY, + SKIP_TRANSLATION, + SKIP_REWRITE, + SKIP_PARENT, + SKIP_IDENTITY_CAP, +) + + +@dataclass(frozen=True) +class SyncLimits: + max_batch_records: int = 50 + max_origin_length: int = 63 + min_origin_length: int = 2 + max_origin_uid_length: int = 64 + max_username_length: int = 32 + min_username_length: int = 3 + max_bio_length: int = 500 + max_location_length: int = 200 + max_url_length: int = 500 + max_avatar_seed_length: int = 64 + max_title_length: int = 500 + min_post_length: int = 10 + max_post_length: int = 125000 + min_comment_length: int = 3 + max_comment_length: int = 125000 + future_skew_seconds: int = 300 + earliest_year: int = 2000 + max_errors_reported: int = 5 + + +@dataclass(frozen=True) +class AliasLimits: + name_attempts: int = 3 + fallback_attempts: int = 60 + min_name_length: int = 3 + max_name_length: int = 32 + length_tolerance: int = 3 + shared_run_length: int = 4 + samples_kept: int = 3 + sample_chars: int = 200 + standalone_min_length: int = 4 + mention_min_length: int = 3 + max_new_identities_per_run: int = 100 + max_translations_per_run: int = 40 + translate_timeout_seconds: int = 90 + name_timeout_seconds: int = 60 + max_jitter_percent: int = 50 + max_min_comments: int = 1000 + sample_percent_default: int = 100 + banned_fragments: tuple[str, ...] = ("user", "bot", "gpt", "claude", "assistant", "anon") + + +@dataclass(frozen=True) +class SyncSpec: + limits: SyncLimits = SyncLimits() + kinds: tuple[str, ...] = KINDS + statuses: tuple[str, ...] = STATUSES + reasons: tuple[str, ...] = REASONS + + def describe(self) -> dict: + return { + "limits": self.limits.__dict__.copy(), + "kinds": list(self.kinds), + "statuses": list(self.statuses), + "reasons": list(self.reasons), + } + + +SPEC = SyncSpec() +LIMITS = SPEC.limits +ALIAS = AliasLimits() diff --git a/devplacepy/services/sync/style.py b/devplacepy/services/sync/style.py new file mode 100644 index 00000000..cb203f1e --- /dev/null +++ b/devplacepy/services/sync/style.py @@ -0,0 +1,70 @@ +# retoor + +from __future__ import annotations + +import json +import re +from dataclasses import asdict, dataclass + +from devplacepy.services.sync.spec import ALIAS + +NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$") + + +@dataclass(frozen=True) +class Style: + casing: str + length: int + has_digit: bool + separator: str + + @classmethod + def of(cls, name: str) -> "Style": + letters = [char for char in name if char.isalpha()] + if not letters or all(char.islower() for char in letters): + casing = "lower" + elif all(char.isupper() for char in letters): + casing = "upper" + elif letters[0].isupper() and all(char.islower() for char in letters[1:]): + casing = "title" + else: + casing = "mixed" + separator = "_" if "_" in name else "-" if "-" in name else "" + return cls(casing, len(name), any(char.isdigit() for char in name), separator) + + @classmethod + def load(cls, raw: str) -> "Style": + return cls(**json.loads(raw)) + + def dump(self) -> str: + return json.dumps(asdict(self), sort_keys=True) + + def describe(self) -> dict: + return asdict(self) + + def matches(self, candidate: str) -> bool: + other = Style.of(candidate) + return ( + other.casing == self.casing + and other.has_digit == self.has_digit + and other.separator == self.separator + and abs(other.length - self.length) <= ALIAS.length_tolerance + ) + + +def shares_run(first: str, second: str, length: int = ALIAS.shared_run_length) -> bool: + left, right = first.lower(), second.lower() + if len(left) < length or len(right) < length: + return left == right + return any(left[start : start + length] in right for start in range(len(left) - length + 1)) + + +def plausible(candidate: str, source: str, style: Style | None) -> bool: + if not ALIAS.min_name_length <= len(candidate) <= ALIAS.max_name_length or not NAME_PATTERN.match(candidate): + return False + lowered = candidate.lower() + if any(fragment in lowered for fragment in ALIAS.banned_fragments): + return False + if shares_run(candidate, source): + return False + return style is None or style.matches(candidate) diff --git a/devplacepy/services/sync/translator.py b/devplacepy/services/sync/translator.py new file mode 100644 index 00000000..584d2928 --- /dev/null +++ b/devplacepy/services/sync/translator.py @@ -0,0 +1,32 @@ +# retoor + +from __future__ import annotations + +from devplacepy.services.sync import ai +from devplacepy.services.sync.spec import ALIAS + +SYSTEM = ( + "You translate forum text into the language with code '{target}'. Preserve the original author's " + "register, tone, complexity and formatting. Do not translate or alter code blocks, inline code, URLs " + "or @mentions. If the text is already in the target language return it unchanged. Reply with the " + "translated text only, no preface, no quotes." +) +SAMPLES_NOTE = "\n\nThe same author previously wrote, match this voice:\n" + + +class Translator: + def __init__(self, target: str): + self.target = target + + def needed(self, source_language: str) -> bool: + source = (source_language or "").strip().lower().split("-")[0] + return not source or source != self.target.split("-")[0] + + async def translate(self, text: str, samples: list[str] | None = None) -> str | None: + if not text.strip(): + return text + system = SYSTEM.format(target=self.target) + if samples: + system += SAMPLES_NOTE + "\n".join(f"- {sample}" for sample in samples) + reply = await ai.complete(system, text, ALIAS.translate_timeout_seconds) + return reply.strip() if reply and reply.strip() else None diff --git a/devplacepy/static/css/admin.css b/devplacepy/static/css/admin.css index 0db45d58..176741e4 100644 --- a/devplacepy/static/css/admin.css +++ b/devplacepy/static/css/admin.css @@ -607,3 +607,95 @@ font-family: monospace; font-size: 0.8125rem; } + +.nodes-state { + display: inline-flex; + margin-right: var(--space-sm); + font-size: 0.8125rem; + font-weight: 700; +} + +.nodes-state.on, +.nodes-status.active { + color: var(--success); +} + +.nodes-state.off, +.nodes-status.revoked { + color: var(--danger); +} + +.nodes-status.suspended { + color: var(--warning); +} + +.nodes-status { + font-weight: 600; +} + +.nodes-banner { + padding: var(--space-sm) var(--space-md); + margin-bottom: var(--space-md); + border: 1px solid var(--border); + border-left: 3px solid var(--warning); + border-radius: var(--radius); + background: var(--bg-card); + color: var(--text-secondary); +} + +.nodes-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(min(100%, 14rem), 1fr)); + gap: var(--space-md); + margin-bottom: var(--space-md); +} + +.nodes-panel { + min-width: 0; + padding: var(--space-md); + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--bg-card); +} + +.nodes-panel h3, +.nodes-heading { + margin-bottom: var(--space-sm); + font-size: 0.875rem; + font-weight: 700; + color: var(--text-secondary); +} + +.nodes-heading { + margin-top: var(--space-md); +} + +.nodes-breakdown { + display: grid; + grid-template-columns: minmax(0, 1fr) max-content; + gap: var(--space-xs) var(--space-md); + font-size: 0.8125rem; +} + +.nodes-breakdown dt { + min-width: 0; + overflow-wrap: anywhere; + color: var(--text-muted); +} + +.nodes-breakdown dd { + font-weight: 600; + text-align: right; +} + +.nodes-detail { + overflow-wrap: anywhere; +} + +.nodes-settings { + max-width: 600px; +} + +.nodes-settings .admin-field input { + max-width: 200px; +} diff --git a/devplacepy/static/css/communities.css b/devplacepy/static/css/communities.css new file mode 100644 index 00000000..d71800f0 --- /dev/null +++ b/devplacepy/static/css/communities.css @@ -0,0 +1,100 @@ +.community-banner { + padding: var(--space-sm) var(--space-md); + margin-bottom: var(--space-md); + border: 1px solid var(--border); + border-left: 3px solid var(--warning); + border-radius: var(--radius); + background: var(--bg-card); + color: var(--text-secondary); +} + +.community-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(0, 1fr)); + gap: var(--space-md); + margin-bottom: var(--space-md); +} + +.community-panel { + min-width: 0; + padding: var(--space-md); + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--bg-card); +} + +.community-panel h3 { + margin-bottom: var(--space-sm); +} + +.community-meta { + display: grid; + grid-template-columns: max-content minmax(0, 1fr); + gap: var(--space-xs) var(--space-md); +} + +.community-meta dt { + color: var(--text-muted); +} + +.community-meta dd { + min-width: 0; + overflow-wrap: anywhere; +} + +.community-add-form { + display: flex; + flex-wrap: wrap; + gap: var(--space-sm); + align-items: flex-end; + margin-bottom: var(--space-md); +} + +.community-add-form label { + flex: 1 1 16rem; + min-width: 0; +} + +.community-log { + max-height: 24rem; + overflow: auto; + padding: var(--space-sm); + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--bg-card); + font-family: var(--font-mono); + font-size: 0.75rem; + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +.community-status.ready { + color: var(--success); +} + +.community-status.pending, +.community-status.learning { + color: var(--info); +} + +.community-status.failed, +.community-status.failing, +.community-status.login_required { + color: var(--danger); +} + +@media (max-width: 768px) { + .community-meta { + grid-template-columns: minmax(0, 1fr); + } +} + +.community-login-fields { + flex: 1 1 100%; + min-width: 0; +} + +.community-login-fields label { + display: block; + margin-top: var(--space-sm); +} diff --git a/devplacepy/static/css/nodes.css b/devplacepy/static/css/nodes.css new file mode 100644 index 00000000..8819e6e7 --- /dev/null +++ b/devplacepy/static/css/nodes.css @@ -0,0 +1,304 @@ +/* retoor */ + +.node-donation { + display: flex; + flex-direction: column; + gap: var(--space-xl); + min-width: 0; +} + +.node-section { + display: flex; + flex-direction: column; + gap: var(--space-md); + min-width: 0; +} + +.node-section h3, +.node-section h4 { + margin: 0; +} + +.node-explain-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--space-lg); +} + +.node-explain-card { + padding: var(--space-lg); + border: 1px solid var(--border-light); + border-radius: var(--radius); + background: var(--bg-secondary); + min-width: 0; +} + +.node-explain-card ul { + margin: var(--space-sm) 0 0; + padding-left: var(--space-xl); + color: var(--text-secondary); +} + +.node-row { + display: flex; + align-items: center; + gap: var(--space-md); + min-width: 0; +} + +.node-row-label { + flex: 1; + min-width: 0; + color: var(--text-primary); +} + +.node-hint { + margin: 0; + font-size: 0.85rem; + color: var(--text-muted); +} + +.node-notice { + margin: 0; + padding: var(--space-md); + border: 1px solid var(--warning); + border-radius: var(--radius); + color: var(--warning); +} + +.node-stats { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: var(--space-md); +} + +.node-stat { + display: flex; + flex-direction: column; + gap: var(--space-xs); + padding: var(--space-md); + border: 1px solid var(--border-light); + border-radius: var(--radius); + background: var(--bg-secondary); + min-width: 0; +} + +.node-stat-label { + font-size: 0.75rem; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--text-secondary); +} + +.node-stat-value { + font-size: 1.25rem; + font-weight: 700; + color: var(--text-primary); + overflow-wrap: anywhere; +} + +.node-stat-date { + font-size: 1rem; +} + +.node-stat-sub { + font-size: 0.8125rem; + color: var(--text-muted); +} + +.node-form { + display: flex; + flex-direction: column; + gap: var(--space-md); +} + +.node-fields { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(10rem, 1fr)); + gap: var(--space-md); +} + +.node-field { + display: flex; + flex-direction: column; + gap: var(--space-xs); + font-size: 0.8125rem; + color: var(--text-secondary); + min-width: 0; +} + +.node-platforms { + display: flex; + flex-wrap: wrap; + gap: var(--space-md); + margin: 0; + padding: var(--space-md); + border: 1px solid var(--border-light); + border-radius: var(--radius); + min-width: 0; +} + +.node-check { + display: inline-flex; + align-items: center; + gap: var(--space-sm); + color: var(--text-primary); +} + +.node-check input { + width: auto; +} + +.node-actions { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--space-sm); +} + +.node-devices, +.node-artifacts { + display: flex; + flex-direction: column; + gap: var(--space-sm); + margin: 0; + padding: 0; + list-style: none; +} + +.node-device { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1.5fr) auto; + align-items: center; + gap: var(--space-md); + padding: var(--space-md); + border: 1px solid var(--border-light); + border-radius: var(--radius); + background: var(--bg-secondary); +} + +.node-device-main, +.node-device-meta, +.node-device-actions { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--space-sm); + min-width: 0; +} + +.node-device-meta { + font-size: 0.8125rem; + color: var(--text-muted); +} + +.node-device-label { + font-weight: 700; + overflow-wrap: anywhere; +} + +.node-device-remove { + color: var(--danger); + border-color: var(--danger); +} + +.node-secret { + display: flex; + flex-direction: column; + gap: var(--space-md); + padding: var(--space-lg); + border: 1px solid var(--accent); + border-radius: var(--radius); + min-width: 0; +} + +.node-install { + display: flex; + flex-direction: column; + gap: var(--space-sm); + min-width: 0; +} + +.node-install-lines { + margin: 0; + padding: var(--space-md); + font-family: var(--font-mono); + font-size: 0.8125rem; + white-space: pre; + overflow-x: auto; + background: var(--bg-input); + color: var(--text-primary); + border-radius: var(--radius); +} + +.node-os-group { + display: flex; + flex-wrap: wrap; + gap: var(--space-sm); +} + +.node-os-select { + display: none; +} + +.node-os-button.active { + border-color: var(--accent); + color: var(--accent); +} + +.node-artifact { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--space-sm); + min-width: 0; +} + +.node-artifact-size { + color: var(--text-muted); + font-size: 0.8125rem; +} + +.node-checksum { + font-family: var(--font-mono); + font-size: 0.75rem; + color: var(--text-muted); + overflow-wrap: anywhere; + min-width: 0; +} + +@media (max-width: 768px) { + .node-explain-grid { + grid-template-columns: minmax(0, 1fr); + } + + .node-stats { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .node-device { + grid-template-columns: minmax(0, 1fr); + } + + .node-os-group { + display: none; + } + + .node-os-select { + display: block; + } +} + +@media (max-width: 480px) { + .node-stats { + grid-template-columns: minmax(0, 1fr); + } + + .node-row { + flex-wrap: wrap; + } + + .node-actions .btn { + flex: 1 1 100%; + } +} diff --git a/devplacepy/static/js/AdminNodes.js b/devplacepy/static/js/AdminNodes.js new file mode 100644 index 00000000..6508e86b --- /dev/null +++ b/devplacepy/static/js/AdminNodes.js @@ -0,0 +1,108 @@ +// retoor + +import { Http } from "./Http.js"; +import { Poller } from "./Poller.js"; + +const REFRESH_MS = 10000; +const PAGE_URL = "/admin/nodes"; + +class AdminNodes { + constructor() { + this.root = document.querySelector("[data-nodes-root]"); + if (!this.root) return; + this.modal = document.getElementById("node-suspend-modal"); + this.suspendForm = document.getElementById("node-suspend-form"); + this.suspendTitle = document.getElementById("node-suspend-title"); + this.killswitch = document.querySelector("[data-nodes-killswitch]"); + this.settings = document.querySelector("[data-nodes-settings]"); + this.bindActions(); + this.bindSuspendForm(); + this.bindKillswitch(); + this.bindSettings(); + this.poller = new Poller(() => this.refresh(), REFRESH_MS, { pauseHidden: true, immediate: false }); + } + + async refresh() { + const response = await fetch(PAGE_URL, { headers: { "Accept": "text/html" } }); + if (!response.ok) return; + const fresh = new DOMParser().parseFromString(await response.text(), "text/html"); + this.root.querySelectorAll("[data-live]").forEach((section) => { + const next = fresh.querySelector(`[data-live="${section.dataset.live}"]`); + if (next) section.innerHTML = next.innerHTML; + }); + const state = fresh.querySelector("[data-nodes-killswitch]"); + if (state && this.killswitch) this.killswitch.innerHTML = state.innerHTML; + } + + bindActions() { + this.root.addEventListener("click", async (event) => { + const button = event.target.closest("[data-node-action]"); + if (!button) return; + const uid = button.dataset.uid; + const action = button.dataset.nodeAction; + if (action === "suspend") { + this.openSuspend(uid, button.dataset.label); + return; + } + const urls = { + "unsuspend": `/admin/nodes/devices/${uid}/unsuspend`, + "resolve-flag": `/admin/nodes/flags/${uid}/resolve`, + }; + await this.post(urls[action], {}, action === "unsuspend" ? "Suspension lifted" : "Flag resolved"); + }); + } + + openSuspend(uid, label) { + this.suspendForm.action = `/admin/nodes/devices/${uid}/suspend`; + this.suspendTitle.textContent = `Suspend device: ${label}`; + this.modal.classList.add("visible"); + } + + bindSuspendForm() { + this.suspendForm.addEventListener("submit", async (event) => { + event.preventDefault(); + const params = Object.fromEntries(new FormData(this.suspendForm).entries()); + if (await this.post(this.suspendForm.action, params, "Device suspended")) { + this.modal.classList.remove("visible"); + this.suspendForm.reset(); + } + }); + } + + bindKillswitch() { + if (!this.killswitch) return; + this.killswitch.addEventListener("submit", async (event) => { + event.preventDefault(); + const turningOn = this.killswitch.querySelector("[name='enabled']").value === "true"; + const message = turningOn + ? "Switch community donation on? Consenting donors can connect again." + : "Switch community donation off? Every connected node is disconnected within seconds."; + const ok = await window.app.dialog.confirm({ message, danger: !turningOn }); + if (!ok) return; + await this.post(this.killswitch.action, { enabled: String(turningOn) }, "Kill switch updated"); + }); + } + + bindSettings() { + if (!this.settings) return; + this.settings.addEventListener("submit", async (event) => { + event.preventDefault(); + const params = Object.fromEntries(new FormData(this.settings).entries()); + await this.post(this.settings.action, params, "Settings saved"); + }); + } + + async post(url, params, message) { + try { + await Http.send(url, params); + window.app.toast.show(message, { type: "success" }); + await this.refresh(); + return true; + } catch { + return false; + } + } +} + +window.AdminNodes = AdminNodes; +export { AdminNodes }; diff --git a/devplacepy/static/js/Application.js b/devplacepy/static/js/Application.js index 8c35b833..af66fc6f 100644 --- a/devplacepy/static/js/Application.js +++ b/devplacepy/static/js/Application.js @@ -25,7 +25,9 @@ import { AiCorrection } from "./AiCorrection.js"; import { AiModifier } from "./AiModifier.js"; import { InteractionsPref } from "./InteractionsPref.js"; import { TelegramPairing } from "./TelegramPairing.js"; +import { NodeDonation } from "./NodeDonation.js"; import { AdminNotificationDefaults } from "./AdminNotificationDefaults.js"; +import { AdminNodes } from "./AdminNodes.js"; import { UserAiUsage } from "./UserAiUsage.js"; import { Tabs } from "./Tabs.js"; import { DeviiTerminal } from "./DeviiTerminal.js"; @@ -100,7 +102,9 @@ class Application { this.aiModifier = new AiModifier(); this.interactionsPref = new InteractionsPref(); this.telegramPairing = new TelegramPairing(this.pubsub); + this.nodeDonation = new NodeDonation(); this.adminNotificationDefaults = new AdminNotificationDefaults(); + this.adminNodes = new AdminNodes(); this.userAiUsage = new UserAiUsage(); this.tabs = new Tabs(); this.windows = new WindowManager(); diff --git a/devplacepy/static/js/NodeDonation.js b/devplacepy/static/js/NodeDonation.js new file mode 100644 index 00000000..8101e115 --- /dev/null +++ b/devplacepy/static/js/NodeDonation.js @@ -0,0 +1,486 @@ +// retoor + +import { Http } from "./Http.js"; +import { Poller } from "./Poller.js"; +import { Format } from "./Format.js"; + +class NodeDonation { + static POLL_MS = 30000; + static BYTES_PER_MB = 1048576; + static MAX_ARTIFACTS = 64; + static MAX_DEVICES = 64; + static MAX_TIMEZONES = 600; + static EXTENSION_MARK = "extension"; + + constructor() { + this.root = document.querySelector("[data-node-donation]"); + if (!this.root) return; + this.username = this.root.dataset.username; + this.me = null; + this.client = null; + this.secret = ""; + this.secretKind = "native"; + this.artifactIndex = 0; + this.el = { + consent: this.root.querySelector("#node-consent-toggle"), + consentState: this.root.querySelector("#node-consent-state"), + consentMessage: this.root.querySelector("#node-consent-message"), + body: this.root.querySelector("#node-body"), + notice: this.root.querySelector("#node-notice"), + settingsForm: this.root.querySelector("#node-settings-form"), + settingsStatus: this.root.querySelector("#node-settings-status"), + devices: this.root.querySelector("#node-devices"), + devicesEmpty: this.root.querySelector("#node-devices-empty"), + deviceForm: this.root.querySelector("#node-device-form"), + deviceStatus: this.root.querySelector("#node-device-status"), + secret: this.root.querySelector("#node-secret"), + secretValue: this.root.querySelector("#node-secret-value"), + secretCopy: this.root.querySelector("#node-secret-copy"), + secretHide: this.root.querySelector("#node-secret-hide"), + os: this.root.querySelector("#node-os"), + osGroup: this.root.querySelector("#node-os-group"), + osSelect: this.root.querySelector("#node-os-select"), + installLines: this.root.querySelector("#node-install-lines"), + installCopy: this.root.querySelector("#node-install-copy"), + installChecksum: this.root.querySelector("#node-install-checksum"), + artifacts: this.root.querySelector("#node-artifacts"), + timezones: this.root.querySelector("#node-timezones"), + }; + this.bind(); + this.fillTimezones(); + this.loadClient(); + this.poller = new Poller(() => this.refresh(), NodeDonation.POLL_MS, { + immediate: false, + pauseHidden: true, + }); + } + + bind() { + this.el.consent.addEventListener("change", () => this.setConsent()); + this.el.settingsForm.addEventListener("submit", (event) => { + event.preventDefault(); + this.saveSettings(); + }); + this.el.deviceForm.addEventListener("submit", (event) => { + event.preventDefault(); + this.createDevice(); + }); + this.el.devices.addEventListener("click", (event) => this.onDeviceClick(event)); + this.el.secretCopy.addEventListener("click", () => this.copy(this.secret, this.el.secretCopy)); + this.el.installCopy.addEventListener("click", () => + this.copy(this.el.installLines.textContent, this.el.installCopy) + ); + this.el.secretHide.addEventListener("click", () => this.hideSecret()); + this.el.osSelect.addEventListener("change", () => this.selectArtifact(Number(this.el.osSelect.value))); + this.el.osGroup.addEventListener("click", (event) => { + const button = event.target.closest("[data-index]"); + if (button) this.selectArtifact(Number(button.dataset.index)); + }); + } + + fillTimezones() { + if (typeof Intl.supportedValuesOf !== "function") return; + const zones = Intl.supportedValuesOf("timeZone").slice(0, NodeDonation.MAX_TIMEZONES); + for (const zone of zones) { + const option = document.createElement("option"); + option.value = zone; + this.el.timezones.appendChild(option); + } + } + + megabytes(bytes) { + return `${(Number(bytes || 0) / NodeDonation.BYTES_PER_MB).toFixed(1)} MB`; + } + + flash(target, text, state) { + target.textContent = text; + target.dataset.state = state; + } + + async refresh() { + const me = await Http.getJson("/nodes/me"); + this.apply(me); + } + + apply(me) { + this.me = me; + this.el.consent.checked = me.consent; + this.el.consentState.textContent = me.consent ? "granted" : "withdrawn"; + this.el.consentState.classList.toggle("status-actioned", me.consent); + this.el.consentState.classList.toggle("status-dismissed", !me.consent); + this.el.body.hidden = !me.consent; + this.el.notice.textContent = me.message || ""; + this.el.notice.hidden = me.can_donate; + this.renderStats(me.donated || {}); + this.renderDevices(me.devices || []); + } + + renderStats(donated) { + const text = { + lifetime_requests: Format.exact(donated.lifetime_requests), + lifetime_bytes: this.megabytes(donated.lifetime_bytes), + month_requests: Format.exact(donated.month_requests), + month_bytes_up: this.megabytes(donated.month_bytes_up), + month_bytes_down: this.megabytes(donated.month_bytes_down), + today_requests: Format.exact(donated.today_requests), + today_bytes: this.megabytes(donated.today_bytes), + }; + for (const [key, value] of Object.entries(text)) { + this.root.querySelectorAll(`[data-node-stat="${key}"]`).forEach((node) => { + node.textContent = value; + }); + } + for (const key of ["first_donated_at", "last_donated_at"]) { + const node = this.root.querySelector(`[data-node-stat-date="${key}"]`); + if (node) node.replaceChildren(this.time(donated[key], "date", "never")); + } + } + + time(iso, mode, empty) { + if (!iso) return document.createTextNode(empty); + const node = document.createElement("time"); + node.setAttribute("datetime", iso); + node.dataset.dt = ""; + node.dataset.dtMode = mode; + node.textContent = iso; + return node; + } + + span(text, className) { + const node = document.createElement("span"); + if (className) node.className = className; + node.textContent = text; + return node; + } + + button(text, className, uid) { + const node = document.createElement("button"); + node.type = "button"; + node.className = className; + node.dataset.uid = uid; + node.textContent = text; + return node; + } + + renderDevice(device) { + const item = document.createElement("li"); + item.className = "node-device"; + item.dataset.uid = device.uid; + const main = document.createElement("div"); + main.className = "node-device-main"; + const badge = { active: "actioned", suspended: "open" }[device.status] || "dismissed"; + main.append( + this.span(device.label || device.uid, "node-device-label"), + this.span(device.status, `status-badge status-${badge}`) + ); + const meta = document.createElement("div"); + meta.className = "node-device-meta"; + const seen = this.span("Last seen "); + seen.appendChild(this.time(device.last_seen_at, "ago", "never")); + meta.append( + this.span(device.platform ? `${device.kind} / ${device.platform}` : device.kind), + this.span(`${device.country || "unknown country"}, ${device.asn_class}`), + seen + ); + const actions = document.createElement("div"); + actions.className = "node-device-actions"; + const remove = this.button("Remove", "btn btn-sm btn-danger node-device-remove", device.uid); + remove.dataset.confirm = "Remove this device? It disconnects at once and its secret stops working."; + remove.dataset.confirmDanger = ""; + remove.dataset.confirmTitle = "Remove device"; + remove.dataset.confirmLabel = "Remove"; + actions.append(this.button("New secret", "btn btn-sm node-device-rotate", device.uid), remove); + item.append(main, meta, actions); + return item; + } + + renderDevices(devices) { + const capped = devices.slice(0, NodeDonation.MAX_DEVICES); + this.el.devices.replaceChildren(...capped.map((device) => this.renderDevice(device))); + this.el.devicesEmpty.hidden = capped.length > 0; + } + + async setConsent() { + const granted = this.el.consent.checked; + this.el.consent.disabled = true; + try { + await Http.sendForm(`/profile/${this.username}/consent`, { + kind: "bandwidth_sharing", + granted: granted ? "1" : "0", + }); + this.flash(this.el.consentMessage, granted ? "Consent given." : "Consent withdrawn. Donation is off.", "success"); + await this.refresh(); + } catch (error) { + this.el.consent.checked = !granted; + this.flash(this.el.consentMessage, "Could not change the consent.", "error"); + } finally { + this.el.consent.disabled = false; + } + } + + settingsPayload() { + const form = this.el.settingsForm; + const value = (name) => form.elements[name].value; + return { + enabled: form.elements.enabled.checked, + bandwidth_mb_day: Number(value("bandwidth_mb_day")), + max_concurrent: Number(value("max_concurrent")), + active_from: value("active_from"), + active_to: value("active_to"), + timezone: value("timezone").trim() || "UTC", + allowed_platforms: [...form.querySelectorAll(".node-platform:checked")].map((box) => box.value), + pause_on_battery: form.elements.pause_on_battery.checked, + public_display: form.elements.public_display.checked, + }; + } + + async saveSettings() { + const save = this.root.querySelector("#node-settings-save"); + save.disabled = true; + this.flash(this.el.settingsStatus, "Saving...", ""); + try { + await Http.postJson("/nodes/settings", this.settingsPayload()); + this.flash(this.el.settingsStatus, "Saved", "success"); + await this.refresh(); + } catch (error) { + this.flash(this.el.settingsStatus, "Could not save the settings", "error"); + } finally { + save.disabled = false; + } + } + + async loadClient() { + try { + this.client = await Http.getJson("/nodes/client-version"); + } catch (error) { + this.client = { version: "", artifacts: [] }; + } + this.renderArtifacts(); + } + + artifacts() { + return (this.client.artifacts || []).slice(0, NodeDonation.MAX_ARTIFACTS); + } + + nativeArtifacts() { + return this.artifacts().filter((item) => !item.name.includes(NodeDonation.EXTENSION_MARK)); + } + + extensionArtifact() { + return this.artifacts().find((item) => item.name.includes(NodeDonation.EXTENSION_MARK)) || null; + } + + renderArtifacts() { + const items = this.artifacts().map((artifact) => { + const item = document.createElement("li"); + item.className = "node-artifact"; + const link = document.createElement("a"); + link.href = `/nodes/download/${encodeURIComponent(artifact.name)}`; + link.textContent = artifact.name; + const checksum = document.createElement("code"); + checksum.className = "node-checksum"; + checksum.textContent = artifact.sha256; + item.append(link, this.span(this.megabytes(artifact.size), "node-artifact-size"), checksum); + return item; + }); + if (!items.length) { + const empty = document.createElement("li"); + empty.className = "node-hint"; + empty.textContent = "No downloads are published yet."; + items.push(empty); + } + this.el.artifacts.replaceChildren(...items); + } + + deviceKind(uid) { + const device = ((this.me && this.me.devices) || []).find((item) => item.uid === uid); + return device ? device.kind : "native"; + } + + async createDevice() { + const form = this.el.deviceForm; + const create = this.root.querySelector("#node-device-create"); + create.disabled = true; + this.flash(this.el.deviceStatus, "Adding...", ""); + try { + const kind = form.elements.kind.value; + const created = await Http.postJson("/nodes/devices", { + label: form.elements.label.value.trim(), + kind, + platform: form.elements.platform.value, + client_version: (this.client && this.client.version) || "", + }); + this.flash(this.el.deviceStatus, "Device added", "success"); + form.elements.label.value = ""; + this.showSecret(created.secret, created.device.kind); + await this.refresh(); + } catch (error) { + this.flash(this.el.deviceStatus, "Could not add the device", "error"); + } finally { + create.disabled = false; + } + } + + async onDeviceClick(event) { + const rotate = event.target.closest(".node-device-rotate"); + const remove = event.target.closest(".node-device-remove"); + if (rotate) await this.rotate(rotate); + if (remove) await this.remove(remove); + } + + async rotate(button) { + button.disabled = true; + try { + const result = await Http.sendForm(`/nodes/devices/${button.dataset.uid}/rotate`, {}); + this.showSecret(result.secret, this.deviceKind(button.dataset.uid)); + } catch (error) { + this.flash(this.el.deviceStatus, "Could not create a new secret", "error"); + } finally { + button.disabled = false; + } + } + + async remove(button) { + button.disabled = true; + try { + await Http.sendForm(`/nodes/devices/${button.dataset.uid}/delete`, {}); + await this.refresh(); + } catch (error) { + button.disabled = false; + this.flash(this.el.deviceStatus, "Could not remove the device", "error"); + } + } + + showSecret(secret, kind) { + this.secret = secret; + this.secretKind = kind; + this.el.secretValue.textContent = secret; + this.el.secret.hidden = false; + this.renderInstall(); + this.el.secret.scrollIntoView({ block: "nearest" }); + } + + hideSecret() { + this.secret = ""; + this.el.secretValue.textContent = ""; + this.el.installLines.textContent = ""; + this.el.secret.hidden = true; + } + + osOf(name) { + const lower = name.toLowerCase(); + if (lower.includes("windows") || lower.endsWith(".exe")) return "Windows"; + if (lower.includes("darwin") || lower.includes("macos")) return "macOS"; + return "Linux"; + } + + detectIndex(artifacts) { + const agent = navigator.userAgent.toLowerCase(); + const wanted = agent.includes("windows") ? "Windows" : agent.includes("mac") ? "macOS" : "Linux"; + const found = artifacts.findIndex((item) => this.osOf(item.name) === wanted); + return found < 0 ? 0 : found; + } + + renderInstall() { + const native = this.secretKind !== "extension"; + const artifacts = native ? this.nativeArtifacts() : []; + this.el.os.hidden = artifacts.length < 2; + if (native && artifacts.length) { + this.artifactIndex = this.detectIndex(artifacts); + this.renderOsChoices(artifacts); + } + this.renderLines(); + } + + renderOsChoices(artifacts) { + const buttons = artifacts.map((artifact, index) => { + const button = document.createElement("button"); + button.type = "button"; + button.className = "btn btn-sm node-os-button"; + button.dataset.index = String(index); + button.textContent = artifact.name.replace("devplace-node-", "").replace(".exe", ""); + return button; + }); + const options = artifacts.map((artifact, index) => { + const option = document.createElement("option"); + option.value = String(index); + option.textContent = artifact.name.replace("devplace-node-", "").replace(".exe", ""); + return option; + }); + this.el.osGroup.replaceChildren(...buttons); + this.el.osSelect.replaceChildren(...options); + this.markOs(); + } + + markOs() { + this.el.osSelect.value = String(this.artifactIndex); + this.el.osGroup.querySelectorAll("[data-index]").forEach((button) => { + button.classList.toggle("active", Number(button.dataset.index) === this.artifactIndex); + }); + } + + selectArtifact(index) { + this.artifactIndex = index; + this.markOs(); + this.renderLines(); + } + + renderLines() { + const origin = location.origin; + if (this.secretKind === "extension") { + const extension = this.extensionArtifact(); + const lines = [ + extension + ? `1. Download ${origin}/nodes/download/${extension.name} and unpack it.` + : "1. The extension download is not published yet.", + "2. Chrome: chrome://extensions, enable Developer mode, Load unpacked, pick the unpacked folder.", + " Firefox: about:debugging#/runtime/this-firefox, Load Temporary Add-on, pick manifest.json.", + "3. Open the extension popup, enter this server and the secret above, then switch sharing on.", + `Server: ${origin}`, + ]; + this.el.installLines.textContent = lines.join("\n"); + this.el.installChecksum.textContent = extension ? `SHA-256 of ${extension.name}: ${extension.sha256}` : ""; + return; + } + const artifact = this.nativeArtifacts()[this.artifactIndex]; + if (!artifact) { + this.el.installLines.textContent = `devplace-node login --server ${origin} --secret ${this.secret}\ndevplace-node run`; + this.el.installChecksum.textContent = "No native client is published yet."; + return; + } + const url = `${origin}/nodes/download/${artifact.name}`; + const windows = this.osOf(artifact.name) === "Windows"; + const lines = windows + ? [ + `curl.exe -fsSL -o devplace-node.exe "${url}"`, + "certutil -hashfile devplace-node.exe SHA256", + `.\\devplace-node.exe login --server ${origin} --secret ${this.secret}`, + ".\\devplace-node.exe run", + ] + : [ + `curl -fsSL -o devplace-node "${url}"`, + `echo "${artifact.sha256} devplace-node" | shasum -a 256 -c -`, + "chmod +x devplace-node", + `./devplace-node login --server ${origin} --secret ${this.secret}`, + "./devplace-node run", + ]; + this.el.installLines.textContent = lines.join("\n"); + this.el.installChecksum.textContent = `SHA-256 of ${artifact.name}: ${artifact.sha256}`; + } + + async copy(text, button) { + const label = button.textContent; + try { + await navigator.clipboard.writeText(text); + button.textContent = "Copied"; + } catch (error) { + button.textContent = "Copy failed"; + } + window.setTimeout(() => { + button.textContent = label; + }, 1500); + } +} + +window.NodeDonation = NodeDonation; +export { NodeDonation }; diff --git a/devplacepy/templates/_node_donation.html b/devplacepy/templates/_node_donation.html new file mode 100644 index 00000000..a4d437eb --- /dev/null +++ b/devplacepy/templates/_node_donation.html @@ -0,0 +1,196 @@ +{% set donated = node_me.donated %} +{% set node_settings = node_me.settings %} +
+
+

Donate to the community

+

Some sites refuse requests that come from data centres. By donating a little of your internet connection, community search can reach those public pages through yours.

+
+
+

What you share

+
    +
  • A small part of your bandwidth, within the daily limit you set.
    • +
  • Your public IP address, as seen by the site that is fetched.
    • +
  • Requests for public pages on a fixed allowlist of sites, nothing else.
    • +
+
+
+

What you never share

+
    +
  • Your cookies, logins and accounts are never used.
    • +
  • Your browsing history, tabs and files are never touched.
    • +
  • You can switch it off at any time, here or by stopping the client.
    • +
+
+
+

Read the full details in Donate your connection and the Privacy Policy. Check the terms of use of your internet line before running a node on a residential or business connection.

+
+ + + +
+

{{ node_me.message }}

+ +
+

Donated by you

+
+
Requests served{{ donated.lifetime_requests }}
+
Data transferred{{ ((donated.lifetime_bytes or 0) / 1048576) | round(1) }} MB
+
This month{{ donated.month_requests }}requests, {{ ((donated.month_bytes_up or 0) / 1048576) | round(1) }} MB up, {{ ((donated.month_bytes_down or 0) / 1048576) | round(1) }} MB down
+
Today{{ donated.today_requests }}requests, {{ ((donated.today_bytes or 0) / 1048576) | round(1) }} MB
+
First donated{% if donated.first_donated_at %}{{ local_dt(donated.first_donated_at, 'date') }}{% else %}never{% endif %}
+
Last donated{% if donated.last_donated_at %}{{ local_dt(donated.last_donated_at, 'date') }}{% else %}never{% endif %}
+
+
+ +
+

Settings

+
+
+ + Donation is switched on +
+
+ + + + + +
+

Leave both hours empty to donate at any time of day.

+
+ Sites you allow + {% for platform in node_me.platforms %} + + {% endfor %} +
+
+ + Pause while running on battery +
+
+ + Show my name on the donor leaderboard +
+
+ + +
+
+
+ +
+

Your devices

+

Each machine or browser that donates is a device with its own secret. You can have up to {{ node_me.max_devices }}.

+
    + {% for device in node_me.devices %} +
  • +
    + {{ device.label or device.uid }} + {{ device.status }} +
    +
    + {{ device.kind }}{% if device.platform %} / {{ device.platform }}{% endif %} + {{ device.country or 'unknown country' }}, {{ device.asn_class }} + Last seen {% if device.last_seen_at %}{{ local_dt(device.last_seen_at, 'ago') }}{% else %}never{% endif %} +
    +
    + + +
    +
    • + {% endfor %} +
+

No devices yet. Add one below.

+ +
+
+ + + +
+
+ + +
+
+ + +
+ +
+

Downloads

+

Verify the checksum of anything you download before running it.

+
    +
  • Loading downloads.
    • +
+
+
+
diff --git a/devplacepy/templates/admin_base.html b/devplacepy/templates/admin_base.html index 19e3961b..fb1f09a3 100644 --- a/devplacepy/templates/admin_base.html +++ b/devplacepy/templates/admin_base.html @@ -38,6 +38,12 @@ ⏰ Devii tasks + + 🌐 Communities + + + 📡 Community nodes + 🤖 Bot Monitor diff --git a/devplacepy/templates/admin_communities.html b/devplacepy/templates/admin_communities.html new file mode 100644 index 00000000..ab8f2743 --- /dev/null +++ b/devplacepy/templates/admin_communities.html @@ -0,0 +1,74 @@ +{% extends "admin_base.html" %} +{% block extra_head %} +{{ super() }} + +{% endblock %} +{% block admin_content %} +
+

Communities

+ {{ items|length }} +
+ +{% if not sync_enabled %} +

Community sync is switched off (sync_enabled in settings). Learning works, dry runs work, live runs are refused.

+{% endif %} +{% if not service_enabled %} +

The Community sync service is stopped, so scheduled runs do not happen. New communities are still learned immediately.

+{% elif service_dry_run %} +

The service is in dry run: it collects and logs what it would do and writes nothing. Switch it off in the service configuration.

+{% endif %} + +
+ + + +
+ +
+ + + + + + + + + + + + + {% for item in items %} + + + + + + + + {% else %} + + {% endfor %} + +
External communities synchronised into this instance
CommunityStatusRunsLast runNext run
+ {{ item['name'] or item['slug'] }} + {{ item['origin'] }} + + {{ item['status'] }} + {% if not item['enabled'] %}disabled{% endif %} + {% if item['last_error'] %}{{ item['last_error'][:120] }}{% endif %} + {{ item['run_count'] }}{% if item['failure_count'] %} {{ item['failure_count'] }} failed{% endif %}{% if item['last_run_at'] %}{{ local_dt(item['last_run_at']) }}{% else %}-{% endif %}{% if item['next_run_at'] %}{{ local_dt(item['next_run_at']) }}{% else %}-{% endif %}
No communities yet. Enter a name or address above.
+
+{% endblock %} diff --git a/devplacepy/templates/admin_community.html b/devplacepy/templates/admin_community.html new file mode 100644 index 00000000..7bf25714 --- /dev/null +++ b/devplacepy/templates/admin_community.html @@ -0,0 +1,153 @@ +{% extends "admin_base.html" %} +{% block extra_head %} +{{ super() }} + +{% endblock %} +{% block admin_content %} +
+

{{ community['name'] or community['slug'] }}

+ All communities +
+ +{% if busy %} +

A run or a discovery is in progress. Reload this page in a moment to see its result.

+{% endif %} +{% if not sync_enabled %} +

Community sync is switched off: live runs are refused, dry runs still work.

+{% endif %} +{% if community['last_error'] %} + +{% endif %} + +
+
+

About

+
+
Status
{{ community['status'] }}
+
Address
{{ community['url'] }}
+
Origin
{{ community['origin'] }}
+
Language
{{ community['language'] or '-' }}
+
Members
{{ community['user_count'] or '-' }}
+
Learned
{% if community['learned_at'] %}{{ local_dt(community['learned_at']) }}{% else %}-{% endif %}
+
Description
{{ community['description'] or '-' }}
+
+
+ +
+

Schedule

+
+ + + {% set opts = community['sync_options'] or {} %} + + + +
+
+
Next run
{% if community['next_run_at'] %}{{ local_dt(community['next_run_at']) }}{% else %}-{% endif %}
+
Runs
{{ community['run_count'] }} ({{ community['failure_count'] }} consecutive failures)
+ {% for key, value in totals.items() %}
{{ key.replace('_', ' ') }}
{{ value }}
{% endfor %} +
+
+ +
+

Actions

+
+
+ + +
+
+ +
+
+ +
+
+ +
+
+ {% if collectors_path %}

Generated collectors: {{ collectors_path }} ({{ collectors_lines }} lines)

{% endif %} +
+
+ +{% if rules %} +
+

Rules and guidelines (as published)

+

{{ rules }}

+
+{% endif %} + +

Learned feeds

+
+ + + + + {% for feed in feeds %} + + + + + + + + {% else %} + + {% endfor %} + +
Feeds learned for this community
FeedTypePost fieldsCommentsPagination
{{ feed['url'] }}{{ feed['type'] }}{{ feed['list_path'] or feed['item_selector'] or '' }}{% for name, paths in feed['post_fields'].items() %}{{ name }}: {{ paths|join(' | ') if paths is not string else paths }}
{% endfor %}		{{ feed['comments'] }}{% if feed['comments_reason'] %}{{ feed['comments_reason'] }}{% endif %}{{ feed['pagination'] }}
No feed learned yet.
+
+ +

Recent runs

+{% for run in runs %} +
+ + {{ local_dt(run['started_at']) }} - {{ run['status'] }}{% if run['dry_run'] %} (dry run){% endif %} + {{ run['trigger'] }} + +

collected {{ run['collected'] }} / pushed {{ run['pushed'] }}

+ {% for error in run['errors'] %}

{{ error }}

{% endfor %} + 
{{ run['log'] }}
+
+{% else %} +

No runs yet.

+{% endfor %} +{% endblock %} diff --git a/devplacepy/templates/admin_nodes.html b/devplacepy/templates/admin_nodes.html new file mode 100644 index 00000000..fad38eee --- /dev/null +++ b/devplacepy/templates/admin_nodes.html @@ -0,0 +1,168 @@ +{% extends "admin_base.html" %} +{% block extra_head %} +{{ super() }} + +{% endblock %} +{% macro breakdown(title, counts) %} +
+

{{ title }}

+ {% if counts %} +
+ {% for name, total in counts.items() %} +
{{ name }}
+
{{ total }}
+ {% endfor %} +
+ {% else %} +

None yet.

+ {% endif %} +
+{% endmacro %} +{% block admin_content %} +
+

Community nodes

+
+ + {{ 'Donation on' if enabled else 'Donation off' }} + +
+
+ +
+
+ {% if not pool.online %} +

The nodes service is not running, so no donor can be connected right now. Start it on the service page.

+ {% endif %} +
+
{{ pool.connected }}Connected
+
{{ pool.capacity }}Capacity
+
{{ queue.depth }} / {{ queue.max_depth }}Queue depth
+
{{ pool.open_circuits }}Open circuits
+
{{ pool.jobs_done }}Jobs done
+
{{ pool.jobs_failed }}Jobs failed
+
{{ pool.kicked }}Kicked
+
{{ flags|length }}Open flags
+
+
+ {{ breakdown("Devices by status", devices_by_status) }} + {{ breakdown("Devices by kind", devices_by_kind) }} + {{ breakdown("Devices by address class", devices_by_asn_class) }} + {{ breakdown("Devices by country", devices_by_country) }} + {{ breakdown("Requests", {"queued": queue.queued, "assigned": queue.assigned, "done": queue.done, "failed": queue.failed, "expired": queue.expired}) }} + {{ breakdown("Today (UTC)", usage_today) }} +
+
+ +

Open flags

+
+ + + + + + + + + + + + + + {% for flag in flags %} + + + + + + + + + {% else %} + + {% endfor %} + +
Open node flags
KindOwnerDeviceDetailRaisedAction
{{ flag.kind }}{% if flag.owner_username %}{{ flag.owner_username }}{% else %}-{% endif %}{{ flag.device_label or flag.device_uid or '-' }}{{ flag.detail or '-' }}{% if flag.created_at %}{{ local_dt(flag.created_at) }}{% else %}-{% endif %}
No open flags.
+
+ +

Recent devices

+
+ + + + + + + + + + + + + + + {% for device in devices %} + + + + + + + + + + {% else %} + + {% endfor %} + +
Most recently seen node devices
DeviceOwnerStatusKindRegionLast seenAction
{{ device.label or device.uid }}{{ device.platform }} {{ device.client_version }}{% if device.owner_username %}{{ device.owner_username }}{% else %}-{% endif %} + {{ device.status }} + {% if device.status == 'suspended' and device.suspend_reason %}{{ device.suspend_reason }}{% endif %} + {{ device.kind }}{{ device.country or '-' }} {{ device.asn_class }}{% if device.last_seen_at %}{{ local_dt(device.last_seen_at) }}{% else %}-{% endif %} + {% if device.status == 'suspended' %} + + {% elif device.status == 'active' %} + + {% endif %} +
No devices registered yet.
+
+
+ +

Settings

+
+ {% for field in settings %} +
+ + {% if field.kind == 'flag' %} + + {% else %} + + Default {{ field.default }}, maximum {{ field.ceiling }} + {% endif %} +
+ {% endfor %} + +
+ + +{% endblock %} diff --git a/devplacepy/templates/admin_settings.html b/devplacepy/templates/admin_settings.html index 69d4c27e..002a4514 100644 --- a/devplacepy/templates/admin_settings.html +++ b/devplacepy/templates/admin_settings.html @@ -104,6 +104,21 @@ When enabled, a page that would otherwise 404 renders a random existing post instead. HTML requests only - JSON/API clients still get a real 404. +
+ + + When enabled, administrators can copy users, posts and comments in from other communities through the sync API. +
+ +
+ + + Comma separated origins allowed to sync, for example other.example.org. Leave empty to allow any origin. +
+
diff --git a/devplacepy/templates/docs/nodes-protocol.html b/devplacepy/templates/docs/nodes-protocol.html index eda232b0..4b7ed79f 100644 --- a/devplacepy/templates/docs/nodes-protocol.html +++ b/devplacepy/templates/docs/nodes-protocol.html @@ -3,7 +3,7 @@ This page is generated from the protocol specification, `services/nodes/spec.py`, the same data served at [`/nodes/spec.json`](/nodes/spec.json). The server, the clients, the tests and this page all read that one source, so the numbers below cannot drift from the code. -**Status today:** the specification, `/nodes/spec.json`, device registration, the socket ticket, the node socket and the consent gate are implemented on the server and are switched off by default. The clients and the search integration are not built yet, so no node can do anything useful and no job is ever sent. Nothing on this page is enabled for any account. +**Status today:** the specification, `/nodes/spec.json`, device registration, the socket ticket, the node socket and the consent gate are implemented on the server and are switched off by default. The job queue, scheduler, tunnel bridge and result checks run on the server, a native Go client and a browser extension speak this protocol, and social search and DeepSearch can use the network. The tunnel path has been run end to end against a real client; how well it works from residential lines is not measured yet. Nothing on this page is enabled for any account until an administrator switches the network on. ## Rules that never change diff --git a/devplacepy/templates/docs/nodes.html b/devplacepy/templates/docs/nodes.html new file mode 100644 index 00000000..05f561a2 --- /dev/null +++ b/devplacepy/templates/docs/nodes.html @@ -0,0 +1,104 @@ +
+# Donate your connection + +Some sites refuse requests that come from data centres. DevPlace lets members donate a little of their +own internet connection so that community search can read a small allowlist of public pages through it. +Donating is optional, off by default, and can be stopped at any time. The wire protocol for client +authors is on the [Node network protocol](/docs/nodes-protocol.html) page. + +## What donating is + +You register one or more **devices**. Each device runs a client, either the native client or a browser +extension, and keeps one connection open to DevPlace. When the network is switched on and a search needs +a page on the allowlist, DevPlace sends that one request to a device, the device fetches the public page +and returns it. The fetch leaves from your internet connection, so the site sees your IP address. + +The network is switched on and off by the administrators. While it is off, nothing is sent to any device. + +## What is shared and what is not + +| Shared | Never shared | +|---|---| +| A small part of your bandwidth, capped by the daily limit you set | Your cookies, logins and accounts, which are never used | +| Your public IP address, as seen by the site that is fetched | Your browsing history, tabs, files and other programs | +| Requests for public pages on the allowlist, and the bytes they move | Any site that is not on the allowlist | + +Today the allowlist holds `reddit.com` and `redd.it`. Clients refuse anything else, a non-public address, +a non-https destination, and any redirect that leaves the allowlist, whatever DevPlace asks. + +A device sees the destination host of every request. In fetch mode, which the browser extension uses, it +can also read the URL and the page that comes back, so only donate with a client you trust. Your own +searches are never sent through your own device. + +## Check the terms of your internet line + +Before you run a node on a residential or business connection, check your provider's terms of use. Some +providers forbid sharing a connection or running services on it. Donating is your decision and your +responsibility. + +## Start donating + +1. Open the **Donate** tab of your profile. +2. Read the explanation and give the bandwidth sharing consent. Everything else stays hidden until you do. +3. Choose your limits: daily bandwidth, simultaneous requests, active hours and timezone, the sites you + allow, whether to pause on battery, and whether your name may appear on the donor leaderboard. +4. Add a device. The **device secret** is shown once, with ready-to-paste install lines and a copy + button. If you lose it, create a new secret for that device; the old one stops working. +5. Switch donation on and save. + +A member can register up to five devices. The consent and the switch are separate: withdrawing the +consent stops donation, and the switch alone starts nothing without it. + +## Install the native client + +The native client is a single program for Linux, macOS and Windows. The Donate tab lists every published +build with its size and SHA-256 checksum, and builds the install lines for the operating system you pick. +They download the program, verify the checksum, store the server and secret, and run it: + +``` +devplace-node login --server https://devplace.net --secret dpn_... +devplace-node run +devplace-node status +``` + +`login` stores the server and secret in your user config directory, `run` connects and serves requests in +the foreground, and `status` shows the stored configuration and today's usage. + +## Install the browser extension + +The extension works in Chrome and Firefox and always runs in fetch mode. Download the extension zip from +the Donate tab, unpack it, load it as an unpacked extension (`chrome://extensions` in Chrome, +`about:debugging` in Firefox), then enter the server and the device secret in its popup and switch +sharing on. A disclosure page opens on first install. Read it before you accept. + +## How to stop + +- Close the program, or switch the extension off, to stop that device at once. +- Switch donation off in the settings to stop every device. +- Withdraw the consent to stop and hide the controls. +- Remove a device to disconnect it immediately and make its secret useless. + +## The counters + +The **Donated by you** card on the Donate tab shows requests served and data transferred, up and down, +for today, this month and your lifetime, plus the dates of your first and last donation. Lifetime totals +are kept when old daily figures are cleaned up. The card refreshes every 30 seconds. + +Each device row shows its status, the last time it was seen and an approximate country and kind of +network (residential, mobile, datacenter or unknown). The numbers are measured by DevPlace, not by the +client. + +## Limits + +| Limit | Value | +|---|---| +| Devices per member | 5 | +| Daily bandwidth you can offer | up to 20 GB, 500 MB by default | +| Simultaneous requests per device | up to 2 | + +DevPlace applies the lowest of the server limit, your setting and the limit set in the client, and never raises a limit you set. + +## Your data + +What DevPlace stores about donating is listed in the [Privacy Policy](/docs/privacy.html). +
diff --git a/devplacepy/templates/docs/privacy.html b/devplacepy/templates/docs/privacy.html index 221c254d..7af47722 100644 --- a/devplacepy/templates/docs/privacy.html +++ b/devplacepy/templates/docs/privacy.html @@ -16,6 +16,7 @@ used, and how it is retained, deleted and shared. | Technical | IP address and user agent on audit-relevant actions, request timing | Security, abuse prevention and rate limiting | | Assistant | Prompts and results of assistant, search and analysis features you invoke | Deliver the feature and attribute its cost to you | | Moderation | Reports you file, reports filed about you, and decisions taken | Operate the safety controls the platform is required to have | +| Donated bandwidth | Only if you donate: your device label and secret hash, a keyed hash of your IP address (never the address itself), an approximate country and network class, request and byte counters, and node events | Run the community node network and show you what you donated. See section 10. | DevPlace runs **no third-party analytics, no advertising and no cross-site or cross-app tracking**. There is no tracking identifier shared with any other company. @@ -34,6 +35,8 @@ never used to build an advertising profile. | Push services (Apple, browser push endpoints) | An encrypted notification payload to a device you registered | Only when you register a device and enable a push channel | | Telegram | Messages in the Telegram conversation you paired | Only when you pair Telegram yourself | | Software another member runs in a container | Your DevPlace API key and user id, injected as `DEVPLACE_API_KEY` and `DEVPLACE_USER_UID` | **Only after you grant `container_credentials` consent**, and only when the container is run by someone else. Your own containers use your own credentials with no separate consent. | +| External communities an administrator synchronises | Requests to those sites, and an account the administrator supplied for a login-protected one | Administrator action only. No member data is sent. | +| The site fetched through your connection | Your public IP address, as the destination sees it | **Only after you grant `bandwidth_sharing` consent**, register a device and switch donation on | | Email and issue-tracker hosts you configure | The content you send through them | Only when you configure them yourself | Every one of these is under your control, is off unless you turn it on, and can be revoked from your @@ -41,20 +44,21 @@ profile privacy tab. ## 4. Consent -DevPlace records five consents, each versioned, each independently grantable and withdrawable, with +DevPlace records six consents, each versioned, each independently grantable and withdrawable, with the full history retained so a decision is always provable: - `terms` - acceptance of the Terms of Service and Community Guidelines; - `privacy` - acknowledgement of this policy; - `ai_third_party` - processing of **your own content** by a third-party AI model provider; - `activity_recording` - recording of your presence and session activity; -- `container_credentials` - sharing your credentials with software another member runs in a container. +- `container_credentials` - sharing your credentials with software another member runs in a container; +- `bandwidth_sharing` - sharing part of your internet connection with the community node network. Withdrawal takes effect immediately. Withdrawing `ai_third_party` turns every assistant feature that would send your content off, whatever the per-feature preference says. Withdrawing `activity_recording` stops presence writes; you simply appear offline. -Manage all five at `/profile/{username}?tab=privacy`. +Manage the consents at `/profile/{username}?tab=privacy`. ## 5. Retention and deletion @@ -96,7 +100,29 @@ that age. If you believe an account belongs to a younger child, report it or use Material changes bump the version at the top of this page. -## 10. Contact +## 10. Donated bandwidth + +The community node network is optional and off until you give the `bandwidth_sharing` consent, register +a device and switch donation on. What it processes and keeps: + +- **IP address and location.** Your IP address is processed to connect your device and to estimate an + approximate country and network class (residential, mobile, datacenter). **Only a keyed hash of the + address is stored**, never the address. +- **Requests.** The table of in-flight requests holds the requested URL and the identifier of the caller + for at most 10 minutes, then it is deleted. +- **Counters.** Request and byte counts per day and for your lifetime are kept so you can see what you + donated. Lifetime totals are kept after old daily figures are cleaned up. +- **Events.** Connection and refusal events for your devices are kept for 30 days. +- **What the donor device sees.** A device sees the destination host of every request. In fetch mode, + which the browser extension uses, it can also read the URL and the page body. Your own requests are not + sent through your own device. +- **What is never used.** Your cookies, logins and accounts are never used, and only public pages on a + fixed allowlist are fetched. + +Withdrawing the consent switches donation off at once. Removing a device disconnects it and makes its +secret useless. See [Donate your connection](/docs/nodes.html). + +## 11. Contact Data-protection enquiries go to the address, email and telephone number on the [Contact](/docs/contact.html) page. diff --git a/devplacepy/templates/profile.html b/devplacepy/templates/profile.html index 2e912a39..afe86906 100644 --- a/devplacepy/templates/profile.html +++ b/devplacepy/templates/profile.html @@ -6,6 +6,7 @@ +{% if current_tab == 'donate' %}{% endif %} {% endblock %} {% block content %}
@@ -562,6 +563,9 @@ 🔔 Notifications 🔒 Privacy {% endif %} + {% if is_owner %} + 🤝 Donate + {% endif %}
@@ -731,6 +735,10 @@ {% endif %} {% endif %} + {% elif current_tab == 'donate' %} + {% if is_owner %} + {% include "_node_donation.html" %} + {% endif %} {% elif current_tab == 'notifications' %} {% if is_owner or viewer_is_admin %}
diff --git a/devplacepy/utils/accounts.py b/devplacepy/utils/accounts.py index 267d263d..bed4369e 100644 --- a/devplacepy/utils/accounts.py +++ b/devplacepy/utils/accounts.py @@ -8,6 +8,53 @@ from devplacepy.utils.badges import award_badge from devplacepy.utils.passwords import hash_password, hash_password_async +def account_row( + uid: str, + username: str, + email: str, + password_hash: str, + role: str, + now: str, + *, + age_band: str = "", + terms_version: str = "", + accepted_terms: bool = False, +) -> dict: + return { + "uid": uid, + "username": username, + "email": email.strip().lower(), + "api_key": generate_uid(), + "password_hash": password_hash, + "bio": "", + "location": "", + "git_link": "", + "website": "", + "role": role, + "is_active": True, + "level": 1, + "xp": 0, + "stars": 0, + "ai_correction_enabled": 0, + "ai_correction_prompt": DEFAULT_CORRECTION_PROMPT, + "ai_correction_sync": 0, + "ai_modifier_enabled": 1, + "ai_modifier_prompt": DEFAULT_MODIFIER_PROMPT, + "ai_modifier_sync": 1, + "interactions_enabled": -1, + "avatar_seed": None, + "terms_version": terms_version, + "terms_accepted_at": now if accepted_terms else "", + "age_band": age_band, + "age_declared_at": now if age_band else "", + "mature_opt_in": 0, + "suspended_until": "", + "suspension_reason": "", + "deletion_requested_at": "", + "created_at": now, + } + + def _create_account( username: str, email: str, @@ -25,39 +72,17 @@ def _create_account( now = datetime.now(timezone.utc).isoformat() terms_version = (get_setting("terms_version", "1") or "1") if accepted_terms else "" users.insert( - { - "uid": uid, - "username": username, - "email": email.strip().lower(), - "api_key": generate_uid(), - "password_hash": password_hash, - "bio": "", - "location": "", - "git_link": "", - "website": "", - "role": role, - "is_active": True, - "level": 1, - "xp": 0, - "stars": 0, - "ai_correction_enabled": 0, - "ai_correction_prompt": DEFAULT_CORRECTION_PROMPT, - "ai_correction_sync": 0, - "ai_modifier_enabled": 1, - "ai_modifier_prompt": DEFAULT_MODIFIER_PROMPT, - "ai_modifier_sync": 1, - "interactions_enabled": -1, - "avatar_seed": None, - "terms_version": terms_version, - "terms_accepted_at": now if accepted_terms else "", - "age_band": age_band, - "age_declared_at": now if age_band else "", - "mature_opt_in": 0, - "suspended_until": "", - "suspension_reason": "", - "deletion_requested_at": "", - "created_at": now, - } + account_row( + uid, + username, + email, + password_hash, + role, + now, + age_band=age_band, + terms_version=terms_version, + accepted_terms=accepted_terms, + ) ) if accepted_terms: set_consent("user", uid, "terms", True, version=terms_version) diff --git a/devplacepy/utils/badges.py b/devplacepy/utils/badges.py index db7657f0..a1dda4eb 100644 --- a/devplacepy/utils/badges.py +++ b/devplacepy/utils/badges.py @@ -33,6 +33,11 @@ BADGE_CATALOG = { "Popular": {"icon": "◎", "description": "Reached 10 followers", "group": "Reputation"}, "Influencer": {"icon": "📢", "description": "Reached 50 followers", "group": "Reputation"}, "Celebrity": {"icon": "👑", "description": "Reached 100 followers", "group": "Reputation"}, + "Donor": {"icon": "🤝", "description": "Donated bandwidth to the community", "group": "Community"}, + "Gigabyte Donor": {"icon": "📡", "description": "Donated 1 GB of bandwidth", "group": "Community"}, + "10 GB Donor": {"icon": "🛰", "description": "Donated 10 GB of bandwidth", "group": "Community"}, + "100 GB Donor": {"icon": "🌐", "description": "Donated 100 GB of bandwidth", "group": "Community"}, + "Steady Donor": {"icon": "🗓", "description": "Donated on 30 different days", "group": "Community"}, "Connector": {"icon": "🔗", "description": "Followed 10 people", "group": "Community"}, "On Fire": {"icon": "🔥", "description": "Maintained a 7-day activity streak", "group": "Dedication"}, "Dedicated": {"icon": "📅", "description": "Maintained a 30-day activity streak", "group": "Dedication"}, diff --git a/events.md b/events.md index d5b01c2c..01d11a93 100644 --- a/events.md +++ b/events.md @@ -2,7 +2,7 @@ retoor -Every state-changing action in DevPlace records one append-only row through `devplacepy/services/audit/record.py` (`record` for a request-scoped actor, `record_system` for a background one). This file is the authoritative catalogue of the 362 event keys currently emitted, grouped by the category `services/audit/categories.py` `category_for` resolves them to. +Every state-changing action in DevPlace records one append-only row through `devplacepy/services/audit/record.py` (`record` for a request-scoped actor, `record_system` for a background one). This file is the authoritative catalogue of the 380 event keys currently emitted, grouped by the category `services/audit/categories.py` `category_for` resolves them to. ## Rules @@ -19,12 +19,12 @@ Every state-changing action in DevPlace records one append-only row through `dev | Category | Keys | |---|---| | `account` | 16 | -| `admin` | 21 | +| `admin` | 26 | | `ai` | 11 | | `attachment` | 5 | | `auth` | 8 | | `backup` | 2 | -| `cli` | 43 | +| `cli` | 49 | | `container` | 41 | | `content` | 37 | | `database` | 4 | @@ -36,7 +36,7 @@ Every state-changing action in DevPlace records one append-only row through `dev | `message` | 2 | | `moderation` | 12 | | `news` | 9 | -| `nodes` | 3 | +| `nodes` | 9 | | `notification` | 4 | | `project` | 12 | | `project_files` | 14 | @@ -46,10 +46,11 @@ Every state-changing action in DevPlace records one append-only row through `dev | `security` | 3 | | `service` | 5 | | `social` | 10 | +| `sync` | 1 | | `telegram` | 5 | | `tools` | 12 | -**Total: 362 keys.** +**Total: 380 keys.** ## Account and profile (`account`) @@ -84,6 +85,11 @@ Every state-changing action in DevPlace records one append-only row through `dev | `admin.backup_schedule.delete` | `routers/admin/backups.py` | | `admin.backup_schedule.toggle` | `routers/admin/backups.py` | | `admin.backup_schedule.update` | `routers/admin/backups.py` | +| `admin.community.add` | `routers/admin/communities.py` | +| `admin.community.delete` | `routers/admin/communities.py` | +| `admin.community.relearn` | `routers/admin/communities.py` | +| `admin.community.run` | `routers/admin/communities.py` | +| `admin.community.update` | `routers/admin/communities.py` | | `admin.devii_task.delete` | `routers/admin/devii_tasks.py` | | `admin.devii_task.disable` | `routers/admin/devii_tasks.py` | | `admin.game.era_end` | `routers/admin/game.py` | @@ -160,7 +166,13 @@ Every state-changing action in DevPlace records one append-only row through `dev | `cli.containers.gc_workspaces` | `cli/containers.py` | | `cli.containers.prune` | `cli/containers.py` | | `cli.containers.prune_builds` | `cli/containers.py` | +| `cli.community.discover` | `cli/community.py` | +| `cli.community.login` | `cli/community.py` | +| `cli.community.run` | `cli/community.py` | | `cli.containers.reconcile` | `cli/containers.py` | +| `cli.nodes.clear` | `cli/nodes.py` | +| `cli.nodes.suspend` | `cli/nodes.py` | +| `cli.nodes.unsuspend` | `cli/nodes.py` | | `cli.deepsearch.clear` | `cli/jobs.py` | | `cli.deepsearch.prune` | `cli/jobs.py` | | `cli.devii.lessons.clear` | `cli/devii.py` | @@ -548,9 +560,21 @@ Every state-changing action in DevPlace records one append-only row through `dev | Event key | Recorded in | |---|---| +| `node.admin.settings` | `routers/admin/nodes.py` | | `node.device.create` | `routers/nodes/devices.py` | | `node.device.delete` | `routers/nodes/devices.py` | | `node.device.rotate` | `routers/nodes/devices.py` | +| `node.flag.resolve` | `routers/admin/nodes.py` | +| `node.killswitch` | `routers/admin/nodes.py` | +| `node.settings.update` | `routers/nodes/me.py` | +| `node.suspend` | `routers/admin/nodes.py` | +| `node.unsuspend` | `routers/admin/nodes.py` | + +## Community sync (`sync`) + +| Event key | Recorded in | +|---|---| +| `sync.batch` | `routers/sync/_shared.py` | ## Developer tools (`tools`) diff --git a/nodes/client/README.md b/nodes/client/README.md new file mode 100644 index 00000000..75621cc7 --- /dev/null +++ b/nodes/client/README.md @@ -0,0 +1,62 @@ +# devplace-node + +Native donor client for the DevPlace community node network. One static Go binary, standard library only. + +## What you share + +When running, and only inside the limits you configured: + +- Your internet connection and your IP address, used to fetch public pages from the destinations on the allowlist (today: `reddit.com`, `redd.it` and their subdomains) over ports 80 and 443, GET only. +- The fetched public page, which DevPlace receives and may cache for a short time. +- Byte counters (up and down) so your daily cap and your lifetime totals are correct. + +## What is never shared + +- Your cookies, logins, accounts and browser. The client has no cookie jar and drops `Cookie`, `Authorization` and `Proxy-Authorization` from every request. It uses no browser profile and reads no files besides its own config. +- Any access to your local network: addresses that are private, loopback, link-local, CGNAT or multicast are refused after local DNS resolution, and the connection goes to the address that was checked. +- Any destination outside the baked-in allowlist. A list sent by the server can only be equal or narrower. Redirects that leave the allowlist are refused. +- Anything on ports other than 80 and 443. + +The device secret (`dpn_...`) is stored in the config file with mode 0600 and is never logged or printed. + +## Use + +``` +devplace-node login --server https://devplace.net --secret dpn_... +devplace-node status +devplace-node run +devplace-node version +``` + +Create the device secret on your DevPlace profile after consenting to bandwidth sharing. `login` asks the server to accept the secret before storing it (`--no-verify` skips that). The secret can also come from `DEVPLACE_NODE_SECRET`. Every command takes `--config PATH`. + +`run` is quiet: it logs start, disconnects, and one totals line per day. Add `-v` for detail. Exit code 3 means the server ended this device for good (revoked, suspended, kill switch, no consent, not adult, replaced by another instance of the same device); fix the cause and start it again. + +## Config file + +Stored at `/devplace-node/config.json`, with `usage.json` (today's byte count) beside it. + +```json +{ + "server": "https://devplace.net", + "secret": "dpn_...", + "bandwidth_mb_day": 500, + "max_concurrent": 2, + "max_tunnels": 2, + "active_from": "", + "active_to": "", + "timezone": "" +} +``` + +`active_from` and `active_to` are `HH:MM` in `timezone` (empty means the system timezone); a window may wrap midnight. These limits are enforced locally and the server can only make them stricter. + +## Build and package + +`./build.sh [outdir]` cross-compiles linux/amd64, linux/arm64, darwin/arm64, darwin/amd64 and windows/amd64 with `CGO_ENABLED=0 -trimpath`, and writes `manifest.json` (name, os, arch, sha256, size, version) beside the binaries. `VERSION=1.2.3 ./build.sh` overrides the version. + +`packaging/` holds a systemd user unit (`loginctl enable-linger $USER` keeps it running without a login), a launchd plist, a Windows scheduled-task script and a container compose file. + +## Tests + +`go vet ./... && go test ./...` diff --git a/nodes/client/build.sh b/nodes/client/build.sh new file mode 100755 index 00000000..ffc5a3ce --- /dev/null +++ b/nodes/client/build.sh @@ -0,0 +1,54 @@ +#!/bin/sh +set -eu + +here=$(cd "$(dirname "$0")" && pwd) +out=${1:-./dist} +version=${VERSION:-$(git -C "$here" describe --tags --always --dirty 2>/dev/null || echo dev)} +targets="linux/amd64 linux/arm64 darwin/arm64 darwin/amd64 windows/amd64" + +mkdir -p "$out" +out=$(cd "$out" && pwd) +manifest="$out/manifest.json" +rm -f "$manifest" + +hash_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} + +size_of() { + wc -c <"$1" | tr -d ' ' +} + +printf '[\n' >"$manifest" +first=1 +for target in $targets; do + goos=${target%/*} + goarch=${target#*/} + name="devplace-node-$goos-$goarch" + if [ "$goos" = "windows" ]; then + name="$name.exe" + fi + (cd "$here" && CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags "-s -w -X main.Version=$version" -o "$out/$name" .) + if [ "$first" -eq 0 ]; then + printf ',\n' >>"$manifest" + fi + first=0 + printf ' {"name": "%s", "os": "%s", "arch": "%s", "sha256": "%s", "size": %s, "version": "%s"}' \ + "$name" "$goos" "$goarch" "$(hash_of "$out/$name")" "$(size_of "$out/$name")" "$version" >>"$manifest" +done +printf '\n]\n' >>"$manifest" + +for target in $targets; do + goos=${target%/*} + goarch=${target#*/} + name="devplace-node-$goos-$goarch" + if [ "$goos" = "windows" ]; then + name="$name.exe" + fi + echo "$out/$name" +done +echo "$manifest" diff --git a/nodes/client/config.go b/nodes/client/config.go new file mode 100644 index 00000000..c052a4a0 --- /dev/null +++ b/nodes/client/config.go @@ -0,0 +1,177 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +const ( + DefaultBandwidthMB = 500 + DefaultMaxConcurrent = 2 + DefaultMaxTunnels = 2 + ConfigDirName = "devplace-node" + ConfigFileName = "config.json" + UsageFileName = "usage.json" + maxConfigBytes = 64 * 1024 +) + +var hoursPattern = regexp.MustCompile(`^([01]\d|2[0-3]):[0-5]\d$`) + +type Config struct { + Server string `json:"server"` + Secret string `json:"secret"` + BandwidthMBDay int `json:"bandwidth_mb_day"` + MaxConcurrent int `json:"max_concurrent"` + MaxTunnels int `json:"max_tunnels"` + ActiveFrom string `json:"active_from"` + ActiveTo string `json:"active_to"` + Timezone string `json:"timezone"` +} + +func DefaultConfig() Config { + return Config{ + BandwidthMBDay: DefaultBandwidthMB, + MaxConcurrent: DefaultMaxConcurrent, + MaxTunnels: DefaultMaxTunnels, + } +} + +func DefaultConfigPath() (string, error) { + base, err := os.UserConfigDir() + if err != nil { + return "", fmt.Errorf("cannot find the user config directory: %w", err) + } + return filepath.Join(base, ConfigDirName, ConfigFileName), nil +} + +func (c Config) Validate() error { + parsed, err := url.Parse(c.Server) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return errors.New("server must be an http or https url such as https://devplace.net") + } + if !strings.HasPrefix(c.Secret, SecretPrefix) || len(c.Secret) < len(SecretPrefix)+8 { + return errors.New("secret must be a device secret starting with " + SecretPrefix) + } + if c.BandwidthMBDay < 1 || c.BandwidthMBDay > MaxDonorMBPerDay { + return fmt.Errorf("bandwidth_mb_day must be between 1 and %d", MaxDonorMBPerDay) + } + if c.MaxConcurrent < 1 || c.MaxConcurrent > MaxConcurrentCeiling { + return fmt.Errorf("max_concurrent must be between 1 and %d", MaxConcurrentCeiling) + } + if c.MaxTunnels < 1 || c.MaxTunnels > MaxTunnelsCeiling { + return fmt.Errorf("max_tunnels must be between 1 and %d", MaxTunnelsCeiling) + } + for _, value := range []string{c.ActiveFrom, c.ActiveTo} { + if value != "" && !hoursPattern.MatchString(value) { + return errors.New("active_from and active_to must be empty or HH:MM") + } + } + if (c.ActiveFrom == "") != (c.ActiveTo == "") { + return errors.New("active_from and active_to must be set together") + } + if _, err := c.Location(); err != nil { + return err + } + return nil +} + +func (c Config) Location() (*time.Location, error) { + if c.Timezone == "" { + return time.Local, nil + } + location, err := time.LoadLocation(c.Timezone) + if err != nil { + return nil, fmt.Errorf("timezone %q is not known", c.Timezone) + } + return location, nil +} + +func (c Config) BaseURL() string { + return strings.TrimRight(c.Server, "/") +} + +func LoadConfig(path string) (Config, error) { + file, err := os.Open(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return Config{}, fmt.Errorf("no config at %s, run: devplace-node login --server URL --secret dpn_...", path) + } + return Config{}, err + } + defer file.Close() + config := DefaultConfig() + decoder := json.NewDecoder(&limitedReader{r: file, left: maxConfigBytes}) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&config); err != nil { + return Config{}, fmt.Errorf("config %s is invalid: %w", path, err) + } + if err := config.Validate(); err != nil { + return Config{}, fmt.Errorf("config %s is invalid: %w", path, err) + } + return config, nil +} + +type limitedReader struct { + r *os.File + left int +} + +func (l *limitedReader) Read(p []byte) (int, error) { + if l.left <= 0 { + return 0, errors.New("file is too large") + } + if len(p) > l.left { + p = p[:l.left] + } + n, err := l.r.Read(p) + l.left -= n + return n, err +} + +func SaveConfig(path string, config Config) error { + if err := config.Validate(); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + data, err := json.MarshalIndent(config, "", " ") + if err != nil { + return err + } + return writeFileAtomic(path, append(data, '\n')) +} + +func writeFileAtomic(path string, data []byte) error { + temp, err := os.CreateTemp(filepath.Dir(path), ".tmp-*") + if err != nil { + return err + } + name := temp.Name() + if err := temp.Chmod(0o600); err != nil { + temp.Close() + os.Remove(name) + return err + } + if _, err := temp.Write(data); err != nil { + temp.Close() + os.Remove(name) + return err + } + if err := temp.Close(); err != nil { + os.Remove(name) + return err + } + if err := os.Rename(name, path); err != nil { + os.Remove(name) + return err + } + return nil +} diff --git a/nodes/client/config_test.go b/nodes/client/config_test.go new file mode 100644 index 00000000..156cec26 --- /dev/null +++ b/nodes/client/config_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func validConfig() Config { + config := DefaultConfig() + config.Server = "https://devplace.net" + config.Secret = "dpn_abcdefghijklmnop" + return config +} + +func TestConfigRoundTripShapeAndMode(t *testing.T) { + path := filepath.Join(t.TempDir(), "sub", "config.json") + if err := SaveConfig(path, validConfig()); err != nil { + t.Fatal(err) + } + raw, _ := os.ReadFile(path) + keys := map[string]any{} + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + for _, key := range []string{"server", "secret", "bandwidth_mb_day", "max_concurrent", "max_tunnels", "active_from", "active_to", "timezone"} { + if _, ok := keys[key]; !ok { + t.Errorf("missing key %s", key) + } + } + if len(keys) != 8 { + t.Errorf("unexpected keys %v", keys) + } + if runtime.GOOS != "windows" { + info, _ := os.Stat(path) + if info.Mode().Perm() != 0o600 { + t.Errorf("mode %v", info.Mode().Perm()) + } + } + loaded, err := LoadConfig(path) + if err != nil || loaded != validConfig() { + t.Fatalf("loaded %+v %v", loaded, err) + } +} + +func TestConfigValidationTable(t *testing.T) { + cases := map[string]func(*Config){ + "server scheme": func(c *Config) { c.Server = "ftp://x" }, + "server empty": func(c *Config) { c.Server = "" }, + "secret prefix": func(c *Config) { c.Secret = "abc_1234567890" }, + "secret short": func(c *Config) { c.Secret = "dpn_" }, + "bandwidth low": func(c *Config) { c.BandwidthMBDay = 0 }, + "bandwidth high": func(c *Config) { c.BandwidthMBDay = MaxDonorMBPerDay + 1 }, + "concurrency": func(c *Config) { c.MaxConcurrent = 99 }, + "tunnels": func(c *Config) { c.MaxTunnels = 0 }, + "hours format": func(c *Config) { c.ActiveFrom, c.ActiveTo = "9:00", "17:00" }, + "hours lone": func(c *Config) { c.ActiveFrom = "09:00" }, + "timezone": func(c *Config) { c.Timezone = "Mars/Base" }, + } + for name, mutate := range cases { + config := validConfig() + mutate(&config) + if config.Validate() == nil { + t.Errorf("%s accepted", name) + } + } + good := validConfig() + good.ActiveFrom, good.ActiveTo, good.Timezone = "22:00", "06:00", "Europe/Amsterdam" + if err := good.Validate(); err != nil { + t.Errorf("good config: %v", err) + } +} + +func TestLoadRejectsUnknownFieldsAndMissingFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.json") + if _, err := LoadConfig(path); err == nil || !strings.Contains(err.Error(), "login") { + t.Errorf("missing file: %v", err) + } + _ = os.WriteFile(path, []byte(`{"server":"https://x.y","secret":"dpn_abcdefghijk","surprise":1}`), 0o600) + if _, err := LoadConfig(path); err == nil { + t.Error("unknown field accepted") + } +} + +func TestCommandsVersionAndUsage(t *testing.T) { + var out, errOut strings.Builder + if code := execute([]string{"version"}, &out, &errOut); code != exitOK || !strings.Contains(out.String(), Version) { + t.Fatalf("version: %d %q", code, out.String()) + } + if code := execute(nil, &out, &errOut); code != exitUsage { + t.Fatalf("no args: %d", code) + } + if code := execute([]string{"bogus"}, &out, &errOut); code != exitUsage { + t.Fatalf("bogus: %d", code) + } +} + +func TestLoginStoresWithoutVerifyAndNeverEchoesSecret(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + var out, errOut strings.Builder + args := []string{"--config", path, "--server", "https://devplace.net", "--secret", "dpn_abcdefghijklmnop", "--no-verify"} + if code := execute(append([]string{"login"}, args...), &out, &errOut); code != exitOK { + t.Fatalf("login: %d %s", code, errOut.String()) + } + if strings.Contains(out.String()+errOut.String(), "dpn_abcdefghijklmnop") { + t.Fatal("secret was printed") + } + out.Reset() + if code := execute([]string{"login", "--config", path, "--server", "nope", "--secret", "x"}, &out, &errOut); code != exitUsage { + t.Fatalf("bad login accepted: %d", code) + } +} diff --git a/nodes/client/fetcher.go b/nodes/client/fetcher.go new file mode 100644 index 00000000..176a71f8 --- /dev/null +++ b/nodes/client/fetcher.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "time" +) + +var hopByHop = map[string]bool{ + "connection": true, "keep-alive": true, "proxy-authenticate": true, "proxy-authorization": true, + "proxy-connection": true, "te": true, "trailer": true, "trailers": true, + "transfer-encoding": true, "upgrade": true, "content-length": true, "host": true, +} + +var identityHeaders = map[string]bool{ + "cookie": true, "cookie2": true, "authorization": true, "set-cookie": true, "set-cookie2": true, +} + +type FetchResult struct { + Status int + Headers map[string]string + Body []byte + Truncated bool + Elapsed time.Duration +} + +type Fetcher struct { + policy *Policy + client *http.Client +} + +func NewFetcher(policy *Policy) *Fetcher { + transport := &http.Transport{ + Proxy: nil, + DialContext: policy.DialContext, + ForceAttemptHTTP2: true, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 20 * time.Second, + MaxResponseHeaderBytes: 64 * 1024, + MaxIdleConns: 4, + MaxIdleConnsPerHost: 2, + IdleConnTimeout: 30 * time.Second, + } + client := &http.Client{ + Transport: transport, + Jar: nil, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > MaxRedirects { + return refuse(RefuseRedirect, "too many redirects") + } + _, err := policy.CheckRedirect(req.URL.String(), req.Method) + return err + }, + } + return &Fetcher{policy: policy, client: client} +} + +func requestHeaders(source map[string]any) http.Header { + header := http.Header{} + count := 0 + for name, raw := range source { + value, ok := raw.(string) + lower := strings.ToLower(strings.TrimSpace(name)) + if !ok || lower == "" || hopByHop[lower] || identityHeaders[lower] || len(value) > MaxHeaderValueBytes { + continue + } + if strings.ContainsAny(value, "\r\n") || strings.ContainsAny(lower, "\r\n: ") { + continue + } + count++ + if count > MaxRequestHeaders { + break + } + header.Set(lower, value) + } + return header +} + +func responseHeaders(source http.Header) map[string]string { + out := map[string]string{} + for name, values := range source { + lower := strings.ToLower(name) + if hopByHop[lower] || identityHeaders[lower] || len(out) >= MaxResponseHeaders { + continue + } + joined := strings.Join(values, ", ") + if len(joined) > MaxHeaderValueBytes { + joined = joined[:MaxHeaderValueBytes] + } + out[lower] = joined + } + return out +} + +func ErrorCode(err error) string { + var policyErr *PolicyError + if errors.As(err, &policyErr) { + return policyErr.Code + } + if errors.Is(err, context.DeadlineExceeded) { + return RefuseTimeout + } + return FailFetch +} + +func clampTimeout(seconds int) time.Duration { + if seconds < 1 { + seconds = DefaultJobTimeout + } + if seconds > MaxJobTimeout { + seconds = MaxJobTimeout + } + return time.Duration(seconds) * time.Second +} + +func (f *Fetcher) Do(ctx context.Context, frame Frame, maxBody int, budget int64) (*FetchResult, error) { + target, err := f.policy.Check(frame.URL, frame.Method, "fetch") + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, clampTimeout(frame.Timeout)) + defer cancel() + started := time.Now() + request, err := http.NewRequestWithContext(ctx, target.Method, target.URL, nil) + if err != nil { + return nil, refuse(RefuseBadURL, "request cannot be built") + } + request.Header = requestHeaders(frame.Headers) + response, err := f.client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + allowed := int64(maxBody) + if budget < allowed { + allowed = budget + } + body, err := io.ReadAll(io.LimitReader(response.Body, allowed+1)) + if err != nil { + return nil, err + } + truncated := int64(len(body)) > allowed + if truncated { + body = body[:allowed] + } + return &FetchResult{ + Status: response.StatusCode, + Headers: responseHeaders(response.Header), + Body: body, + Truncated: truncated, + Elapsed: time.Since(started), + }, nil +} diff --git a/nodes/client/fetcher_test.go b/nodes/client/fetcher_test.go new file mode 100644 index 00000000..06915dbd --- /dev/null +++ b/nodes/client/fetcher_test.go @@ -0,0 +1,198 @@ +package main + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "sync" + "testing" + "time" +) + +type fetchRig struct { + policy *Policy + f *Fetcher + mu sync.Mutex + dialed []string +} + +func newFetchRig(t *testing.T, handler http.Handler) *fetchRig { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + rig := &fetchRig{} + rig.policy = testPolicy(publicAnswer) + rig.policy.dial = func(ctx context.Context, network, address string) (net.Conn, error) { + rig.mu.Lock() + rig.dialed = append(rig.dialed, address) + rig.mu.Unlock() + var dialer net.Dialer + return dialer.DialContext(ctx, "tcp", server.Listener.Addr().String()) + } + rig.f = NewFetcher(rig.policy) + return rig +} + +func job(url string) Frame { + return Frame{Type: "job", JobID: "j1", Method: "GET", URL: url, Timeout: 5} +} + +func TestFetchSuccessStripsIdentityAndHopByHop(t *testing.T) { + var seen http.Header + rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = r.Header.Clone() + w.Header().Set("Set-Cookie", "session=abc") + w.Header().Set("X-Thing", "kept") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + frame := job("http://www.reddit.com/r/go.json") + frame.Headers = map[string]any{ + "Cookie": "a=b", "Authorization": "Bearer x", "Proxy-Authorization": "y", "Connection": "close", + "Accept": "application/json", "X-Bad": "a\r\nb", "Host": "evil.com", "X-Number": 5, + } + result, err := rig.f.Do(context.Background(), frame, 1000, 1<<20) + if err != nil { + t.Fatal(err) + } + if result.Status != 200 || string(result.Body) != `{"ok":true}` || result.Truncated { + t.Fatalf("result %+v", result) + } + for _, name := range []string{"Cookie", "Authorization", "Proxy-Authorization", "X-Bad", "X-Number"} { + if seen.Get(name) != "" { + t.Errorf("%s was forwarded", name) + } + } + if seen.Get("Accept") != "application/json" { + t.Error("allowed header dropped") + } + if _, ok := result.Headers["set-cookie"]; ok { + t.Error("set-cookie reported") + } + if result.Headers["x-thing"] != "kept" { + t.Errorf("headers %v", result.Headers) + } + if len(rig.dialed) != 1 || rig.dialed[0] != "93.184.216.34:80" { + t.Errorf("dialed %v", rig.dialed) + } +} + +func TestFetchTruncatesAtBodyCapAndBudget(t *testing.T) { + rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(strings.Repeat("x", 5000))) + })) + result, err := rig.f.Do(context.Background(), job("http://reddit.com/a"), 1000, 1<<20) + if err != nil || len(result.Body) != 1000 || !result.Truncated { + t.Fatalf("cap: %v %+v", err, result) + } + result, err = rig.f.Do(context.Background(), job("http://reddit.com/a"), 1000, 300) + if err != nil || len(result.Body) != 300 || !result.Truncated { + t.Fatalf("budget: %v %+v", err, result) + } + result, err = rig.f.Do(context.Background(), job("http://reddit.com/a"), 10000, 1<<20) + if err != nil || len(result.Body) != 5000 || result.Truncated { + t.Fatalf("whole: %v %+v", err, result) + } +} + +func TestFetchRefusals(t *testing.T) { + rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("ok")) + })) + cases := []struct { + name string + frame Frame + prep func() + expect string + }{ + {"host", job("http://evilreddit.com/"), nil, RefuseHost}, + {"port", job("http://reddit.com:8080/"), nil, RefusePort}, + {"method", Frame{JobID: "j", Method: "POST", URL: "http://reddit.com/", Timeout: 5}, nil, RefuseMethod}, + {"literal", job("http://10.0.0.1/"), nil, RefuseHost}, + {"private resolve", job("http://reddit.com/"), func() { + rig.policy.lookup = func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("192.168.1.10")}, nil + } + }, RefuseAddress}, + } + for _, item := range cases { + if item.prep != nil { + item.prep() + } + before := len(rig.dialed) + _, err := rig.f.Do(context.Background(), item.frame, 1000, 1<<20) + if got := ErrorCode(err); got != item.expect { + t.Errorf("%s: got %q (%v)", item.name, got, err) + } + if len(rig.dialed) != before { + t.Errorf("%s: a connection was attempted", item.name) + } + } +} + +func TestFetchRedirects(t *testing.T) { + var hits []string + rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits = append(hits, r.Host+r.URL.Path) + switch r.URL.Path { + case "/to-allowed": + http.Redirect(w, r, "http://old.reddit.com/final", http.StatusFound) + case "/to-evil": + http.Redirect(w, r, "http://evil.example/steal", http.StatusFound) + case "/to-private": + http.Redirect(w, r, "http://127.0.0.1/admin", http.StatusFound) + case "/to-port": + http.Redirect(w, r, "http://reddit.com:8080/x", http.StatusFound) + case "/loop": + http.Redirect(w, r, "/loop", http.StatusFound) + default: + _, _ = w.Write([]byte("final")) + } + })) + result, err := rig.f.Do(context.Background(), job("http://reddit.com/to-allowed"), 1000, 1<<20) + if err != nil || string(result.Body) != "final" { + t.Fatalf("allowed redirect: %v", err) + } + for _, path := range []string{"/to-evil", "/to-private", "/to-port", "/loop"} { + _, err := rig.f.Do(context.Background(), job("http://reddit.com"+path), 1000, 1<<20) + if ErrorCode(err) != RefuseRedirect { + t.Errorf("%s: got %q (%v)", path, ErrorCode(err), err) + } + } + for _, hit := range hits { + if strings.HasPrefix(hit, "evil.example") || strings.HasPrefix(hit, "127.0.0.1") { + t.Errorf("request reached %s", hit) + } + } +} + +func TestFetchTimeout(t *testing.T) { + rig := newFetchRig(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-time.After(3 * time.Second): + } + })) + frame := job("http://reddit.com/slow") + frame.Timeout = 1 + started := time.Now() + _, err := rig.f.Do(context.Background(), frame, 1000, 1<<20) + if ErrorCode(err) != RefuseTimeout { + t.Fatalf("got %q (%v)", ErrorCode(err), err) + } + if time.Since(started) > 2500*time.Millisecond { + t.Fatal("timeout was not enforced") + } +} + +func TestClampTimeout(t *testing.T) { + cases := map[int]time.Duration{0: 25 * time.Second, -3: 25 * time.Second, 7: 7 * time.Second, 500: 60 * time.Second} + for in, want := range cases { + if got := clampTimeout(in); got != want { + t.Errorf("%d: %v", in, got) + } + } +} diff --git a/nodes/client/go.mod b/nodes/client/go.mod new file mode 100644 index 00000000..ac6ac2a5 --- /dev/null +++ b/nodes/client/go.mod @@ -0,0 +1,3 @@ +module devplace-node + +go 1.22 diff --git a/nodes/client/limits.go b/nodes/client/limits.go new file mode 100644 index 00000000..3256b483 --- /dev/null +++ b/nodes/client/limits.go @@ -0,0 +1,238 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "sync" + "time" +) + +const ( + mebibyte = 1024 * 1024 + persistInterval = 5 * time.Second +) + +type usageState struct { + Day string `json:"day"` + Bytes int64 `json:"bytes"` +} + +type Limiter struct { + mu sync.Mutex + cfg Config + loc *time.Location + now func() time.Time + statePath string + serverLimit ServerLimits + serverSet ServerSettings + hasServer bool + day string + used int64 + active int + lastPersist time.Time +} + +func NewLimiter(cfg Config, statePath string, now func() time.Time) (*Limiter, error) { + loc, err := cfg.Location() + if err != nil { + return nil, err + } + limiter := &Limiter{cfg: cfg, loc: loc, now: now, statePath: statePath} + limiter.load() + return limiter, nil +} + +func (l *Limiter) load() { + if l.statePath == "" { + return + } + data, err := os.ReadFile(l.statePath) + if err != nil || len(data) > 4096 { + return + } + var state usageState + if json.Unmarshal(data, &state) != nil || state.Bytes < 0 { + return + } + l.day, l.used = state.Day, state.Bytes +} + +func (l *Limiter) persist(force bool) { + if l.statePath == "" { + return + } + moment := l.now() + if !force && moment.Sub(l.lastPersist) < persistInterval { + return + } + l.lastPersist = moment + data, err := json.Marshal(usageState{Day: l.day, Bytes: l.used}) + if err != nil { + return + } + if err := os.MkdirAll(filepath.Dir(l.statePath), 0o700); err != nil { + return + } + _ = writeFileAtomic(l.statePath, data) +} + +func (l *Limiter) Flush() { + l.mu.Lock() + defer l.mu.Unlock() + l.roll() + l.persist(true) +} + +func (l *Limiter) ApplyServer(limits *ServerLimits, settings *ServerSettings) { + l.mu.Lock() + defer l.mu.Unlock() + if limits != nil { + l.serverLimit = *limits + } + if settings != nil { + l.serverSet = *settings + } + l.hasServer = limits != nil || settings != nil +} + +func (l *Limiter) roll() { + today := l.now().In(l.loc).Format("2006-01-02") + if l.day != today { + l.day = today + l.used = 0 + } +} + +func minPositive(values ...int) int { + best := 0 + for _, value := range values { + if value > 0 && (best == 0 || value < best) { + best = value + } + } + return best +} + +func (l *Limiter) capBytes() int64 { + mb := minPositive(l.cfg.BandwidthMBDay, l.serverSet.BandwidthMBDay, l.serverLimit.MaxDonorMBPerDay) + return int64(mb) * mebibyte +} + +func (l *Limiter) concurrencyCap() int { + return minPositive(l.cfg.MaxConcurrent, l.serverSet.MaxConcurrent, l.serverLimit.MaxConcurrentPerNode) +} + +func minutesOf(text string) int { + var hours, minutes int + if len(text) != 5 { + return -1 + } + hours = int(text[0]-'0')*10 + int(text[1]-'0') + minutes = int(text[3]-'0')*10 + int(text[4]-'0') + return hours*60 + minutes +} + +func InWindow(from, to string, moment time.Time) bool { + if from == "" || to == "" || from == to { + return true + } + start, end := minutesOf(from), minutesOf(to) + if start < 0 || end < 0 { + return true + } + now := moment.Hour()*60 + moment.Minute() + if start < end { + return now >= start && now < end + } + return now >= start || now < end +} + +func (l *Limiter) hoursCode() string { + moment := l.now() + if !InWindow(l.cfg.ActiveFrom, l.cfg.ActiveTo, moment.In(l.loc)) { + return RefuseHours + } + if l.serverSet.ActiveFrom != "" { + zone, err := time.LoadLocation(l.serverSet.Timezone) + if err != nil { + zone = time.UTC + } + if !InWindow(l.serverSet.ActiveFrom, l.serverSet.ActiveTo, moment.In(zone)) { + return RefuseHours + } + } + return "" +} + +func (l *Limiter) Acquire() string { + l.mu.Lock() + defer l.mu.Unlock() + l.roll() + if code := l.hoursCode(); code != "" { + return code + } + if l.used >= l.capBytes() { + return RefuseBandwidth + } + if l.active >= l.concurrencyCap() { + return RefuseConcurrency + } + l.active++ + return "" +} + +func (l *Limiter) Release() { + l.mu.Lock() + defer l.mu.Unlock() + if l.active > 0 { + l.active-- + } +} + +func (l *Limiter) Remaining() int64 { + l.mu.Lock() + defer l.mu.Unlock() + l.roll() + left := l.capBytes() - l.used + if left < 0 { + return 0 + } + return left +} + +func (l *Limiter) AddBytes(n int64) { + if n <= 0 { + return + } + l.mu.Lock() + defer l.mu.Unlock() + l.roll() + l.used += n + l.persist(false) +} + +func (l *Limiter) Usage() (day string, used, capacity int64) { + l.mu.Lock() + defer l.mu.Unlock() + l.roll() + return l.day, l.used, l.capBytes() +} + +func ReadUsage(path string, loc *time.Location, now time.Time) int64 { + data, err := os.ReadFile(path) + if err != nil || len(data) > 4096 { + return 0 + } + var state usageState + if json.Unmarshal(data, &state) != nil || state.Day != now.In(loc).Format("2006-01-02") { + return 0 + } + return state.Bytes +} + +func (l *Limiter) ConcurrencyCap() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.concurrencyCap() +} diff --git a/nodes/client/limits_test.go b/nodes/client/limits_test.go new file mode 100644 index 00000000..1f2264eb --- /dev/null +++ b/nodes/client/limits_test.go @@ -0,0 +1,162 @@ +package main + +import ( + "path/filepath" + "testing" + "time" +) + +type clock struct{ now time.Time } + +func (c *clock) Now() time.Time { return c.now } + +func testLimiter(t *testing.T, cfg Config, at string) (*Limiter, *clock) { + t.Helper() + moment, err := time.Parse(time.RFC3339, at) + if err != nil { + t.Fatal(err) + } + cfg.Timezone = "UTC" + c := &clock{now: moment} + limiter, err := NewLimiter(cfg, "", c.Now) + if err != nil { + t.Fatal(err) + } + return limiter, c +} + +func TestInWindowTable(t *testing.T) { + at := func(text string) time.Time { + moment, _ := time.Parse("15:04", text) + return moment + } + cases := []struct { + from, to, now string + want bool + }{ + {"", "", "12:00", true}, + {"09:00", "17:00", "09:00", true}, + {"09:00", "17:00", "16:59", true}, + {"09:00", "17:00", "17:00", false}, + {"09:00", "17:00", "08:59", false}, + {"22:00", "06:00", "23:30", true}, + {"22:00", "06:00", "05:59", true}, + {"22:00", "06:00", "06:00", false}, + {"22:00", "06:00", "12:00", false}, + {"10:00", "10:00", "03:00", true}, + } + for _, item := range cases { + if got := InWindow(item.from, item.to, at(item.now)); got != item.want { + t.Errorf("%s-%s at %s: got %v", item.from, item.to, item.now, got) + } + } +} + +func TestAcquireConcurrencyCapAndServerCannotWiden(t *testing.T) { + cfg := DefaultConfig() + cfg.MaxConcurrent = 2 + limiter, _ := testLimiter(t, cfg, "2026-10-15T12:00:00Z") + limiter.ApplyServer(&ServerLimits{MaxConcurrentPerNode: 8}, &ServerSettings{MaxConcurrent: 8}) + if limiter.Acquire() != "" || limiter.Acquire() != "" { + t.Fatal("two slots should be free") + } + if got := limiter.Acquire(); got != RefuseConcurrency { + t.Fatalf("third acquire: %q", got) + } + limiter.Release() + if got := limiter.Acquire(); got != "" { + t.Fatalf("after release: %q", got) + } +} + +func TestServerCanNarrowConcurrency(t *testing.T) { + cfg := DefaultConfig() + cfg.MaxConcurrent = 4 + limiter, _ := testLimiter(t, cfg, "2026-10-15T12:00:00Z") + limiter.ApplyServer(&ServerLimits{MaxConcurrentPerNode: 2}, &ServerSettings{MaxConcurrent: 1}) + if limiter.Acquire() != "" { + t.Fatal("first slot") + } + if got := limiter.Acquire(); got != RefuseConcurrency { + t.Fatalf("got %q", got) + } +} + +func TestBandwidthCapLocalAndDayRollover(t *testing.T) { + cfg := DefaultConfig() + cfg.BandwidthMBDay = 1 + limiter, c := testLimiter(t, cfg, "2026-10-15T23:00:00Z") + limiter.ApplyServer(&ServerLimits{MaxDonorMBPerDay: 20480}, &ServerSettings{BandwidthMBDay: 9999}) + if limiter.Acquire() != "" { + t.Fatal("fresh day must allow") + } + limiter.Release() + limiter.AddBytes(mebibyte - 10) + if got := limiter.Remaining(); got != 10 { + t.Fatalf("remaining %d", got) + } + limiter.AddBytes(10) + if got := limiter.Acquire(); got != RefuseBandwidth { + t.Fatalf("got %q", got) + } + c.now = c.now.Add(2 * time.Hour) + if got := limiter.Acquire(); got != "" { + t.Fatalf("next day: %q", got) + } +} + +func TestServerCanNarrowBandwidth(t *testing.T) { + cfg := DefaultConfig() + cfg.BandwidthMBDay = 500 + limiter, _ := testLimiter(t, cfg, "2026-10-15T12:00:00Z") + limiter.ApplyServer(nil, &ServerSettings{BandwidthMBDay: 2}) + if got := limiter.Remaining(); got != 2*mebibyte { + t.Fatalf("remaining %d", got) + } +} + +func TestActiveHoursLocalAndServer(t *testing.T) { + cfg := DefaultConfig() + cfg.ActiveFrom, cfg.ActiveTo = "22:00", "06:00" + limiter, c := testLimiter(t, cfg, "2026-10-15T12:00:00Z") + if got := limiter.Acquire(); got != RefuseHours { + t.Fatalf("noon: %q", got) + } + c.now = c.now.Add(11 * time.Hour) + if got := limiter.Acquire(); got != "" { + t.Fatalf("23:00: %q", got) + } + limiter.Release() + limiter.ApplyServer(nil, &ServerSettings{ActiveFrom: "01:00", ActiveTo: "02:00", Timezone: "UTC"}) + if got := limiter.Acquire(); got != RefuseHours { + t.Fatalf("server window: %q", got) + } +} + +func TestUsagePersistsAcrossRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "usage.json") + cfg := DefaultConfig() + cfg.Timezone = "UTC" + moment := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + c := &clock{now: moment} + first, err := NewLimiter(cfg, path, c.Now) + if err != nil { + t.Fatal(err) + } + first.AddBytes(12345) + first.Flush() + second, err := NewLimiter(cfg, path, c.Now) + if err != nil { + t.Fatal(err) + } + if _, used, _ := second.Usage(); used != 12345 { + t.Fatalf("used %d", used) + } + c.now = moment.Add(24 * time.Hour) + if _, used, _ := second.Usage(); used != 0 { + t.Fatalf("next day used %d", used) + } + if got := ReadUsage(path, time.UTC, moment); got != 12345 { + t.Fatalf("read usage %d", got) + } +} diff --git a/nodes/client/logger.go b/nodes/client/logger.go new file mode 100644 index 00000000..afe0a3ff --- /dev/null +++ b/nodes/client/logger.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "io" + "log" +) + +type Logger struct { + out *log.Logger + verbose bool +} + +func NewLogger(w io.Writer, verbose bool) *Logger { + return &Logger{out: log.New(w, "devplace-node ", log.LstdFlags|log.LUTC), verbose: verbose} +} + +func (l *Logger) Printf(format string, args ...any) { + l.out.Output(2, fmt.Sprintf(format, args...)) +} + +func (l *Logger) Debugf(format string, args ...any) { + if l.verbose { + l.out.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/nodes/client/main.go b/nodes/client/main.go new file mode 100644 index 00000000..93e4db33 --- /dev/null +++ b/nodes/client/main.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + _ "time/tzdata" +) + +var Version = "dev" + +const ( + exitOK = 0 + exitError = 1 + exitUsage = 2 + exitFatal = 3 + usageText = `usage: devplace-node [flags] + +commands: + login store the server and device secret (--server URL --secret dpn_...) + status show the stored configuration, today's usage and whether the server accepts this device + run connect and serve jobs in the foreground + version print the version + +flags for every command: --config PATH (default: /devplace-node/config.json) +` +) + +func main() { + os.Exit(execute(os.Args[1:], os.Stdout, os.Stderr)) +} + +func execute(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprint(stderr, usageText) + return exitUsage + } + command, rest := args[0], args[1:] + switch command { + case "version": + fmt.Fprintln(stdout, "devplace-node", Version) + return exitOK + case "login": + return runLogin(rest, stdout, stderr) + case "status": + return runStatus(rest, stdout, stderr) + case "run": + return runRun(rest, stderr) + case "help", "-h", "--help": + fmt.Fprint(stdout, usageText) + return exitOK + } + fmt.Fprintf(stderr, "unknown command %q\n%s", command, usageText) + return exitUsage +} + +func resolveConfigPath(flagValue string) (string, error) { + if flagValue != "" { + return flagValue, nil + } + return DefaultConfigPath() +} + +func newFlags(name string, stderr io.Writer) (*flag.FlagSet, *string) { + set := flag.NewFlagSet(name, flag.ContinueOnError) + set.SetOutput(stderr) + path := set.String("config", "", "config file path") + return set, path +} + +func runLogin(args []string, stdout, stderr io.Writer) int { + set, pathFlag := newFlags("login", stderr) + server := set.String("server", "", "server url, for example https://devplace.net") + secret := set.String("secret", "", "device secret (or set DEVPLACE_NODE_SECRET)") + bandwidth := set.Int("bandwidth-mb", DefaultBandwidthMB, "daily bandwidth cap in MB") + concurrent := set.Int("concurrency", DefaultMaxConcurrent, "maximum parallel jobs") + tunnels := set.Int("tunnels", DefaultMaxTunnels, "maximum parallel tunnels") + from := set.String("from", "", "active hours start HH:MM") + to := set.String("to", "", "active hours end HH:MM") + zone := set.String("timezone", "", "IANA timezone for the active hours (default: system)") + skip := set.Bool("no-verify", false, "store without asking the server to accept the secret") + if err := set.Parse(args); err != nil { + return exitUsage + } + if *secret == "" { + *secret = os.Getenv("DEVPLACE_NODE_SECRET") + } + path, err := resolveConfigPath(*pathFlag) + if err != nil { + fmt.Fprintln(stderr, err) + return exitError + } + config := Config{ + Server: *server, Secret: *secret, BandwidthMBDay: *bandwidth, MaxConcurrent: *concurrent, + MaxTunnels: *tunnels, ActiveFrom: *from, ActiveTo: *to, Timezone: *zone, + } + if err := config.Validate(); err != nil { + fmt.Fprintln(stderr, "login refused:", err) + return exitUsage + } + if !*skip { + if code := verifyDevice(config, stderr); code != exitOK { + return code + } + } + if err := SaveConfig(path, config); err != nil { + fmt.Fprintln(stderr, "config could not be written:", err) + return exitError + } + fmt.Fprintf(stdout, "stored %s (mode 0600)\n", path) + return exitOK +} + +func verifyDevice(config Config, stderr io.Writer) int { + session := NewSession(config, NewPolicy(), nil, nil, nil, NewLogger(stderr, false)) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(TicketTimeoutSeconds+5)*time.Second) + defer cancel() + _, result := session.fetchTicket(ctx) + if result.fatal != "" { + fmt.Fprintln(stderr, "server refused this device:", result.fatal) + return exitFatal + } + if result.reason != "" { + fmt.Fprintln(stderr, "server not verified:", result.reason, "(use --no-verify to store anyway)") + return exitError + } + return exitOK +} + +func runStatus(args []string, stdout, stderr io.Writer) int { + set, pathFlag := newFlags("status", stderr) + if err := set.Parse(args); err != nil { + return exitUsage + } + path, err := resolveConfigPath(*pathFlag) + if err != nil { + fmt.Fprintln(stderr, err) + return exitError + } + config, err := LoadConfig(path) + if err != nil { + fmt.Fprintln(stderr, err) + return exitError + } + location, _ := config.Location() + used := ReadUsage(filepath.Join(filepath.Dir(path), UsageFileName), location, time.Now()) + fmt.Fprintf(stdout, "version: %s\nconfig: %s\nserver: %s\nsecret: stored\n", Version, path, config.Server) + fmt.Fprintf(stdout, "bandwidth: %d MB/day (used today: %.1f MB)\n", config.BandwidthMBDay, float64(used)/mebibyte) + fmt.Fprintf(stdout, "concurrency: %d jobs, %d tunnels\n", config.MaxConcurrent, config.MaxTunnels) + hours := "always" + if config.ActiveFrom != "" { + hours = config.ActiveFrom + "-" + config.ActiveTo + } + fmt.Fprintf(stdout, "active hours: %s (%s)\n", hours, location) + if code := verifyDevice(config, stderr); code != exitOK { + fmt.Fprintln(stdout, "server: not accepting this device") + return code + } + fmt.Fprintln(stdout, "server: accepts this device") + return exitOK +} + +func runRun(args []string, stderr io.Writer) int { + set, pathFlag := newFlags("run", stderr) + verbose := set.Bool("v", false, "verbose logging") + if err := set.Parse(args); err != nil { + return exitUsage + } + path, err := resolveConfigPath(*pathFlag) + if err != nil { + fmt.Fprintln(stderr, err) + return exitError + } + config, err := LoadConfig(path) + if err != nil { + fmt.Fprintln(stderr, err) + return exitError + } + logger := NewLogger(stderr, *verbose) + limiter, err := NewLimiter(config, filepath.Join(filepath.Dir(path), UsageFileName), time.Now) + if err != nil { + fmt.Fprintln(stderr, err) + return exitError + } + policy := NewPolicy() + stats := NewStats(time.Now()) + session := NewSession(config, policy, limiter, NewFetcher(policy), stats, logger) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + logger.Printf("started version %s, server %s", Version, config.Server) + err = session.Run(ctx) + limiter.Flush() + logger.Printf("%s", stats.Line()) + var fatal *FatalError + if errors.As(err, &fatal) { + logger.Printf("%s", fatal.Error()) + return exitFatal + } + if err != nil { + logger.Printf("stopped: %v", err) + return exitError + } + return exitOK +} diff --git a/nodes/client/packaging/compose.yaml b/nodes/client/packaging/compose.yaml new file mode 100644 index 00000000..1b94ad2a --- /dev/null +++ b/nodes/client/packaging/compose.yaml @@ -0,0 +1,15 @@ +services: + devplace-node: + image: gcr.io/distroless/static-debian12:nonroot + command: ["/devplace-node", "run", "--config", "/config/config.json"] + volumes: + - ./devplace-node-linux-amd64:/devplace-node:ro + - ./config:/config + restart: on-failure + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + mem_limit: 128m + pids_limit: 64 diff --git a/nodes/client/packaging/devplace-node.service b/nodes/client/packaging/devplace-node.service new file mode 100644 index 00000000..0d154ee0 --- /dev/null +++ b/nodes/client/packaging/devplace-node.service @@ -0,0 +1,16 @@ +[Unit] +Description=DevPlace community node +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=%h/.local/bin/devplace-node run +Restart=on-failure +RestartSec=30 +RestartPreventExitStatus=3 +Nice=10 +NoNewPrivileges=true + +[Install] +WantedBy=default.target diff --git a/nodes/client/packaging/nl.devplace.node.plist b/nodes/client/packaging/nl.devplace.node.plist new file mode 100644 index 00000000..1c760e3d --- /dev/null +++ b/nodes/client/packaging/nl.devplace.node.plist @@ -0,0 +1,32 @@ + + + + + Label + nl.devplace.node + ProgramArguments + + /usr/local/bin/devplace-node + run + + RunAtLoad + + KeepAlive + + SuccessfulExit + + + ThrottleInterval + 60 + ProcessType + Background + LowPriorityIO + + Nice + 10 + StandardErrorPath + /tmp/devplace-node.log + StandardOutPath + /dev/null + + diff --git a/nodes/client/packaging/windows-install.ps1 b/nodes/client/packaging/windows-install.ps1 new file mode 100644 index 00000000..5f6c1e45 --- /dev/null +++ b/nodes/client/packaging/windows-install.ps1 @@ -0,0 +1,34 @@ +param( + [string]$Binary = "$env:LOCALAPPDATA\devplace-node\devplace-node.exe", + [switch]$Remove +) + +$ErrorActionPreference = "Stop" +$TaskName = "DevPlaceNode" + +if ($Remove) { + Unregister-ScheduledTask -TaskName $TaskName -Confirm:$false + Write-Host "Removed scheduled task $TaskName" + exit 0 +} + +if (-not (Test-Path $Binary)) { + throw "devplace-node.exe not found at $Binary. Download it there first." +} + +$action = New-ScheduledTaskAction -Execute $Binary -Argument "run" +$trigger = New-ScheduledTaskTrigger -AtLogOn -User $env:USERNAME +$settings = New-ScheduledTaskSettingsSet ` + -AllowStartIfOnBatteries ` + -DontStopIfGoingOnBatteries ` + -RestartCount 3 ` + -RestartInterval (New-TimeSpan -Minutes 5) ` + -ExecutionTimeLimit ([TimeSpan]::Zero) ` + -StartWhenAvailable +$principal = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive -RunLevel Limited + +Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Settings $settings -Principal $principal -Force | Out-Null +Write-Host "Registered scheduled task $TaskName. It starts at your next logon." +Write-Host "Store the device secret first: devplace-node.exe login --server https://devplace.net --secret dpn_..." +Write-Host "Start now: Start-ScheduledTask -TaskName $TaskName" +Write-Host "Stop and remove: .\windows-install.ps1 -Remove" diff --git a/nodes/client/policy.go b/nodes/client/policy.go new file mode 100644 index 00000000..806d58b3 --- /dev/null +++ b/nodes/client/policy.go @@ -0,0 +1,317 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "net/url" + "slices" + "strconv" + "strings" + "sync" + "time" +) + +type PolicyError struct { + Code string + Reason string +} + +func (e *PolicyError) Error() string { + if e.Reason == "" { + return e.Code + } + return e.Code + ": " + e.Reason +} + +func refuse(code, reason string) error { + return &PolicyError{Code: code, Reason: reason} +} + +type Target struct { + URL string + Method string + Host string + Port int + Dest Destination +} + +type Policy struct { + mu sync.RWMutex + dests []Destination + lookup func(ctx context.Context, host string) ([]netip.Addr, error) + public func(netip.Addr) bool + dial func(ctx context.Context, network, address string) (net.Conn, error) +} + +func NewPolicy() *Policy { + dialer := &net.Dialer{Timeout: time.Duration(TunnelDialSeconds) * time.Second} + return &Policy{ + dests: cloneDestinations(BakedDestinations), + lookup: systemLookup, + public: IsPublicAddr, + dial: dialer.DialContext, + } +} + +func systemLookup(ctx context.Context, host string) ([]netip.Addr, error) { + return net.DefaultResolver.LookupNetIP(ctx, "ip", host) +} + +func cloneDestinations(list []Destination) []Destination { + out := make([]Destination, len(list)) + for i, item := range list { + item.Methods = slices.Clone(item.Methods) + item.Ports = slices.Clone(item.Ports) + item.Modes = slices.Clone(item.Modes) + out[i] = item + } + return out +} + +func normalizeHost(host string) string { + return strings.TrimRight(strings.ToLower(strings.TrimSpace(host)), ".") +} + +func suffixMatches(suffix, host string) bool { + return host == suffix || strings.HasSuffix(host, "."+suffix) +} + +func intersectStrings(a, b []string) []string { + out := []string{} + for _, item := range a { + if slices.ContainsFunc(b, func(other string) bool { return strings.EqualFold(item, other) }) { + out = append(out, item) + } + } + return out +} + +func intersectPorts(a, b []int) []int { + out := []int{} + for _, item := range a { + if slices.Contains(b, item) && slices.Contains(AllowedPorts, item) { + out = append(out, item) + } + } + return out +} + +func (p *Policy) Refine(offered []Destination) []string { + var rejected []string + narrowed := []Destination{} + seen := map[string]bool{} + for _, item := range offered { + suffix := normalizeHost(item.Suffix) + var base *Destination + for i := range BakedDestinations { + if suffixMatches(BakedDestinations[i].Suffix, suffix) { + base = &BakedDestinations[i] + break + } + } + if base == nil || seen[suffix] { + rejected = append(rejected, item.Suffix) + continue + } + seen[suffix] = true + narrowed = append(narrowed, Destination{ + Suffix: suffix, + Adapter: base.Adapter, + Methods: intersectStrings(base.Methods, item.Methods), + Ports: intersectPorts(base.Ports, item.Ports), + Modes: intersectStrings(base.Modes, item.Modes), + MaxRPS: item.MaxRPS, + }) + } + p.mu.Lock() + p.dests = narrowed + p.mu.Unlock() + return rejected +} + +func (p *Policy) Destinations() []Destination { + p.mu.RLock() + defer p.mu.RUnlock() + return cloneDestinations(p.dests) +} + +func (p *Policy) destinationFor(host string) (Destination, bool) { + p.mu.RLock() + defer p.mu.RUnlock() + var best Destination + found := false + for _, item := range p.dests { + if suffixMatches(item.Suffix, host) && (!found || len(item.Suffix) > len(best.Suffix)) { + best, found = item, true + } + } + return best, found +} + +func isAddressLiteral(host string) bool { + _, err := netip.ParseAddr(strings.Trim(host, "[]")) + return err == nil +} + +func (p *Policy) CheckHostPort(host string, port int, mode string) (Destination, error) { + host = normalizeHost(host) + if host == "" { + return Destination{}, refuse(RefuseBadURL, "no host") + } + if isAddressLiteral(host) { + return Destination{}, refuse(RefuseHost, "address literals are not allowed") + } + dest, ok := p.destinationFor(host) + if !ok { + return Destination{}, refuse(RefuseHost, "destination is not on the allowlist") + } + if !slices.Contains(AllowedPorts, port) || !slices.Contains(dest.Ports, port) { + return Destination{}, refuse(RefusePort, "port is not allowed") + } + if mode != "" && mode != "any" && !slices.Contains(dest.Modes, mode) { + return Destination{}, refuse(RefuseHost, "mode is not allowed for this destination") + } + return dest, nil +} + +func (p *Policy) Check(rawURL, method, mode string) (Target, error) { + if rawURL == "" || len(rawURL) > MaxURLChars { + return Target{}, refuse(RefuseBadURL, "url is missing or too long") + } + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return Target{}, refuse(RefuseBadURL, "url cannot be parsed") + } + defaultPort := 0 + switch parsed.Scheme { + case "http": + defaultPort = 80 + case "https": + defaultPort = 443 + default: + return Target{}, refuse(RefuseBadURL, "only http and https are allowed") + } + if parsed.User != nil { + return Target{}, refuse(RefuseBadURL, "credentials in a url are not allowed") + } + port := defaultPort + if text := parsed.Port(); text != "" { + port, err = strconv.Atoi(text) + if err != nil { + return Target{}, refuse(RefuseBadURL, "port is not a number") + } + } + dest, err := p.CheckHostPort(parsed.Hostname(), port, mode) + if err != nil { + return Target{}, err + } + verb := strings.ToUpper(strings.TrimSpace(method)) + if !slices.Contains(upperAll(dest.Methods), verb) { + return Target{}, refuse(RefuseMethod, "method is not allowed here") + } + return Target{URL: parsed.String(), Method: verb, Host: normalizeHost(parsed.Hostname()), Port: port, Dest: dest}, nil +} + +func upperAll(list []string) []string { + out := make([]string, len(list)) + for i, item := range list { + out[i] = strings.ToUpper(item) + } + return out +} + +func (p *Policy) CheckRedirect(location, method string) (Target, error) { + target, err := p.Check(location, method, "fetch") + if err != nil { + var policyErr *PolicyError + if errors.As(err, &policyErr) { + return Target{}, refuse(RefuseRedirect, policyErr.Reason) + } + return Target{}, refuse(RefuseRedirect, err.Error()) + } + return target, nil +} + +func (p *Policy) Resolve(ctx context.Context, host string) ([]netip.Addr, error) { + host = normalizeHost(host) + addrs, err := p.lookup(ctx, host) + if err != nil { + return nil, refuse(FailDNS, "host did not resolve") + } + if len(addrs) == 0 { + return nil, refuse(FailDNS, "host has no addresses") + } + if len(addrs) > 16 { + addrs = addrs[:16] + } + for _, addr := range addrs { + if !p.public(addr.Unmap()) { + return nil, refuse(RefuseAddress, "host resolves to a non public address") + } + } + return addrs, nil +} + +func (p *Policy) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + host, portText, err := net.SplitHostPort(address) + if err != nil { + return nil, refuse(RefuseBadURL, "bad address") + } + port, err := strconv.Atoi(portText) + if err != nil { + return nil, refuse(RefuseBadURL, "bad port") + } + if _, err := p.CheckHostPort(host, port, ""); err != nil { + return nil, err + } + addrs, err := p.Resolve(ctx, host) + if err != nil { + return nil, err + } + var last error + for _, addr := range addrs { + conn, err := p.dial(ctx, network, net.JoinHostPort(addr.Unmap().String(), portText)) + if err == nil { + return conn, nil + } + last = err + } + return nil, fmt.Errorf("connect failed: %w", last) +} + +var blockedPrefixes = mustPrefixes( + "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", + "172.16.0.0/12", "192.0.0.0/24", "192.0.2.0/24", "192.88.99.0/24", "192.168.0.0/16", + "198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24", "224.0.0.0/4", "240.0.0.0/4", + "::/128", "::1/128", "::ffff:0:0/96", "64:ff9b::/96", "64:ff9b:1::/48", "100::/64", + "2001::/32", "2001:db8::/32", "2002::/16", "fc00::/7", "fe80::/10", "ff00::/8", +) + +func mustPrefixes(list ...string) []netip.Prefix { + out := make([]netip.Prefix, len(list)) + for i, item := range list { + out[i] = netip.MustParsePrefix(item) + } + return out +} + +func IsPublicAddr(addr netip.Addr) bool { + addr = addr.Unmap() + if !addr.IsValid() || addr.Zone() != "" { + return false + } + if addr.IsUnspecified() || addr.IsLoopback() || addr.IsPrivate() || addr.IsMulticast() || + addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() || addr.IsInterfaceLocalMulticast() || + !addr.IsGlobalUnicast() { + return false + } + for _, prefix := range blockedPrefixes { + if prefix.Contains(addr) { + return false + } + } + return true +} diff --git a/nodes/client/policy_test.go b/nodes/client/policy_test.go new file mode 100644 index 00000000..14f3820a --- /dev/null +++ b/nodes/client/policy_test.go @@ -0,0 +1,233 @@ +package main + +import ( + "context" + "errors" + "net" + "net/netip" + "testing" +) + +func testPolicy(answer func(host string) []netip.Addr) *Policy { + policy := NewPolicy() + policy.lookup = func(ctx context.Context, host string) ([]netip.Addr, error) { + return answer(host), nil + } + return policy +} + +func publicAnswer(string) []netip.Addr { + return []netip.Addr{netip.MustParseAddr("93.184.216.34")} +} + +func codeOf(err error) string { + var policyErr *PolicyError + if errors.As(err, &policyErr) { + return policyErr.Code + } + if err == nil { + return "" + } + return "other:" + err.Error() +} + +func TestCheckTable(t *testing.T) { + policy := testPolicy(publicAnswer) + cases := []struct { + name string + url string + method string + mode string + want string + }{ + {"apex", "https://reddit.com/r/go", "GET", "fetch", ""}, + {"subdomain", "https://old.reddit.com/r/go.json", "get", "fetch", ""}, + {"short domain", "https://redd.it/abc", "GET", "any", ""}, + {"explicit 443", "https://www.reddit.com:443/", "GET", "fetch", ""}, + {"plain http 80", "http://reddit.com/", "GET", "fetch", ""}, + {"trailing dot", "https://reddit.com./", "GET", "fetch", ""}, + {"upper case host", "https://WWW.REDDIT.COM/", "GET", "fetch", ""}, + {"prefix attack", "https://evilreddit.com/", "GET", "fetch", RefuseHost}, + {"suffix attack", "https://reddit.com.evil.net/", "GET", "fetch", RefuseHost}, + {"hyphen attack", "https://not-reddit.com/", "GET", "fetch", RefuseHost}, + {"other host", "https://example.com/", "GET", "fetch", RefuseHost}, + {"port 8080", "https://reddit.com:8080/", "GET", "fetch", RefusePort}, + {"port 22", "http://reddit.com:22/", "GET", "fetch", RefusePort}, + {"port 0", "https://reddit.com:0/", "GET", "fetch", RefusePort}, + {"ipv4 literal", "http://127.0.0.1/", "GET", "fetch", RefuseHost}, + {"ipv4 literal 80", "http://93.184.216.34/", "GET", "fetch", RefuseHost}, + {"ipv6 literal", "http://[::1]/", "GET", "fetch", RefuseHost}, + {"credentials", "https://user:pw@reddit.com/", "GET", "fetch", RefuseBadURL}, + {"scheme ftp", "ftp://reddit.com/", "GET", "fetch", RefuseBadURL}, + {"scheme file", "file:///etc/passwd", "GET", "fetch", RefuseBadURL}, + {"empty", "", "GET", "fetch", RefuseBadURL}, + {"post", "https://reddit.com/", "POST", "fetch", RefuseMethod}, + {"delete", "https://reddit.com/", "DELETE", "fetch", RefuseMethod}, + {"bad port text", "https://reddit.com:abc/", "GET", "fetch", RefuseBadURL}, + } + for _, item := range cases { + _, err := policy.Check(item.url, item.method, item.mode) + if got := codeOf(err); got != item.want { + t.Errorf("%s: got %q want %q", item.name, got, item.want) + } + } + long := "https://reddit.com/" + string(make([]byte, MaxURLChars)) + if _, err := policy.Check(long, "GET", "fetch"); codeOf(err) != RefuseBadURL { + t.Errorf("long url not refused: %v", err) + } +} + +func TestCheckHostPortModes(t *testing.T) { + policy := testPolicy(publicAnswer) + if _, err := policy.CheckHostPort("reddit.com", 443, "tunnel"); err != nil { + t.Fatal(err) + } + if _, err := policy.CheckHostPort("reddit.com", 25, "tunnel"); codeOf(err) != RefusePort { + t.Fatalf("port: %v", err) + } + if _, err := policy.CheckHostPort("evilreddit.com", 443, "tunnel"); codeOf(err) != RefuseHost { + t.Fatalf("host: %v", err) + } +} + +func TestPublicAddressTable(t *testing.T) { + cases := map[string]bool{ + "8.8.8.8": true, "93.184.216.34": true, "2606:4700:4700::1111": true, + "10.0.0.1": false, "10.255.255.255": false, "127.0.0.1": false, "127.1.2.3": false, + "169.254.169.254": false, "172.16.0.1": false, "172.31.255.255": false, "172.32.0.1": true, + "192.168.1.1": false, "100.64.0.1": false, "100.127.255.255": false, "100.128.0.1": true, + "0.0.0.0": false, "0.1.2.3": false, "224.0.0.1": false, "239.255.255.250": false, + "255.255.255.255": false, "240.0.0.1": false, "198.18.0.1": false, "192.0.2.1": false, + "::": false, "::1": false, "fe80::1": false, "fc00::1": false, "fd12:3456::1": false, + "ff02::1": false, "::ffff:127.0.0.1": false, "::ffff:10.0.0.1": false, "::ffff:8.8.8.8": true, + "2001:db8::1": false, "64:ff9b::7f00:1": false, "2002:7f00:1::1": false, "2001::1": false, + } + for text, want := range cases { + if got := IsPublicAddr(netip.MustParseAddr(text)); got != want { + t.Errorf("%s: got %v want %v", text, got, want) + } + } + if IsPublicAddr(netip.Addr{}) { + t.Error("invalid address counted as public") + } + if IsPublicAddr(netip.MustParseAddr("fe80::1%eth0")) { + t.Error("zoned address counted as public") + } +} + +func TestResolveRefusesAnyPrivateAnswer(t *testing.T) { + cases := map[string][]string{ + "loopback": {"127.0.0.1"}, + "private": {"10.1.2.3"}, + "mixed": {"93.184.216.34", "192.168.0.1"}, + "v6 loopback": {"::1"}, + "v6 mapped": {"::ffff:10.0.0.1"}, + "cgnat": {"100.64.5.5"}, + } + for name, list := range cases { + addrs := []netip.Addr{} + for _, item := range list { + addrs = append(addrs, netip.MustParseAddr(item)) + } + policy := testPolicy(func(string) []netip.Addr { return addrs }) + if _, err := policy.Resolve(context.Background(), "reddit.com"); codeOf(err) != RefuseAddress { + t.Errorf("%s: %v", name, err) + } + } + empty := testPolicy(func(string) []netip.Addr { return nil }) + if _, err := empty.Resolve(context.Background(), "reddit.com"); codeOf(err) != FailDNS { + t.Errorf("empty answer: %v", err) + } +} + +func TestDialConnectsToValidatedIPAndDefeatsRebinding(t *testing.T) { + calls := 0 + policy := testPolicy(func(string) []netip.Addr { + calls++ + if calls == 1 { + return []netip.Addr{netip.MustParseAddr("93.184.216.34")} + } + return []netip.Addr{netip.MustParseAddr("127.0.0.1")} + }) + var dialed []string + policy.dial = func(ctx context.Context, network, address string) (net.Conn, error) { + dialed = append(dialed, address) + client, server := net.Pipe() + server.Close() + return client, nil + } + if _, err := policy.DialContext(context.Background(), "tcp", "www.reddit.com:443"); err != nil { + t.Fatal(err) + } + if len(dialed) != 1 || dialed[0] != "93.184.216.34:443" { + t.Fatalf("dialed %v", dialed) + } + if _, err := policy.DialContext(context.Background(), "tcp", "www.reddit.com:443"); codeOf(err) != RefuseAddress { + t.Fatalf("rebinding not refused: %v", err) + } + if len(dialed) != 1 { + t.Fatalf("private address was dialed: %v", dialed) + } + if _, err := policy.DialContext(context.Background(), "tcp", "evil.com:443"); codeOf(err) != RefuseHost { + t.Fatalf("host: %v", err) + } + if _, err := policy.DialContext(context.Background(), "tcp", "reddit.com:22"); codeOf(err) != RefusePort { + t.Fatalf("port: %v", err) + } +} + +func TestRedirectLeavingAllowlistRefused(t *testing.T) { + policy := testPolicy(publicAnswer) + cases := map[string]string{ + "https://old.reddit.com/x": "", + "https://redd.it/y": "", + "https://evil.com/": RefuseRedirect, + "https://evilreddit.com/": RefuseRedirect, + "http://reddit.com:8080/": RefuseRedirect, + "http://127.0.0.1/": RefuseRedirect, + "ftp://reddit.com/": RefuseRedirect, + "https://reddit.com.evil.io": RefuseRedirect, + } + for location, want := range cases { + _, err := policy.CheckRedirect(location, "GET") + if got := codeOf(err); got != want { + t.Errorf("%s: got %q want %q", location, got, want) + } + } + if _, err := policy.CheckRedirect("https://reddit.com/", "POST"); codeOf(err) != RefuseRedirect { + t.Error("method change through redirect allowed") + } +} + +func TestRefineOnlyNarrows(t *testing.T) { + policy := testPolicy(publicAnswer) + rejected := policy.Refine([]Destination{ + {Suffix: "old.reddit.com", Methods: []string{"GET", "POST"}, Ports: []int{80, 443, 8080}, Modes: []string{"fetch", "any"}}, + {Suffix: "redd.it", Methods: []string{"GET"}, Ports: []int{443}, Modes: []string{"tunnel", "fetch"}}, + {Suffix: "evil.com", Methods: []string{"GET"}, Ports: []int{443}, Modes: []string{"fetch"}}, + {Suffix: "evilreddit.com", Methods: []string{"GET"}, Ports: []int{443}, Modes: []string{"fetch"}}, + {Suffix: "old.reddit.com", Methods: []string{"GET"}, Ports: []int{443}, Modes: []string{"fetch"}}, + }) + if len(rejected) != 3 { + t.Fatalf("rejected %v", rejected) + } + if _, err := policy.Check("https://old.reddit.com/", "GET", "fetch"); err != nil { + t.Fatal(err) + } + if _, err := policy.Check("https://old.reddit.com/", "POST", "fetch"); codeOf(err) != RefuseMethod { + t.Errorf("post widened: %v", err) + } + if _, err := policy.Check("http://old.reddit.com:8080/", "GET", "fetch"); codeOf(err) != RefusePort { + t.Errorf("port widened: %v", err) + } + if _, err := policy.Check("https://www.reddit.com/", "GET", "fetch"); codeOf(err) != RefuseHost { + t.Errorf("dropped destination still allowed: %v", err) + } + if _, err := policy.Check("http://redd.it/a", "GET", "fetch"); codeOf(err) != RefusePort { + t.Errorf("redd.it port 80 should be gone: %v", err) + } + policy.Refine(nil) + if _, err := policy.Check("https://redd.it/a", "GET", "fetch"); codeOf(err) != RefuseHost { + t.Errorf("empty refinement must allow nothing: %v", err) + } +} diff --git a/nodes/client/protocol.go b/nodes/client/protocol.go new file mode 100644 index 00000000..47541ed2 --- /dev/null +++ b/nodes/client/protocol.go @@ -0,0 +1,146 @@ +package main + +import "encoding/json" + +const ( + ProtocolVersion = 1 + DeviceKind = "native" + SecretPrefix = "dpn_" + + MaxWSFramePayload = 256 * 1024 + MaxInboundMessage = 256 * 1024 + MaxResultMessage = 3 * 1024 * 1024 + + DefaultReconnectMin = 1 + DefaultReconnectMax = 60 + DefaultPingSeconds = 20 + DefaultMaxBodyBytes = 2 * 1024 * 1024 + DefaultJobTimeout = 25 + MaxJobTimeout = 60 + HelloTimeoutSeconds = 10 + TicketTimeoutSeconds = 15 + MissedPingsBeforeDead = 3 + UsageIntervalSeconds = 60 + MaxRedirects = 5 + MaxRequestHeaders = 32 + MaxResponseHeaders = 64 + MaxHeaderValueBytes = 1024 + MaxIDBytes = 64 + MaxURLChars = 2048 + MaxConcurrentCeiling = 8 + MaxTunnelsCeiling = 8 + TunnelChunkBytes = 32 * 1024 + TunnelQueueDepth = 64 + TunnelIdleSeconds = 60 + TunnelLifetimeSeconds = 300 + TunnelDialSeconds = 10 + MaxDonorMBPerDay = 20 * 1024 + StableSessionSeconds = 30 + FastRetryMillis = 1500 +) + +const ( + RefuseHost = "host_not_allowed" + RefusePort = "port_not_allowed" + RefuseAddress = "address_not_public" + RefuseRedirect = "redirect_not_allowed" + RefuseMethod = "method_not_allowed" + RefuseBandwidth = "bandwidth_exhausted" + RefuseConcurrency = "concurrency_exceeded" + RefuseHours = "outside_hours" + RefuseTimeout = "timeout" + RefuseBadURL = "bad_url" + FailFetch = "fetch_failed" + FailDNS = "dns_failed" +) + +const ( + CloseNormal = 1000 + CloseGoingAway = 1001 + CloseProtocolError = 1002 + CloseUnsupported = 1003 + CloseInvalidPayload = 1007 + ClosePolicy = 1008 + CloseTooLarge = 1009 + CloseTryAgain = 1013 + CloseRetryOwner = 4013 + CloseCapacity = 4029 + CloseProtocol = 4400 + CloseKicked = 4403 +) + +var FatalReasons = map[string]bool{ + "revoked": true, + "suspended": true, + "kill_switch": true, + "no_consent": true, + "no_account": true, + "deletion_pending": true, + "account_inactive": true, + "not_adult": true, + "not_enabled": true, + "replaced": true, + "device": true, +} + +var BakedDestinations = []Destination{ + {Suffix: "reddit.com", Adapter: "reddit", Methods: []string{"GET"}, Ports: []int{80, 443}, Modes: []string{"tunnel", "fetch"}}, + {Suffix: "redd.it", Adapter: "reddit", Methods: []string{"GET"}, Ports: []int{80, 443}, Modes: []string{"tunnel", "fetch"}}, +} + +var AllowedPorts = []int{80, 443} + +type Destination struct { + Suffix string `json:"suffix"` + Adapter string `json:"adapter"` + Methods []string `json:"methods"` + Ports []int `json:"ports"` + Modes []string `json:"modes"` + MaxRPS float64 `json:"max_rps"` +} + +type ServerLimits struct { + MaxBodyBytes int `json:"max_body_bytes"` + MaxConcurrentPerNode int `json:"max_concurrent_per_node"` + MaxDonorMBPerDay int `json:"max_donor_mb_per_day"` + PingSeconds int `json:"ping_seconds"` + ReconnectMinSeconds int `json:"reconnect_min_seconds"` + ReconnectMaxSeconds int `json:"reconnect_max_seconds"` + JobTimeoutSeconds int `json:"job_timeout_seconds"` + MaxFrameBytes int `json:"max_frame_bytes"` +} + +type ServerSettings struct { + Enabled bool `json:"enabled"` + BandwidthMBDay int `json:"bandwidth_mb_day"` + MaxConcurrent int `json:"max_concurrent"` + ActiveFrom string `json:"active_from"` + ActiveTo string `json:"active_to"` + Timezone string `json:"timezone"` +} + +type Frame struct { + Type string `json:"type"` + JobID string `json:"job_id"` + Method string `json:"method"` + URL string `json:"url"` + Headers map[string]any `json:"headers"` + Timeout int `json:"timeout"` + ConnID string `json:"conn_id"` + Host string `json:"host"` + Port int `json:"port"` + B64 string `json:"b64"` + Reason string `json:"reason"` + DeviceUID string `json:"device_uid"` + Limits *ServerLimits `json:"limits"` + Destinations []Destination `json:"destinations"` + Settings *ServerSettings `json:"settings"` +} + +func decodeFrame(raw []byte) (Frame, error) { + var frame Frame + if err := json.Unmarshal(raw, &frame); err != nil { + return frame, err + } + return frame, nil +} diff --git a/nodes/client/session.go b/nodes/client/session.go new file mode 100644 index 00000000..5a29cc38 --- /dev/null +++ b/nodes/client/session.go @@ -0,0 +1,445 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "runtime" + "strings" + "sync" + "time" +) + +type FatalError struct { + Reason string +} + +func (e *FatalError) Error() string { + return "the server ended this device for good: " + e.Reason +} + +type outcome struct { + fatal string + fast bool + stable bool + reason string +} + +type Session struct { + cfg Config + policy *Policy + limiter *Limiter + fetcher *Fetcher + stats *Stats + log *Logger + httpc *http.Client + jitter func() float64 + dialWS func(ctx context.Context, rawURL string) (*Conn, error) + limits ServerLimits + maxBody int +} + +func NewSession(cfg Config, policy *Policy, limiter *Limiter, fetcher *Fetcher, stats *Stats, logger *Logger) *Session { + return &Session{ + cfg: cfg, + policy: policy, + limiter: limiter, + fetcher: fetcher, + stats: stats, + log: logger, + httpc: &http.Client{Timeout: time.Duration(TicketTimeoutSeconds) * time.Second, Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}}, + jitter: rand.Float64, + dialWS: Dial, + limits: defaultLimits(), + maxBody: DefaultMaxBodyBytes, + } +} + +func defaultLimits() ServerLimits { + return ServerLimits{ + MaxBodyBytes: DefaultMaxBodyBytes, + PingSeconds: DefaultPingSeconds, + ReconnectMinSeconds: DefaultReconnectMin, + ReconnectMaxSeconds: DefaultReconnectMax, + JobTimeoutSeconds: DefaultJobTimeout, + MaxFrameBytes: MaxInboundMessage, + } +} + +func (s *Session) Backoff(attempt int) time.Duration { + low := s.limits.ReconnectMinSeconds + high := s.limits.ReconnectMaxSeconds + if low < 1 { + low = DefaultReconnectMin + } + if high < low { + high = low + } + if attempt > 16 { + attempt = 16 + } + ceiling := low << attempt + if ceiling > high || ceiling <= 0 { + ceiling = high + } + seconds := float64(low) + s.jitter()*float64(ceiling-low) + return time.Duration(seconds * float64(time.Second)) +} + +func sleepContext(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +func (s *Session) Run(ctx context.Context) error { + attempt := 0 + for ctx.Err() == nil { + result := s.connectOnce(ctx) + if result.fatal != "" { + return &FatalError{Reason: result.fatal} + } + if ctx.Err() != nil { + break + } + if result.stable { + attempt = 0 + } + var delay time.Duration + if result.fast { + delay = time.Duration((0.5 + s.jitter()) * float64(FastRetryMillis) * float64(time.Millisecond)) + } else { + delay = s.Backoff(attempt) + attempt++ + } + s.log.Printf("disconnected (%s), reconnecting in %s", result.reason, delay.Round(100*time.Millisecond)) + sleepContext(ctx, delay) + } + return nil +} + +type ticketAnswer struct { + Ticket string `json:"ticket"` + WSPath string `json:"ws_path"` +} + +type errorAnswer struct { + Error struct { + Message string `json:"message"` + Reason string `json:"reason"` + } `json:"error"` +} + +func (s *Session) fetchTicket(ctx context.Context) (ticketAnswer, outcome) { + var answer ticketAnswer + request, err := http.NewRequestWithContext(ctx, http.MethodPost, s.cfg.BaseURL()+"/nodes/ticket", nil) + if err != nil { + return answer, outcome{fatal: "bad server url"} + } + request.Header.Set("Authorization", "Bearer "+s.cfg.Secret) + request.Header.Set("Accept", "application/json") + request.Header.Set("User-Agent", "devplace-node/"+Version) + response, err := s.httpc.Do(request) + if err != nil { + return answer, outcome{reason: "ticket request failed: " + stripURL(err).Error()} + } + defer response.Body.Close() + body, _ := io.ReadAll(io.LimitReader(response.Body, 64*1024)) + switch { + case response.StatusCode == http.StatusOK: + if json.Unmarshal(body, &answer) != nil || answer.Ticket == "" || !strings.HasPrefix(answer.WSPath, "/") { + return answer, outcome{reason: "ticket answer is malformed"} + } + return answer, outcome{} + case response.StatusCode == http.StatusUnauthorized: + return answer, outcome{fatal: "unauthorized, the device secret is not valid (revoked or rotated)"} + case response.StatusCode == http.StatusForbidden: + var failure errorAnswer + _ = json.Unmarshal(body, &failure) + reason := failure.Error.Reason + if FatalReasons[reason] { + return answer, outcome{fatal: reason} + } + return answer, outcome{reason: "ticket refused: " + reason} + default: + return answer, outcome{reason: fmt.Sprintf("ticket request answered http %d", response.StatusCode)} + } +} + +func (s *Session) socketURL(answer ticketAnswer) (string, error) { + base, err := url.Parse(s.cfg.BaseURL()) + if err != nil { + return "", err + } + scheme := "ws" + if base.Scheme == "https" { + scheme = "wss" + } + return fmt.Sprintf("%s://%s%s?ticket=%s", scheme, base.Host, answer.WSPath, url.QueryEscape(answer.Ticket)), nil +} + +func (s *Session) connectOnce(ctx context.Context) outcome { + answer, failed := s.fetchTicket(ctx) + if failed.fatal != "" || failed.reason != "" { + return failed + } + target, err := s.socketURL(answer) + if err != nil { + return outcome{fatal: "bad server url"} + } + conn, err := s.dialWS(ctx, target) + if err != nil { + return outcome{reason: err.Error()} + } + defer conn.Close() + sessionCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + <-sessionCtx.Done() + conn.Close() + }() + started := time.Now() + result := s.converse(sessionCtx, conn) + if time.Since(started) >= time.Duration(StableSessionSeconds)*time.Second { + result.stable = true + } + return result +} + +func (s *Session) sender(conn *Conn) Sender { + return func(frameType string, fields map[string]any) error { + payload := map[string]any{"type": frameType} + for key, value := range fields { + payload[key] = value + } + data, err := json.Marshal(payload) + if err != nil { + return err + } + limit := MaxInboundMessage + if frameType == "job_result" { + limit = MaxResultMessage + } + if len(data) > limit { + return fmt.Errorf("%s frame of %d bytes exceeds the %d byte cap", frameType, len(data), limit) + } + return conn.WriteText(string(data)) + } +} + +func (s *Session) helloFields() map[string]any { + return map[string]any{ + "protocol": ProtocolVersion, + "client_version": Version, + "kind": DeviceKind, + "platform": runtime.GOOS + "/" + runtime.GOARCH, + "modes": []string{"fetch", "tunnel"}, + "limits": map[string]any{"max_concurrent": s.limiter.ConcurrencyCap()}, + } +} + +func (s *Session) applyWelcome(frame Frame) { + limits := defaultLimits() + if frame.Limits != nil { + given := *frame.Limits + if given.MaxBodyBytes > 0 { + limits.MaxBodyBytes = given.MaxBodyBytes + } + if given.PingSeconds > 0 { + limits.PingSeconds = given.PingSeconds + } + if given.ReconnectMinSeconds > 0 { + limits.ReconnectMinSeconds = given.ReconnectMinSeconds + } + if given.ReconnectMaxSeconds > 0 { + limits.ReconnectMaxSeconds = given.ReconnectMaxSeconds + } + limits.MaxConcurrentPerNode = given.MaxConcurrentPerNode + limits.MaxDonorMBPerDay = given.MaxDonorMBPerDay + if given.JobTimeoutSeconds > 0 { + limits.JobTimeoutSeconds = given.JobTimeoutSeconds + } + } + s.limits = limits + s.maxBody = min(limits.MaxBodyBytes, DefaultMaxBodyBytes) + if rejected := s.policy.Refine(frame.Destinations); len(rejected) > 0 { + s.log.Printf("server offered destinations outside the baked-in list, ignored: %s", strings.Join(rejected, ", ")) + } + s.limiter.ApplyServer(&limits, frame.Settings) +} + +func (s *Session) converse(ctx context.Context, conn *Conn) outcome { + send := s.sender(conn) + if err := send("hello", s.helloFields()); err != nil { + return outcome{reason: "hello failed: " + err.Error()} + } + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(HelloTimeoutSeconds) * time.Second)) + raw, err := conn.ReadMessage() + if err != nil { + return s.closeOutcome(err) + } + welcome, err := decodeFrame(raw) + if err != nil { + return s.protocolFailure(conn, "welcome is not valid json") + } + if welcome.Type == "kick" { + return s.kickOutcome(welcome.Reason) + } + if welcome.Type != "welcome" { + return s.protocolFailure(conn, "expected welcome, got "+welcome.Type) + } + s.applyWelcome(welcome) + s.log.Debugf("connected as device %s with %d allowed destinations", welcome.DeviceUID, len(s.policy.Destinations())) + + var jobs sync.WaitGroup + tunnels := NewTunnels(s.policy, s.limiter, s.stats, send, s.cfg.MaxTunnels) + background, stop := context.WithCancel(ctx) + jobs.Add(1) + go func() { + defer jobs.Done() + s.housekeeping(background, send) + }() + defer func() { + stop() + tunnels.CloseAll() + jobs.Wait() + s.limiter.Flush() + }() + + patience := time.Duration(s.limits.PingSeconds*MissedPingsBeforeDead) * time.Second + for ctx.Err() == nil { + _ = conn.SetReadDeadline(time.Now().Add(patience)) + raw, err := conn.ReadMessage() + if err != nil { + return s.closeOutcome(err) + } + frame, err := decodeFrame(raw) + if err != nil { + return s.protocolFailure(conn, "frame is not valid json") + } + switch frame.Type { + case "ping": + if err := send("pong", nil); err != nil { + return outcome{reason: "pong failed: " + err.Error()} + } + case "job": + s.startJob(background, &jobs, send, frame) + case "tunnel_open": + tunnels.Open(background, frame) + case "tunnel_data": + tunnels.Data(frame) + case "tunnel_close": + tunnels.Close(frame.ConnID) + case "settings_update": + s.limiter.ApplyServer(nil, frame.Settings) + case "kick": + return s.kickOutcome(frame.Reason) + default: + return s.protocolFailure(conn, "unexpected frame "+frame.Type) + } + } + return outcome{reason: "stopped"} +} + +func (s *Session) protocolFailure(conn *Conn, reason string) outcome { + _ = conn.WriteClose(CloseProtocol, "protocol") + return outcome{reason: "protocol error: " + reason} +} + +func (s *Session) kickOutcome(reason string) outcome { + if FatalReasons[reason] { + return outcome{fatal: reason} + } + return outcome{reason: "kicked: " + reason} +} + +func (s *Session) closeOutcome(err error) outcome { + var closeErr *CloseError + if !errors.As(err, &closeErr) { + return outcome{reason: "connection lost: " + err.Error()} + } + if FatalReasons[closeErr.Reason] { + return outcome{fatal: closeErr.Reason} + } + if closeErr.Code == CloseRetryOwner { + return outcome{fast: true, reason: "server asked for a retry on the owner worker"} + } + return outcome{reason: closeErr.Error()} +} + +func (s *Session) housekeeping(ctx context.Context, send Sender) { + ticker := time.NewTicker(time.Duration(UsageIntervalSeconds) * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if line := s.stats.Roll(time.Now()); line != "" { + s.log.Printf("%s", line) + } + up, down := s.stats.Take() + if up+down > 0 { + _ = send("usage", map[string]any{"bytes_up": up, "bytes_down": down}) + } + } + } +} + +func (s *Session) startJob(ctx context.Context, jobs *sync.WaitGroup, send Sender, frame Frame) { + if frame.JobID == "" || len(frame.JobID) > MaxIDBytes { + s.log.Printf("job without a usable id ignored") + return + } + if code := s.limiter.Acquire(); code != "" { + s.stats.Refused() + _ = send("job_error", map[string]any{"job_id": frame.JobID, "code": code}) + return + } + jobs.Add(1) + go func() { + defer jobs.Done() + defer s.limiter.Release() + s.runJob(ctx, send, frame) + }() +} + +func (s *Session) runJob(ctx context.Context, send Sender, frame Frame) { + result, err := s.fetcher.Do(ctx, frame, s.maxBody, s.limiter.Remaining()) + if err != nil { + code := ErrorCode(err) + var policyErr *PolicyError + if errors.As(err, &policyErr) { + s.stats.Refused() + } else { + s.stats.Failed() + } + s.log.Debugf("job %s failed: %s", frame.JobID, code) + _ = send("job_error", map[string]any{"job_id": frame.JobID, "code": code}) + return + } + up := int64(len(frame.URL) + 256) + down := int64(len(result.Body)) + s.limiter.AddBytes(up + down) + s.stats.Job(up, down) + fields := map[string]any{ + "job_id": frame.JobID, + "status": result.Status, + "headers": result.Headers, + "body_b64": base64.StdEncoding.EncodeToString(result.Body), + "truncated": result.Truncated, + "elapsed_ms": int(result.Elapsed.Milliseconds()), + } + if err := send("job_result", fields); err != nil { + s.log.Printf("job %s result could not be sent: %v", frame.JobID, err) + } +} diff --git a/nodes/client/session_test.go b/nodes/client/session_test.go new file mode 100644 index 00000000..2447528b --- /dev/null +++ b/nodes/client/session_test.go @@ -0,0 +1,362 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +const testSecret = "dpn_testsecret0123456789" + +type harness struct { + server *httptest.Server + content *httptest.Server + session *Session + policy *Policy + conns atomic.Int32 + tickets atomic.Int32 + failure atomic.Value +} + +func readFrame(t *testing.T, c *Conn) map[string]any { + t.Helper() + _ = c.SetReadDeadline(time.Now().Add(10 * time.Second)) + raw, err := c.ReadMessage() + if err != nil { + t.Errorf("fake server read: %v", err) + return map[string]any{"type": "read_error"} + } + frame := map[string]any{} + if err := json.Unmarshal(raw, &frame); err != nil { + t.Errorf("fake server decode: %v", err) + } + return frame +} + +func writeFrame(t *testing.T, c *Conn, frame map[string]any) { + t.Helper() + data, _ := json.Marshal(frame) + if err := c.WriteText(string(data)); err != nil { + t.Errorf("fake server write: %v", err) + } +} + +func welcome() map[string]any { + return map[string]any{ + "type": "welcome", + "device_uid": "dev1", + "limits": map[string]any{ + "max_body_bytes": 1000, "ping_seconds": 20, "reconnect_min_seconds": 1, "reconnect_max_seconds": 2, + "max_concurrent_per_node": 2, "max_donor_mb_per_day": 20480, "job_timeout_seconds": 25, + }, + "destinations": []any{ + map[string]any{"suffix": "reddit.com", "adapter": "reddit", "methods": []string{"GET"}, "ports": []int{80, 443}, "modes": []string{"tunnel", "fetch"}, "max_rps": 0.5}, + map[string]any{"suffix": "redd.it", "adapter": "reddit", "methods": []string{"GET"}, "ports": []int{80, 443}, "modes": []string{"tunnel", "fetch"}, "max_rps": 0.5}, + }, + "settings": map[string]any{"enabled": true, "bandwidth_mb_day": 500, "max_concurrent": 2, "active_from": "", "active_to": "", "timezone": "UTC"}, + } +} + +func newHarness(t *testing.T, script func(n int, c *Conn)) *harness { + t.Helper() + h := &harness{} + h.content = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Cookie") != "" || r.Header.Get("Authorization") != "" { + h.failure.Store("identity header reached the destination") + } + _, _ = w.Write([]byte(strings.Repeat("r", 5000))) + })) + mux := http.NewServeMux() + mux.HandleFunc("/nodes/ticket", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.Header.Get("Authorization") != "Bearer "+testSecret { + w.WriteHeader(http.StatusUnauthorized) + return + } + n := h.tickets.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{"ticket": "t" + string(rune('0'+n)), "expires_in": 30, "ws_path": "/nodes/ws"}) + }) + mux.HandleFunc("/nodes/ws", func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.URL.Query().Get("ticket"), "t") || strings.Contains(r.URL.RawQuery, testSecret) { + h.failure.Store("bad ticket query") + } + conn, err := upgrade(w, r) + if err != nil { + return + } + defer conn.Close() + script(int(h.conns.Add(1)), conn) + }) + h.server = httptest.NewServer(mux) + t.Cleanup(h.server.Close) + t.Cleanup(h.content.Close) + cfg := DefaultConfig() + cfg.Server = h.server.URL + cfg.Secret = testSecret + cfg.Timezone = "UTC" + h.policy = testPolicy(publicAnswer) + h.policy.dial = func(ctx context.Context, network, address string) (net.Conn, error) { + var dialer net.Dialer + return dialer.DialContext(ctx, "tcp", h.content.Listener.Addr().String()) + } + limiter, err := NewLimiter(cfg, "", time.Now) + if err != nil { + t.Fatal(err) + } + h.session = NewSession(cfg, h.policy, limiter, NewFetcher(h.policy), NewStats(time.Now()), NewLogger(testWriter{t}, true)) + h.session.jitter = func() float64 { return 0 } + return h +} + +type testWriter struct{ t *testing.T } + +func (w testWriter) Write(p []byte) (int, error) { + w.t.Log(strings.TrimSpace(string(p))) + return len(p), nil +} + +func (h *harness) run(t *testing.T) error { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + err := h.session.Run(ctx) + if failure := h.failure.Load(); failure != nil { + t.Error(failure) + } + return err +} + +func TestSessionServesJobsRefusesBadOnesAndStopsOnKick(t *testing.T) { + h := newHarness(t, func(n int, c *Conn) { + hello := readFrame(t, c) + if hello["type"] != "hello" || hello["protocol"] != float64(1) || hello["kind"] != "native" || hello["client_version"] != Version { + t.Errorf("hello %v", hello) + } + if modes, _ := hello["modes"].([]any); len(modes) == 0 { + t.Errorf("hello modes %v", hello["modes"]) + } + if n == 2 { + writeFrame(t, c, welcome()) + writeFrame(t, c, map[string]any{"type": "kick", "reason": "revoked"}) + return + } + writeFrame(t, c, welcome()) + writeFrame(t, c, map[string]any{"type": "ping"}) + if frame := readFrame(t, c); frame["type"] != "pong" { + t.Errorf("expected pong, got %v", frame) + } + writeFrame(t, c, map[string]any{"type": "job", "job_id": "ok1", "method": "GET", "url": "http://www.reddit.com/r/go.json", "timeout": 5}) + result := readFrame(t, c) + if result["type"] != "job_result" || result["job_id"] != "ok1" || result["status"] != float64(200) || result["truncated"] != true { + t.Errorf("result %v", result) + } else { + body, err := base64.StdEncoding.DecodeString(result["body_b64"].(string)) + if err != nil || len(body) != 1000 { + t.Errorf("body %d %v", len(body), err) + } + } + refused := map[string]string{ + "bad1": "http://evilreddit.com/", "bad2": "http://reddit.com:8080/", "bad3": "http://127.0.0.1/", + } + for id, target := range refused { + writeFrame(t, c, map[string]any{"type": "job", "job_id": id, "method": "GET", "url": target, "timeout": 5}) + answer := readFrame(t, c) + want := RefuseHost + if id == "bad2" { + want = RefusePort + } + if answer["type"] != "job_error" || answer["job_id"] != id || answer["code"] != want { + t.Errorf("%s: %v", id, answer) + } + } + writeFrame(t, c, map[string]any{"type": "job", "job_id": "post", "method": "POST", "url": "http://reddit.com/", "timeout": 5}) + if answer := readFrame(t, c); answer["code"] != RefuseMethod { + t.Errorf("post: %v", answer) + } + writeFrame(t, c, map[string]any{"type": "tunnel_open", "conn_id": "c1", "host": "evil.com", "port": 443}) + if answer := readFrame(t, c); answer["type"] != "tunnel_opened" || answer["ok"] != false || answer["code"] != RefuseHost { + t.Errorf("tunnel: %v", answer) + } + _ = c.WriteClose(CloseRetryOwner, "owner") + }) + started := time.Now() + err := h.run(t) + var fatal *FatalError + if !errors.As(err, &fatal) || fatal.Reason != "revoked" { + t.Fatalf("run returned %v", err) + } + if h.conns.Load() != 2 || h.tickets.Load() != 2 { + t.Fatalf("connections %d tickets %d", h.conns.Load(), h.tickets.Load()) + } + if time.Since(started) > 5*time.Second { + t.Fatal("4013 was not retried quickly") + } + if line := h.session.stats.Line(); !strings.Contains(line, "jobs=1") || !strings.Contains(line, "refused=") { + t.Fatalf("stats %s", line) + } +} + +func TestSessionRelaysTunnelBothWays(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + go func() { + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + buffer := make([]byte, 64) + for { + n, err := conn.Read(buffer) + if err != nil { + return + } + _, _ = conn.Write([]byte("echo:" + string(buffer[:n]))) + } + }() + h := newHarness(t, func(n int, c *Conn) { + readFrame(t, c) + writeFrame(t, c, welcome()) + if n == 2 { + writeFrame(t, c, map[string]any{"type": "kick", "reason": "kill_switch"}) + return + } + writeFrame(t, c, map[string]any{"type": "tunnel_open", "conn_id": "c1", "host": "www.reddit.com", "port": 443}) + if answer := readFrame(t, c); answer["type"] != "tunnel_opened" || answer["ok"] != true { + t.Errorf("opened: %v", answer) + return + } + writeFrame(t, c, map[string]any{"type": "tunnel_data", "conn_id": "c1", "b64": base64.StdEncoding.EncodeToString([]byte("hi"))}) + data := readFrame(t, c) + decoded, _ := base64.StdEncoding.DecodeString(data["b64"].(string)) + if data["type"] != "tunnel_data" || string(decoded) != "echo:hi" { + t.Errorf("data: %v %q", data, decoded) + } + writeFrame(t, c, map[string]any{"type": "tunnel_close", "conn_id": "c1"}) + _ = c.WriteClose(CloseGoingAway, "restart") + }) + h.policy.dial = func(ctx context.Context, network, address string) (net.Conn, error) { + var dialer net.Dialer + return dialer.DialContext(ctx, "tcp", listener.Addr().String()) + } + err = h.run(t) + var fatal *FatalError + if !errors.As(err, &fatal) || fatal.Reason != "kill_switch" { + t.Fatalf("run returned %v", err) + } +} + +func TestSessionLimitsAnswerWithDonorCodes(t *testing.T) { + h := newHarness(t, func(n int, c *Conn) { + readFrame(t, c) + writeFrame(t, c, welcome()) + if n == 1 { + writeFrame(t, c, map[string]any{"type": "job", "job_id": "a", "method": "GET", "url": "http://reddit.com/", "timeout": 5}) + if answer := readFrame(t, c); answer["code"] != RefuseBandwidth { + t.Errorf("bandwidth: %v", answer) + } + _ = c.WriteClose(CloseGoingAway, "bye") + return + } + writeFrame(t, c, map[string]any{"type": "kick", "reason": "suspended"}) + }) + h.session.limiter.AddBytes(1 << 40) + err := h.run(t) + var fatal *FatalError + if !errors.As(err, &fatal) || fatal.Reason != "suspended" { + t.Fatalf("run returned %v", err) + } +} + +func TestFatalTicketStatuses(t *testing.T) { + cases := []struct { + status int + body string + fatal string + }{ + {401, "", "unauthorized"}, + {403, `{"error":{"status":403,"message":"x","reason":"no_consent"}}`, "no_consent"}, + {403, `{"error":{"status":403,"message":"x","reason":"suspended"}}`, "suspended"}, + } + for _, item := range cases { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(item.status) + _, _ = w.Write([]byte(item.body)) + })) + cfg := DefaultConfig() + cfg.Server, cfg.Secret = server.URL, testSecret + limiter, _ := NewLimiter(cfg, "", time.Now) + policy := testPolicy(publicAnswer) + session := NewSession(cfg, policy, limiter, NewFetcher(policy), NewStats(time.Now()), NewLogger(testWriter{t}, true)) + err := session.Run(context.Background()) + var fatal *FatalError + if !errors.As(err, &fatal) || !strings.Contains(fatal.Reason, item.fatal) { + t.Errorf("%d: %v", item.status, err) + } + server.Close() + } +} + +func TestTransientTicketFailureRetries(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) < 3 { + w.WriteHeader(http.StatusBadGateway) + return + } + w.WriteHeader(http.StatusUnauthorized) + })) + defer server.Close() + cfg := DefaultConfig() + cfg.Server, cfg.Secret = server.URL, testSecret + limiter, _ := NewLimiter(cfg, "", time.Now) + policy := testPolicy(publicAnswer) + session := NewSession(cfg, policy, limiter, NewFetcher(policy), NewStats(time.Now()), NewLogger(testWriter{t}, true)) + session.jitter = func() float64 { return 0 } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var fatal *FatalError + if err := session.Run(ctx); !errors.As(err, &fatal) || calls.Load() != 3 { + t.Fatalf("err %v calls %d", err, calls.Load()) + } +} + +func TestBackoffStaysInsideBounds(t *testing.T) { + session := &Session{limits: ServerLimits{ReconnectMinSeconds: 1, ReconnectMaxSeconds: 60}} + for _, jitter := range []float64{0, 0.5, 0.999999} { + session.jitter = func() float64 { return jitter } + for attempt := 0; attempt < 40; attempt++ { + d := session.Backoff(attempt) + if d < time.Second || d > 60*time.Second { + t.Fatalf("attempt %d jitter %v: %v", attempt, jitter, d) + } + } + } + session.jitter = func() float64 { return 0.999999 } + if first, later := session.Backoff(0), session.Backoff(5); first > time.Second+time.Millisecond || later <= first { + t.Fatalf("growth: %v %v", first, later) + } +} + +func TestFatalReasonsMatchPlan(t *testing.T) { + for _, reason := range []string{"revoked", "suspended", "kill_switch", "no_consent"} { + if !FatalReasons[reason] { + t.Errorf("%s must stop reconnecting", reason) + } + } + for _, reason := range []string{"heartbeat", "ticket", "capacity"} { + if FatalReasons[reason] { + t.Errorf("%s must reconnect", reason) + } + } +} diff --git a/nodes/client/stats.go b/nodes/client/stats.go new file mode 100644 index 00000000..15d62bfd --- /dev/null +++ b/nodes/client/stats.go @@ -0,0 +1,90 @@ +package main + +import ( + "fmt" + "sync" + "time" +) + +type Stats struct { + mu sync.Mutex + day string + jobs int + refused int + failed int + tunnels int + up int64 + down int64 + sentUp int64 + sentDown int64 +} + +func NewStats(now time.Time) *Stats { + return &Stats{day: now.Format("2006-01-02")} +} + +func (s *Stats) Job(up, down int64) { + s.mu.Lock() + s.jobs++ + s.up += up + s.down += down + s.mu.Unlock() +} + +func (s *Stats) Refused() { + s.mu.Lock() + s.refused++ + s.mu.Unlock() +} + +func (s *Stats) Failed() { + s.mu.Lock() + s.failed++ + s.mu.Unlock() +} + +func (s *Stats) Tunnel() { + s.mu.Lock() + s.tunnels++ + s.mu.Unlock() +} + +func (s *Stats) Bytes(up, down int64) { + s.mu.Lock() + s.up += up + s.down += down + s.mu.Unlock() +} + +func (s *Stats) Take() (up, down int64) { + s.mu.Lock() + defer s.mu.Unlock() + up, down = s.up-s.sentUp, s.down-s.sentDown + s.sentUp, s.sentDown = s.up, s.down + return +} + +func (s *Stats) line() string { + return fmt.Sprintf("totals day=%s jobs=%d refused=%d failed=%d tunnels=%d bytes_up=%d bytes_down=%d", + s.day, s.jobs, s.refused, s.failed, s.tunnels, s.up, s.down) +} + +func (s *Stats) Line() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.line() +} + +func (s *Stats) Roll(now time.Time) string { + s.mu.Lock() + defer s.mu.Unlock() + today := now.Format("2006-01-02") + if today == s.day { + return "" + } + line := s.line() + s.day = today + s.jobs, s.refused, s.failed, s.tunnels = 0, 0, 0, 0 + s.up, s.down, s.sentUp, s.sentDown = 0, 0, 0, 0 + return line +} diff --git a/nodes/client/tunnel.go b/nodes/client/tunnel.go new file mode 100644 index 00000000..ddaf91e7 --- /dev/null +++ b/nodes/client/tunnel.go @@ -0,0 +1,201 @@ +package main + +import ( + "context" + "encoding/base64" + "net" + "strconv" + "sync" + "time" +) + +type Sender func(frameType string, fields map[string]any) error + +type tunnel struct { + cancel context.CancelFunc + out chan []byte + remote bool +} + +type Tunnels struct { + policy *Policy + limiter *Limiter + stats *Stats + send Sender + max int + mu sync.Mutex + items map[string]*tunnel + wg sync.WaitGroup +} + +func NewTunnels(policy *Policy, limiter *Limiter, stats *Stats, send Sender, max int) *Tunnels { + return &Tunnels{policy: policy, limiter: limiter, stats: stats, send: send, max: max, items: map[string]*tunnel{}} +} + +func (t *Tunnels) deny(id, code string) { + _ = t.send("tunnel_opened", map[string]any{"conn_id": id, "ok": false, "code": code}) +} + +func (t *Tunnels) Open(ctx context.Context, frame Frame) { + id := frame.ConnID + if id == "" || len(id) > MaxIDBytes { + return + } + if _, err := t.policy.CheckHostPort(frame.Host, frame.Port, "tunnel"); err != nil { + t.deny(id, ErrorCode(err)) + t.stats.Refused() + return + } + t.mu.Lock() + _, exists := t.items[id] + full := len(t.items) >= t.max + t.mu.Unlock() + if exists || full { + t.deny(id, RefuseConcurrency) + t.stats.Refused() + return + } + if code := t.limiter.Acquire(); code != "" { + t.deny(id, code) + t.stats.Refused() + return + } + runCtx, cancel := context.WithTimeout(ctx, time.Duration(TunnelLifetimeSeconds)*time.Second) + item := &tunnel{cancel: cancel, out: make(chan []byte, TunnelQueueDepth)} + t.mu.Lock() + if _, taken := t.items[id]; taken || len(t.items) >= t.max { + t.mu.Unlock() + cancel() + t.limiter.Release() + t.deny(id, RefuseConcurrency) + t.stats.Refused() + return + } + t.items[id] = item + t.mu.Unlock() + t.wg.Add(1) + go t.run(runCtx, id, frame.Host, frame.Port, item) +} + +func (t *Tunnels) finish(id string, item *tunnel, announce bool) { + t.mu.Lock() + if t.items[id] == item { + delete(t.items, id) + } + t.mu.Unlock() + item.cancel() + t.limiter.Release() + if announce { + _ = t.send("tunnel_close", map[string]any{"conn_id": id}) + } +} + +func (t *Tunnels) run(ctx context.Context, id, host string, port int, item *tunnel) { + defer t.wg.Done() + conn, err := t.policy.DialContext(ctx, "tcp", net.JoinHostPort(host, strconv.Itoa(port))) + if err != nil { + t.finish(id, item, false) + t.deny(id, ErrorCode(err)) + t.stats.Failed() + return + } + t.stats.Tunnel() + if err := t.send("tunnel_opened", map[string]any{"conn_id": id, "ok": true}); err != nil { + conn.Close() + t.finish(id, item, false) + return + } + go func() { + <-ctx.Done() + conn.Close() + }() + var writers sync.WaitGroup + writers.Add(1) + go func() { + defer writers.Done() + for { + select { + case <-ctx.Done(): + return + case chunk := <-item.out: + _ = conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + if _, err := conn.Write(chunk); err != nil { + item.cancel() + return + } + t.stats.Bytes(int64(len(chunk)), 0) + t.limiter.AddBytes(int64(len(chunk))) + } + } + }() + buffer := make([]byte, TunnelChunkBytes) + for ctx.Err() == nil { + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(TunnelIdleSeconds) * time.Second)) + n, err := conn.Read(buffer) + if n > 0 { + t.stats.Bytes(0, int64(n)) + t.limiter.AddBytes(int64(n)) + encoded := base64.StdEncoding.EncodeToString(buffer[:n]) + if sendErr := t.send("tunnel_data", map[string]any{"conn_id": id, "b64": encoded}); sendErr != nil { + break + } + if t.limiter.Remaining() == 0 { + break + } + } + if err != nil { + break + } + } + conn.Close() + item.cancel() + writers.Wait() + t.mu.Lock() + announce := !item.remote + t.mu.Unlock() + t.finish(id, item, announce) +} + +func (t *Tunnels) Data(frame Frame) { + t.mu.Lock() + item := t.items[frame.ConnID] + t.mu.Unlock() + if item == nil { + return + } + if len(frame.B64) > MaxInboundMessage { + item.cancel() + return + } + chunk, err := base64.StdEncoding.Strict().DecodeString(frame.B64) + if err != nil { + item.cancel() + return + } + select { + case item.out <- chunk: + default: + item.cancel() + } +} + +func (t *Tunnels) Close(id string) { + t.mu.Lock() + item := t.items[id] + if item != nil { + item.remote = true + } + t.mu.Unlock() + if item != nil { + item.cancel() + } +} + +func (t *Tunnels) CloseAll() { + t.mu.Lock() + for _, item := range t.items { + item.cancel() + } + t.mu.Unlock() + t.wg.Wait() +} diff --git a/nodes/client/ws.go b/nodes/client/ws.go new file mode 100644 index 00000000..c89578f2 --- /dev/null +++ b/nodes/client/ws.go @@ -0,0 +1,358 @@ +package main + +import ( + "bufio" + "context" + "crypto/rand" + "crypto/sha1" + "crypto/tls" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" + "unicode/utf8" +) + +const ( + opContinuation = 0x0 + opText = 0x1 + opBinary = 0x2 + opClose = 0x8 + opPing = 0x9 + opPong = 0xA + + wsGUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + wsWriteTimeout = 30 * time.Second + wsHandshakeLimit = 15 * time.Second + maxControlBytes = 125 +) + +type CloseError struct { + Code int + Reason string +} + +func (e *CloseError) Error() string { + return fmt.Sprintf("websocket closed code=%d reason=%q", e.Code, e.Reason) +} + +type Conn struct { + rwc net.Conn + br *bufio.Reader + client bool + maxMessage int + wmu sync.Mutex + closed bool +} + +func newConn(rwc net.Conn, br *bufio.Reader, client bool) *Conn { + return &Conn{rwc: rwc, br: br, client: client, maxMessage: MaxInboundMessage} +} + +func Dial(ctx context.Context, rawURL string) (*Conn, error) { + target, err := url.Parse(rawURL) + if err != nil { + return nil, errors.New("websocket url cannot be parsed") + } + secure := false + switch target.Scheme { + case "wss", "https": + secure = true + case "ws", "http": + default: + return nil, errors.New("websocket url scheme must be ws, wss, http or https") + } + host := target.Hostname() + port := target.Port() + if port == "" { + port = "80" + if secure { + port = "443" + } + } + address := net.JoinHostPort(host, port) + dialer := &net.Dialer{Timeout: wsHandshakeLimit} + raw, err := dialer.DialContext(ctx, "tcp", address) + if err != nil { + return nil, fmt.Errorf("websocket connect to %s failed: %w", address, stripURL(err)) + } + var rwc net.Conn = raw + if secure { + tlsConn := tls.Client(raw, &tls.Config{ServerName: host, MinVersion: tls.VersionTLS12}) + _ = raw.SetDeadline(time.Now().Add(wsHandshakeLimit)) + if err := tlsConn.HandshakeContext(ctx); err != nil { + raw.Close() + return nil, fmt.Errorf("websocket tls handshake failed: %w", err) + } + rwc = tlsConn + } + _ = rwc.SetDeadline(time.Now().Add(wsHandshakeLimit)) + keyBytes := make([]byte, 16) + if _, err := rand.Read(keyBytes); err != nil { + rwc.Close() + return nil, err + } + key := base64.StdEncoding.EncodeToString(keyBytes) + path := target.RequestURI() + if path == "" { + path = "/" + } + request := fmt.Sprintf( + "GET %s HTTP/1.1\r\nHost: %s\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: %s\r\nSec-WebSocket-Version: 13\r\nUser-Agent: devplace-node/%s\r\n\r\n", + path, target.Host, key, Version, + ) + if _, err := io.WriteString(rwc, request); err != nil { + rwc.Close() + return nil, fmt.Errorf("websocket handshake write failed: %w", err) + } + br := bufio.NewReaderSize(rwc, 16*1024) + response, err := http.ReadResponse(br, &http.Request{Method: "GET"}) + if err != nil { + rwc.Close() + return nil, fmt.Errorf("websocket handshake read failed: %w", err) + } + if response.StatusCode != http.StatusSwitchingProtocols { + rwc.Close() + return nil, fmt.Errorf("websocket handshake refused with http status %d", response.StatusCode) + } + if !strings.EqualFold(response.Header.Get("Upgrade"), "websocket") || + response.Header.Get("Sec-WebSocket-Accept") != acceptKey(key) { + rwc.Close() + return nil, errors.New("websocket handshake answer is invalid") + } + _ = rwc.SetDeadline(time.Time{}) + return newConn(rwc, br, true), nil +} + +func stripURL(err error) error { + var urlErr *url.Error + if errors.As(err, &urlErr) { + return urlErr.Err + } + return err +} + +func acceptKey(key string) string { + sum := sha1.Sum([]byte(key + wsGUID)) + return base64.StdEncoding.EncodeToString(sum[:]) +} + +func (c *Conn) SetReadDeadline(t time.Time) error { return c.rwc.SetReadDeadline(t) } + +func (c *Conn) Close() error { + c.wmu.Lock() + c.closed = true + c.wmu.Unlock() + return c.rwc.Close() +} + +func (c *Conn) writeFrame(fin bool, op byte, payload []byte) error { + header := make([]byte, 0, 14) + first := op + if fin { + first |= 0x80 + } + header = append(header, first) + maskBit := byte(0) + if c.client { + maskBit = 0x80 + } + size := len(payload) + switch { + case size <= 125: + header = append(header, maskBit|byte(size)) + case size <= 0xFFFF: + header = append(header, maskBit|126, 0, 0) + binary.BigEndian.PutUint16(header[len(header)-2:], uint16(size)) + default: + header = append(header, maskBit|127, 0, 0, 0, 0, 0, 0, 0, 0) + binary.BigEndian.PutUint64(header[len(header)-8:], uint64(size)) + } + body := payload + if c.client { + mask := make([]byte, 4) + if _, err := rand.Read(mask); err != nil { + return err + } + header = append(header, mask...) + body = make([]byte, size) + for i := range payload { + body[i] = payload[i] ^ mask[i%4] + } + } + if err := c.rwc.SetWriteDeadline(time.Now().Add(wsWriteTimeout)); err != nil { + return err + } + if _, err := c.rwc.Write(header); err != nil { + return err + } + _, err := c.rwc.Write(body) + return err +} + +func (c *Conn) WriteText(text string) error { + data := []byte(text) + c.wmu.Lock() + defer c.wmu.Unlock() + if c.closed { + return net.ErrClosed + } + op := byte(opText) + for first := true; first || len(data) > 0; first = false { + chunk := data + if len(chunk) > MaxWSFramePayload { + chunk = data[:MaxWSFramePayload] + } + data = data[len(chunk):] + if err := c.writeFrame(len(data) == 0, op, chunk); err != nil { + return err + } + op = opContinuation + } + return nil +} + +func (c *Conn) writeControl(op byte, payload []byte) error { + if len(payload) > maxControlBytes { + payload = payload[:maxControlBytes] + } + c.wmu.Lock() + defer c.wmu.Unlock() + if c.closed { + return net.ErrClosed + } + return c.writeFrame(true, op, payload) +} + +func (c *Conn) WriteClose(code int, reason string) error { + if len(reason) > 100 { + reason = reason[:100] + } + payload := make([]byte, 2, 2+len(reason)) + binary.BigEndian.PutUint16(payload, uint16(code)) + payload = append(payload, reason...) + return c.writeControl(opClose, payload) +} + +func (c *Conn) readFrame() (fin bool, op byte, payload []byte, err error) { + head := make([]byte, 2) + if _, err = io.ReadFull(c.br, head); err != nil { + return + } + fin = head[0]&0x80 != 0 + if head[0]&0x70 != 0 { + err = &CloseError{Code: CloseProtocolError, Reason: "reserved bits set"} + return + } + op = head[0] & 0x0F + masked := head[1]&0x80 != 0 + if masked == c.client { + err = &CloseError{Code: CloseProtocolError, Reason: "bad masking"} + return + } + length := uint64(head[1] & 0x7F) + switch length { + case 126: + ext := make([]byte, 2) + if _, err = io.ReadFull(c.br, ext); err != nil { + return + } + length = uint64(binary.BigEndian.Uint16(ext)) + case 127: + ext := make([]byte, 8) + if _, err = io.ReadFull(c.br, ext); err != nil { + return + } + length = binary.BigEndian.Uint64(ext) + } + if op&0x8 != 0 && (length > maxControlBytes || !fin) { + err = &CloseError{Code: CloseProtocolError, Reason: "bad control frame"} + return + } + if length > uint64(c.maxMessage) || length > uint64(MaxResultMessage) { + err = &CloseError{Code: CloseTooLarge, Reason: "frame too large"} + return + } + var mask []byte + if masked { + mask = make([]byte, 4) + if _, err = io.ReadFull(c.br, mask); err != nil { + return + } + } + payload = make([]byte, length) + if _, err = io.ReadFull(c.br, payload); err != nil { + return + } + if masked { + for i := range payload { + payload[i] ^= mask[i%4] + } + } + return +} + +func (c *Conn) fail(code int, reason string) error { + _ = c.WriteClose(code, reason) + return &CloseError{Code: code, Reason: reason} +} + +func (c *Conn) ReadMessage() ([]byte, error) { + var message []byte + inProgress := false + for { + fin, op, payload, err := c.readFrame() + if err != nil { + var closeErr *CloseError + if errors.As(err, &closeErr) { + return nil, c.fail(closeErr.Code, closeErr.Reason) + } + return nil, err + } + switch op { + case opPing: + if err := c.writeControl(opPong, payload); err != nil { + return nil, err + } + case opPong: + case opClose: + code, reason := CloseNormal, "" + if len(payload) >= 2 { + code = int(binary.BigEndian.Uint16(payload)) + reason = string(payload[2:]) + } + _ = c.WriteClose(code, "") + return nil, &CloseError{Code: code, Reason: reason} + case opText, opBinary, opContinuation: + if op == opContinuation && !inProgress { + return nil, c.fail(CloseProtocolError, "unexpected continuation") + } + if op != opContinuation && inProgress { + return nil, c.fail(CloseProtocolError, "interleaved data frames") + } + if op == opBinary { + return nil, c.fail(CloseUnsupported, "binary frames are not supported") + } + inProgress = !fin + if len(message)+len(payload) > c.maxMessage { + return nil, c.fail(CloseTooLarge, "message too large") + } + message = append(message, payload...) + if fin { + if !utf8.Valid(message) { + return nil, c.fail(CloseInvalidPayload, "invalid utf-8") + } + return message, nil + } + default: + return nil, c.fail(CloseProtocolError, "unknown opcode") + } + } +} diff --git a/nodes/client/ws_test.go b/nodes/client/ws_test.go new file mode 100644 index 00000000..4a628035 --- /dev/null +++ b/nodes/client/ws_test.go @@ -0,0 +1,220 @@ +package main + +import ( + "bytes" + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func upgrade(w http.ResponseWriter, r *http.Request) (*Conn, error) { + hijacker := w.(http.Hijacker) + rwc, brw, err := hijacker.Hijack() + if err != nil { + return nil, err + } + answer := "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: " + + acceptKey(r.Header.Get("Sec-WebSocket-Key")) + "\r\n\r\n" + if _, err := rwc.Write([]byte(answer)); err != nil { + return nil, err + } + conn := newConn(rwc, brw.Reader, false) + conn.maxMessage = MaxResultMessage + return conn, nil +} + +func wsServer(t *testing.T, handler func(*Conn)) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrade(w, r) + if err != nil { + t.Errorf("upgrade: %v", err) + return + } + defer conn.Close() + handler(conn) + })) + t.Cleanup(server.Close) + return server +} + +func dialTest(t *testing.T, server *httptest.Server) *Conn { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/socket") + if err != nil { + t.Fatalf("dial: %v", err) + } + t.Cleanup(func() { conn.Close() }) + _ = conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + return conn +} + +func TestAcceptKeyVector(t *testing.T) { + if got := acceptKey("dGhlIHNhbXBsZSBub25jZQ=="); got != "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" { + t.Fatalf("accept key %s", got) + } +} + +func TestRoundTripAndFragmentation(t *testing.T) { + sizes := []int{0, 1, 125, 126, 65535, 65536, MaxWSFramePayload, MaxWSFramePayload + 1, 700 * 1024} + server := wsServer(t, func(c *Conn) { + for range sizes { + message, err := c.ReadMessage() + if err != nil { + t.Errorf("server read: %v", err) + return + } + if len(message) <= MaxInboundMessage { + if err := c.WriteText(string(message)); err != nil { + t.Errorf("server write: %v", err) + return + } + } else if err := c.WriteText("big"); err != nil { + return + } + } + }) + conn := dialTest(t, server) + for _, size := range sizes { + text := strings.Repeat("a", size) + if err := conn.WriteText(text); err != nil { + t.Fatalf("write %d: %v", size, err) + } + got, err := conn.ReadMessage() + if err != nil { + t.Fatalf("read %d: %v", size, err) + } + want := text + if size > MaxInboundMessage { + want = "big" + } + if string(got) != want { + t.Fatalf("size %d: got %d bytes", size, len(got)) + } + } +} + +func TestPingAnsweredWithPong(t *testing.T) { + done := make(chan []byte, 1) + server := wsServer(t, func(c *Conn) { + _ = c.writeControl(opPing, []byte("probe")) + _ = c.WriteText("after") + _, op, payload, err := c.readFrame() + if err != nil || op != opPong { + t.Errorf("expected pong, got op %d err %v", op, err) + } + done <- payload + }) + conn := dialTest(t, server) + message, err := conn.ReadMessage() + if err != nil || string(message) != "after" { + t.Fatalf("message %q err %v", message, err) + } + if got := <-done; !bytes.Equal(got, []byte("probe")) { + t.Fatalf("pong payload %q", got) + } +} + +func TestClientFramesAreMasked(t *testing.T) { + masked := make(chan bool, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrade(w, r) + if err != nil { + return + } + defer conn.Close() + head := make([]byte, 2) + if _, err := conn.br.Read(head); err != nil { + return + } + masked <- head[1]&0x80 != 0 + })) + defer server.Close() + conn := dialTest(t, server) + if err := conn.WriteText("x"); err != nil { + t.Fatal(err) + } + if !<-masked { + t.Fatal("client frame was not masked") + } +} + +func TestCloseCodeSurfaces(t *testing.T) { + server := wsServer(t, func(c *Conn) { _ = c.WriteClose(CloseRetryOwner, "owner") }) + conn := dialTest(t, server) + _, err := conn.ReadMessage() + var closeErr *CloseError + if !errors.As(err, &closeErr) || closeErr.Code != CloseRetryOwner || closeErr.Reason != "owner" { + t.Fatalf("got %v", err) + } +} + +func TestOversizeFrameRejectedWithClose(t *testing.T) { + server := wsServer(t, func(c *Conn) { + _ = c.writeFrame(true, opText, []byte(strings.Repeat("b", MaxWSFramePayload+1))) + _, _, _, _ = c.readFrame() + }) + conn := dialTest(t, server) + _, err := conn.ReadMessage() + var closeErr *CloseError + if !errors.As(err, &closeErr) || closeErr.Code != CloseTooLarge { + t.Fatalf("got %v", err) + } +} + +func TestOversizeMessageAcrossFragmentsRejected(t *testing.T) { + server := wsServer(t, func(c *Conn) { + chunk := []byte(strings.Repeat("c", MaxWSFramePayload)) + _ = c.writeFrame(false, opText, chunk) + _ = c.writeFrame(true, opContinuation, chunk) + _, _, _, _ = c.readFrame() + }) + conn := dialTest(t, server) + _, err := conn.ReadMessage() + var closeErr *CloseError + if !errors.As(err, &closeErr) || closeErr.Code != CloseTooLarge { + t.Fatalf("got %v", err) + } +} + +func TestBinaryAndBadUTF8Rejected(t *testing.T) { + cases := []struct { + name string + op byte + payload []byte + code int + }{ + {"binary", opBinary, []byte("x"), CloseUnsupported}, + {"utf8", opText, []byte{0xff, 0xfe}, CloseInvalidPayload}, + {"continuation", opContinuation, []byte("x"), CloseProtocolError}, + } + for _, item := range cases { + server := wsServer(t, func(c *Conn) { + _ = c.writeFrame(true, item.op, item.payload) + _, _, _, _ = c.readFrame() + }) + conn := dialTest(t, server) + _, err := conn.ReadMessage() + var closeErr *CloseError + if !errors.As(err, &closeErr) || closeErr.Code != item.code { + t.Fatalf("%s: got %v", item.name, err) + } + } +} + +func TestHandshakeRefusal(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "no", http.StatusForbidden) + })) + defer server.Close() + _, err := Dial(context.Background(), "ws"+strings.TrimPrefix(server.URL, "http")) + if err == nil || !strings.Contains(err.Error(), "403") { + t.Fatalf("got %v", err) + } +} diff --git a/nodes/extension/README.md b/nodes/extension/README.md new file mode 100644 index 00000000..a2675c31 --- /dev/null +++ b/nodes/extension/README.md @@ -0,0 +1,70 @@ +# DevPlace Node, browser extension + +A Manifest V3 extension for Chrome and Firefox from one source tree. It lets a DevPlace member donate a little of their connection to the community node network, in **fetch mode only** (an extension has no raw sockets). The protocol is `devplacepy/services/nodes/spec.py`, served at `GET /nodes/spec.json`; this extension never restates a number from it except in `src/spec.js`, which a test compares against `spec.py`. + +## Install (sideload) + +1. Chrome: `chrome://extensions`, enable Developer mode, Load unpacked, pick this directory. Firefox: `about:debugging#/runtime/this-firefox`, Load Temporary Add-on, pick `manifest.json`. A temporary Firefox add-on is removed on restart. +2. A disclosure page opens on first install. Read it and accept. +3. On your DevPlace profile give consent to bandwidth sharing, register a device and copy the `dpn_...` secret (shown once). +4. Open the popup, enter the server URL and the secret, grant the host permission for that server when asked, then switch sharing on. + +Packaged build: `./build.sh ` writes `devplace-node-extension.zip` and `devplace-node-extension.zip.sha256`. The zip holds `manifest.json`, `src/`, `ui/` and this file; tests are not shipped. + +## Chrome and Firefox from one tree + +`manifest.json` declares both `background.service_worker` and `background.scripts`, both pointing at the same ES module. + +| Key | Used by | +|-----|---------| +| `background.service_worker` | Chrome (service worker). Firefox ignores it with a console warning. | +| `background.scripts` | Firefox (event page). Chrome ignores it when a service worker is present. | +| `browser_specific_settings.gecko` | Firefox only (add-on id, minimum 128). Chrome warns about the unknown key and ignores it. | + +Both use the `chrome.*` namespace with promises. Firefox store submission additionally needs a `data_collection_permissions` declaration, not added yet. + +## How it works + +- The background worker holds the WebSocket. It exchanges the device secret for a 30 second ticket (`POST /nodes/ticket`, `Authorization: Bearer dpn_...`) and opens `WS /nodes/ws?ticket=...`, then sends `hello` with `kind: "extension"` and `modes: ["fetch"]`. +- A `pong` is sent every `ping_seconds` as a keepalive and for every server `ping`. A `chrome.alarms` alarm every minute is the reconnect backstop when the worker has been suspended. +- Reconnects use full-jitter backoff between `reconnect_min_seconds` and `reconnect_max_seconds`. Close `4013` retries at the minimum delay. A kick (`4403`), a consent or device refusal (`1008` with a reason) or a rejected secret (HTTP 401/403) **halts** and shows the reason; nothing hammers the server. +- State (connection, last error) is in `chrome.storage.session`. Config (server URL, secret, limits, disclosure flag) and the usage counters are in `chrome.storage.local`. +- The secret is never logged and never shown after saving; the popup shows its first 8 characters only. + +## Safety enforced in the extension, before every request + +Independent of anything the server says: + +- allowlist: the hosts baked into `src/spec.js` (`reddit.com`, `redd.it` and their subdomains, dot-safe suffix match). A refreshed list from `welcome` is accepted only entry by entry when it is equal or narrower (same or sub-suffix, methods and ports subsets, fetch mode present); a widening entry is dropped and reported in the popup +- https only, port 443 only (the spec's 80 is never used because plain http is refused), no credentials in the URL, method must be permitted (GET) +- `fetch` with `credentials: "omit"`, `redirect: "manual"`, `cache: "no-store"`, no referrer; only `accept` and `accept-language` are forwarded from the job, so no cookie or authorization header can be injected +- each redirect hop is re-checked against the allowlist, at most 5 hops; a redirect that leaves the allowlist is refused with `redirect_not_allowed` +- response body capped at `max_body_bytes` and by the remaining daily budget, with `truncated: true`; `set-cookie` style response headers are dropped +- timeout by `AbortController`, daily bandwidth cap, concurrency cap and active hours from the local settings, each answered with its refusal code (`timeout`, `bandwidth_exhausted`, `concurrency_exceeded`, `outside_hours`). A server value can only lower these, never raise them + +Known limit: browsers hide the target of a `redirect: "manual"` response (an opaque redirect), so the extension cannot read where it points. It fails closed with `redirect_not_allowed`. A followable redirect is only handled when the response exposes its `Location`. + +## Permissions + +| Permission | Why | +|------------|-----| +| `storage` | config, usage counters, connection state | +| `alarms` | reconnect backstop while the worker is suspended | +| `host_permissions`: `reddit.com`, `*.reddit.com`, `redd.it`, `*.redd.it` over https | fetch the allowlisted destinations without CORS. These are exactly the destinations in `spec.py`. | +| `optional_host_permissions`: any https origin, `localhost`, `127.0.0.1` over http | requested at connect time for the one server origin you enter, so the ticket request works. Not granted at install. | + +No `tabs`, `cookies`, `history`, `webRequest` or `` permission. + +## What is and is not shared + +Shared: the URL of the requested public page, the page content that comes back, your public IP address as seen by the destination, and request and byte counts. Not used: your cookies, logins, browsing history, tabs, other sites, any host outside the list, any non-https or non-443 destination. See `ui/disclosure.html`, shown on first install. + +## Tests + +`node --test nodes/extension/test/*.mjs` (no dependencies). They cover Policy, Limits, Fetcher with a stubbed `fetch`, WsClient with a fake WebSocket, and a drift check of `src/spec.js` against `spec.py`. Not covered by automated tests: `background.js` and the popup, which need a real browser. + +## Gaps + +- No icons yet; the browser shows a default. +- The `welcome` frame carries no device label, so the label in the popup is local only. +- Not tested in a real Chrome or Firefox session in this change. diff --git a/nodes/extension/build.sh b/nodes/extension/build.sh new file mode 100755 index 00000000..b56ece03 --- /dev/null +++ b/nodes/extension/build.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -eu + +if [ "$#" -ne 1 ]; then + echo "usage: build.sh " >&2 + exit 2 +fi + +here=$(cd "$(dirname "$0")" && pwd) +mkdir -p "$1" +out=$(cd "$1" && pwd) +name=devplace-node-extension.zip +archive="$out/$name" +rm -f "$archive" + +cd "$here" +if command -v zip >/dev/null 2>&1; then + zip -q -X -r "$archive" manifest.json src ui README.md +else + python3 - "$archive" <<'PY' +import sys +import zipfile +from pathlib import Path + +archive = sys.argv[1] +paths = [Path("manifest.json"), Path("README.md")] +for folder in ("src", "ui"): + paths.extend(sorted(item for item in Path(folder).rglob("*") if item.is_file())) +with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as bundle: + for path in paths: + bundle.write(path, path.as_posix()) +PY +fi + +cd "$out" +if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$name" > "$name.sha256" +else + shasum -a 256 "$name" > "$name.sha256" +fi +cat "$name.sha256" diff --git a/nodes/extension/manifest.json b/nodes/extension/manifest.json new file mode 100644 index 00000000..d5d3a7ef --- /dev/null +++ b/nodes/extension/manifest.json @@ -0,0 +1,33 @@ +{ + "manifest_version": 3, + "name": "DevPlace Node", + "version": "0.1.0", + "description": "Donate a little of your connection to the DevPlace community node network. Fetch mode only, allowlisted hosts only, never your cookies.", + "background": { + "service_worker": "src/background.js", + "scripts": ["src/background.js"], + "type": "module" + }, + "action": { + "default_title": "DevPlace Node", + "default_popup": "ui/popup.html" + }, + "permissions": ["storage", "alarms"], + "host_permissions": [ + "https://reddit.com/*", + "https://*.reddit.com/*", + "https://redd.it/*", + "https://*.redd.it/*" + ], + "optional_host_permissions": [ + "https://*/*", + "http://localhost/*", + "http://127.0.0.1/*" + ], + "browser_specific_settings": { + "gecko": { + "id": "node@devplace.net", + "strict_min_version": "128.0" + } + } +} diff --git a/nodes/extension/package.json b/nodes/extension/package.json new file mode 100644 index 00000000..92502414 --- /dev/null +++ b/nodes/extension/package.json @@ -0,0 +1,5 @@ +{ + "name": "devplace-node-extension", + "private": true, + "type": "module" +} diff --git a/nodes/extension/src/Config.js b/nodes/extension/src/Config.js new file mode 100644 index 00000000..35a92a6e --- /dev/null +++ b/nodes/extension/src/Config.js @@ -0,0 +1,126 @@ +import { LIMITS } from "./spec.js"; + +const KEY = "config"; +const SECRET_PREFIX = "dpn_"; +const SECRET_MIN_CHARS = 16; +const SECRET_MAX_CHARS = 128; +const LABEL_MAX_CHARS = 64; +const SHOWN_SECRET_CHARS = 8; +const LOOPBACK = ["localhost", "127.0.0.1", "[::1]"]; + +const DEFAULTS = Object.freeze({ + serverUrl: "", + secret: "", + label: "", + disclosed: false, + enabled: false, + bandwidthMb: LIMITS.default_donor_mb_per_day, + maxConcurrent: LIMITS.max_concurrent_per_node, + activeFrom: "", + activeTo: "", +}); + +export class ConfigError extends Error {} + +export class Config { + constructor(area) { + this.area = area; + this.data = { ...DEFAULTS }; + } + + static validateServerUrl(raw) { + let url; + try { + url = new URL(String(raw).trim()); + } catch (error) { + throw new ConfigError("The server URL is not a valid URL"); + } + const loopback = LOOPBACK.includes(url.hostname); + if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback)) { + throw new ConfigError("The server URL must use https"); + } + if (url.username || url.password) { + throw new ConfigError("The server URL must not contain credentials"); + } + return url.origin; + } + + static validateSecret(raw) { + const secret = String(raw).trim(); + if ( + !secret.startsWith(SECRET_PREFIX) || + secret.length < SECRET_MIN_CHARS || + secret.length > SECRET_MAX_CHARS || + /\s/.test(secret) + ) { + throw new ConfigError("That is not a DevPlace device secret (it starts with dpn_)"); + } + return secret; + } + + async load() { + const stored = (await this.area.get(KEY))[KEY]; + this.data = { ...DEFAULTS, ...(stored && typeof stored === "object" ? stored : {}) }; + } + + async save() { + await this.area.set({ [KEY]: { ...this.data } }); + } + + get hasSecret() { + return Boolean(this.data.secret); + } + + get secretPrefix() { + return this.data.secret ? this.data.secret.slice(0, SHOWN_SECRET_CHARS) + "..." : ""; + } + + credentials() { + return { serverUrl: this.data.serverUrl, secret: this.data.secret }; + } + + view() { + const { secret, ...rest } = this.data; + return { ...rest, hasSecret: this.hasSecret, secretPrefix: this.secretPrefix }; + } + + async connect({ serverUrl, secret, label }) { + this.data.serverUrl = Config.validateServerUrl(serverUrl); + this.data.secret = Config.validateSecret(secret); + this.data.label = String(label || "").trim().slice(0, LABEL_MAX_CHARS); + await this.save(); + } + + async acceptDisclosure() { + this.data.disclosed = true; + await this.save(); + } + + async setEnabled(enabled) { + if (enabled && !this.data.disclosed) { + throw new ConfigError("Read and accept the disclosure first"); + } + if (enabled && !this.hasSecret) { + throw new ConfigError("Connect a device secret first"); + } + this.data.enabled = Boolean(enabled); + await this.save(); + } + + async saveSettings(values) { + for (const key of ["bandwidthMb", "maxConcurrent", "activeFrom", "activeTo"]) { + if (key in values) { + this.data[key] = values[key]; + } + } + await this.save(); + } + + async forget() { + this.data.secret = ""; + this.data.serverUrl = ""; + this.data.label = ""; + this.data.enabled = false; + await this.save(); + } +} diff --git a/nodes/extension/src/Fetcher.js b/nodes/extension/src/Fetcher.js new file mode 100644 index 00000000..a4dcf636 --- /dev/null +++ b/nodes/extension/src/Fetcher.js @@ -0,0 +1,193 @@ +import { + DROPPED_RESPONSE_HEADERS, + FETCH_FAILED, + FORWARDED_REQUEST_HEADERS, + MAX_BODY_CHUNKS, + MAX_HEADER_VALUE_CHARS, + MAX_REDIRECTS, + MAX_RESPONSE_HEADERS, + REFUSALS, +} from "./spec.js"; + +const REDIRECT_STATUSES = [301, 302, 303, 307, 308]; +const REQUEST_OVERHEAD_BYTES = 256; +const BASE64_CHUNK = 0x8000; + +export function toBase64(bytes) { + let text = ""; + for (let index = 0; index < bytes.length; index += BASE64_CHUNK) { + text += String.fromCharCode.apply(null, bytes.subarray(index, index + BASE64_CHUNK)); + } + return btoa(text); +} + +function concat(parts, total) { + const joined = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + joined.set(part, offset); + offset += part.length; + } + return joined; +} + +export class Fetcher { + constructor({ policy, limits, usage, fetchImpl, clock }) { + this.policy = policy; + this.limits = limits; + this.usage = usage; + this.fetchImpl = fetchImpl || ((...args) => globalThis.fetch(...args)); + this.clock = clock || (() => Date.now()); + } + + setPolicy(policy) { + this.policy = policy; + } + + failure(jobId, code) { + return { type: "job_error", job_id: jobId, code }; + } + + requestHeaders(raw) { + const headers = {}; + if (!raw || typeof raw !== "object") { + return headers; + } + for (const [name, value] of Object.entries(raw)) { + const key = name.toLowerCase(); + if (FORWARDED_REQUEST_HEADERS.includes(key) && typeof value === "string") { + headers[key] = value; + } + } + return headers; + } + + responseHeaders(response) { + const headers = {}; + let count = 0; + response.headers.forEach((value, name) => { + const key = name.toLowerCase(); + if (DROPPED_RESPONSE_HEADERS.includes(key) || count >= MAX_RESPONSE_HEADERS) { + return; + } + headers[key] = String(value).slice(0, MAX_HEADER_VALUE_CHARS); + count += 1; + }); + return headers; + } + + isRedirect(response) { + return response.type === "opaqueredirect" || REDIRECT_STATUSES.includes(response.status); + } + + async readBody(response, cap) { + if (!response.body || typeof response.body.getReader !== "function") { + const whole = new Uint8Array(await response.arrayBuffer()); + return whole.length > cap + ? { bytes: whole.slice(0, cap), truncated: true } + : { bytes: whole, truncated: false }; + } + const reader = response.body.getReader(); + const parts = []; + let total = 0; + let truncated = false; + for (let chunks = 0; chunks < MAX_BODY_CHUNKS; chunks += 1) { + const { done, value } = await reader.read(); + if (done) { + return { bytes: concat(parts, total), truncated }; + } + const room = cap - total; + if (value.length > room) { + parts.push(value.subarray(0, room)); + total += room; + truncated = true; + break; + } + parts.push(value); + total += value.length; + } + truncated = true; + await reader.cancel(); + return { bytes: concat(parts, total), truncated }; + } + + async follow(job, verdict, signal) { + let current = verdict.url; + const headers = this.requestHeaders(job.headers); + for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) { + const response = await this.fetchImpl(current.href, { + method: verdict.method, + headers, + credentials: "omit", + redirect: "manual", + cache: "no-store", + referrerPolicy: "no-referrer", + signal, + }); + if (!this.isRedirect(response)) { + return { response, final: current }; + } + const location = response.headers.get("location"); + if (response.type === "opaqueredirect" || !location || hop === MAX_REDIRECTS) { + if (response.body && typeof response.body.cancel === "function") { + await response.body.cancel(); + } + return { refused: REFUSALS.REDIRECT_NOT_ALLOWED }; + } + const next = this.policy.checkRedirect(location, current, verdict.method); + if (response.body && typeof response.body.cancel === "function") { + await response.body.cancel(); + } + if (!next.ok) { + return { refused: REFUSALS.REDIRECT_NOT_ALLOWED }; + } + current = next.url; + } + return { refused: REFUSALS.REDIRECT_NOT_ALLOWED }; + } + + async execute(job) { + const verdict = this.policy.check(job.method, job.url); + if (!verdict.ok) { + return this.failure(job.job_id, verdict.code); + } + const refusal = this.limits.admit(this.usage.usedBytes); + if (refusal) { + return this.failure(job.job_id, refusal); + } + const started = this.clock(); + const up = job.url.length + REQUEST_OVERHEAD_BYTES; + const controller = new AbortController(); + const timer = setTimeout( + () => controller.abort(), + this.limits.timeoutSeconds(job.timeout) * 1000, + ); + try { + const outcome = await this.follow(job, verdict, controller.signal); + if (outcome.refused) { + await this.usage.record({ ok: false, up, down: 0 }); + return this.failure(job.job_id, outcome.refused); + } + const budget = this.limits.remainingBytes(this.usage.usedBytes + up); + const cap = Math.min(this.limits.maxBodyBytes, budget); + const body = await this.readBody(outcome.response, cap); + const down = body.bytes.length; + await this.usage.record({ ok: true, up, down }); + return { + type: "job_result", + job_id: job.job_id, + status: outcome.response.status, + headers: this.responseHeaders(outcome.response), + body_b64: toBase64(body.bytes), + truncated: body.truncated, + elapsed_ms: Math.max(0, Math.round(this.clock() - started)), + }; + } catch (error) { + await this.usage.record({ ok: false, up, down: 0 }); + return this.failure(job.job_id, controller.signal.aborted ? REFUSALS.TIMEOUT : FETCH_FAILED); + } finally { + clearTimeout(timer); + this.limits.release(); + } + } +} diff --git a/nodes/extension/src/Limits.js b/nodes/extension/src/Limits.js new file mode 100644 index 00000000..e93b05b3 --- /dev/null +++ b/nodes/extension/src/Limits.js @@ -0,0 +1,167 @@ +import { LIMITS, REFUSALS } from "./spec.js"; + +const MB = 1024 * 1024; +const HOURS = /^([01]\d|2[0-3]):([0-5]\d)$/; +const CAP_KEYS = [ + "max_frame_bytes", + "max_body_bytes", + "max_concurrent_per_node", + "job_timeout_seconds", + "ping_seconds", + "url_max_chars", +]; + +function isCount(value) { + return typeof value === "number" && Number.isInteger(value) && value >= 1; +} + +function clampInt(value, low, high, fallback) { + if (!isCount(value)) { + return fallback; + } + return Math.min(high, Math.max(low, value)); +} + +function minutes(text) { + const match = HOURS.exec(text); + return match ? Number(match[1]) * 60 + Number(match[2]) : null; +} + +export class Limits { + constructor(options = {}) { + this.baked = options.baked || LIMITS; + this.clock = options.clock || (() => new Date()); + this.caps = {}; + for (const key of CAP_KEYS) { + this.caps[key] = this.baked[key]; + } + this.reconnectMin = this.baked.reconnect_min_seconds; + this.reconnectMax = this.baked.reconnect_max_seconds; + this.serverBandwidthMb = this.baked.max_donor_mb_per_day; + this.serverConcurrent = this.baked.max_concurrent_per_node; + this.local = Limits.normaliseLocal({}, this.baked); + this.inflight = 0; + if (options.local) { + this.setLocal(options.local); + } + } + + static normaliseLocal(values, baked = LIMITS) { + const from = minutes(values.activeFrom || ""); + const to = minutes(values.activeTo || ""); + const both = from !== null && to !== null; + return { + bandwidthMb: clampInt( + values.bandwidthMb, + 1, + baked.max_donor_mb_per_day, + baked.default_donor_mb_per_day, + ), + maxConcurrent: clampInt( + values.maxConcurrent, + 1, + baked.max_concurrent_per_node, + baked.max_concurrent_per_node, + ), + activeFrom: both ? values.activeFrom : "", + activeTo: both ? values.activeTo : "", + }; + } + + setLocal(values) { + this.local = Limits.normaliseLocal({ ...this.local, ...values }, this.baked); + } + + applyWelcome(welcomeLimits, serverSettings) { + const incoming = welcomeLimits && typeof welcomeLimits === "object" ? welcomeLimits : {}; + for (const key of CAP_KEYS) { + this.caps[key] = Math.min(this.baked[key], isCount(incoming[key]) ? incoming[key] : this.baked[key]); + } + const ceiling = this.baked.reconnect_max_seconds; + this.reconnectMin = clampInt(incoming.reconnect_min_seconds, 1, ceiling, this.baked.reconnect_min_seconds); + this.reconnectMax = clampInt(incoming.reconnect_max_seconds, this.reconnectMin, ceiling, ceiling); + this.applyServerSettings(serverSettings); + } + + applyServerSettings(settings) { + const values = settings && typeof settings === "object" ? settings : {}; + this.serverBandwidthMb = clampInt( + values.bandwidth_mb_day, + 1, + this.baked.max_donor_mb_per_day, + this.baked.max_donor_mb_per_day, + ); + this.serverConcurrent = clampInt( + values.max_concurrent, + 1, + this.baked.max_concurrent_per_node, + this.baked.max_concurrent_per_node, + ); + } + + get maxBodyBytes() { + return this.caps.max_body_bytes; + } + + get maxFrameBytes() { + return this.caps.max_frame_bytes; + } + + get urlMaxChars() { + return this.caps.url_max_chars; + } + + get pingSeconds() { + return this.caps.ping_seconds; + } + + get concurrency() { + return Math.min(this.caps.max_concurrent_per_node, this.local.maxConcurrent, this.serverConcurrent); + } + + get bandwidthBytes() { + return Math.min(this.local.bandwidthMb, this.serverBandwidthMb) * MB; + } + + timeoutSeconds(requested) { + const asked = isCount(requested) ? requested : this.caps.job_timeout_seconds; + return Math.min(asked, this.caps.job_timeout_seconds); + } + + helloLimits() { + return { max_concurrent: this.concurrency }; + } + + remainingBytes(usedBytes) { + return Math.max(0, this.bandwidthBytes - usedBytes); + } + + inActiveHours() { + const from = minutes(this.local.activeFrom); + const to = minutes(this.local.activeTo); + if (from === null || to === null || from === to) { + return true; + } + const now = this.clock(); + const current = now.getHours() * 60 + now.getMinutes(); + return from < to ? current >= from && current < to : current >= from || current < to; + } + + admit(usedBytes) { + if (!this.inActiveHours()) { + return REFUSALS.OUTSIDE_HOURS; + } + if (this.remainingBytes(usedBytes) <= 0) { + return REFUSALS.BANDWIDTH_EXHAUSTED; + } + if (this.inflight >= this.concurrency) { + return REFUSALS.CONCURRENCY_EXCEEDED; + } + this.inflight += 1; + return null; + } + + release() { + this.inflight = Math.max(0, this.inflight - 1); + } +} diff --git a/nodes/extension/src/Policy.js b/nodes/extension/src/Policy.js new file mode 100644 index 00000000..9c390031 --- /dev/null +++ b/nodes/extension/src/Policy.js @@ -0,0 +1,128 @@ +import { + ALLOWED_PORTS, + DESTINATIONS, + HTTPS_PORT, + LIMITS, + MODE_FETCH, + REFUSALS, +} from "./spec.js"; + +const HTTPS = "https:"; + +function isSubset(candidate, allowed) { + return Array.isArray(candidate) && candidate.every((item) => allowed.includes(item)); +} + +function normaliseHost(host) { + return String(host).trim().toLowerCase().replace(/\.+$/, ""); +} + +export class Policy { + constructor(destinations = DESTINATIONS, limits = LIMITS) { + this.destinations = destinations; + this.urlMaxChars = limits.url_max_chars; + } + + static narrow(baked, refreshed) { + const accepted = []; + const rejected = []; + if (!Array.isArray(refreshed)) { + return { destinations: [], rejected: ["destinations is not a list"] }; + } + for (const item of refreshed) { + const suffix = item && typeof item.suffix === "string" ? normaliseHost(item.suffix) : ""; + const parent = baked.find( + (entry) => suffix === entry.suffix || suffix.endsWith("." + entry.suffix), + ); + if (!parent) { + rejected.push(`${suffix || "?"}: not in the baked-in allowlist`); + continue; + } + const methods = Array.isArray(item.methods) ? item.methods.map((m) => String(m).toUpperCase()) : []; + const ports = Array.isArray(item.ports) ? item.ports : []; + const modes = Array.isArray(item.modes) ? item.modes : []; + if (!methods.length || !isSubset(methods, parent.methods)) { + rejected.push(`${suffix}: methods widen the baked-in list`); + continue; + } + if (!ports.length || !isSubset(ports, parent.ports) || !isSubset(ports, ALLOWED_PORTS)) { + rejected.push(`${suffix}: ports widen the baked-in list`); + continue; + } + if (!modes.includes(MODE_FETCH)) { + rejected.push(`${suffix}: not available in fetch mode`); + continue; + } + accepted.push({ + suffix, + adapter: String(item.adapter || parent.adapter), + methods, + ports, + modes, + max_rps: typeof item.max_rps === "number" ? item.max_rps : parent.max_rps, + }); + } + return { destinations: accepted, rejected }; + } + + static fromRefreshed(baked, refreshed, limits = LIMITS) { + const narrowed = Policy.narrow(baked, refreshed); + return { policy: new Policy(narrowed.destinations, limits), rejected: narrowed.rejected }; + } + + destinationFor(host) { + const name = normaliseHost(host); + return ( + this.destinations.find((item) => name === item.suffix || name.endsWith("." + item.suffix)) || null + ); + } + + refuse(code, reason) { + return { ok: false, code, reason }; + } + + check(method, rawUrl) { + if (typeof rawUrl !== "string" || !rawUrl || rawUrl.length > this.urlMaxChars) { + return this.refuse(REFUSALS.HOST_NOT_ALLOWED, "url is missing or too long"); + } + let url; + try { + url = new URL(rawUrl); + } catch (error) { + return this.refuse(REFUSALS.HOST_NOT_ALLOWED, "url does not parse"); + } + if (url.protocol !== HTTPS) { + return this.refuse(REFUSALS.HOST_NOT_ALLOWED, "only https is allowed"); + } + if (url.username || url.password) { + return this.refuse(REFUSALS.HOST_NOT_ALLOWED, "credentials in the url are refused"); + } + const destination = this.destinationFor(url.hostname); + if (!destination) { + return this.refuse(REFUSALS.HOST_NOT_ALLOWED, `${url.hostname} is not on the allowlist`); + } + const port = url.port === "" ? HTTPS_PORT : Number(url.port); + if (port !== HTTPS_PORT || !destination.ports.includes(port) || !ALLOWED_PORTS.includes(port)) { + return this.refuse(REFUSALS.PORT_NOT_ALLOWED, `port ${port} is not allowed over https`); + } + const verb = String(method || "").toUpperCase(); + if (!destination.methods.includes(verb)) { + return this.refuse(REFUSALS.METHOD_NOT_ALLOWED, `${verb || "?"} is not permitted`); + } + return { ok: true, url, destination, method: verb }; + } + + checkRedirect(location, base, method) { + let target; + try { + target = new URL(location, base); + } catch (error) { + return this.refuse(REFUSALS.REDIRECT_NOT_ALLOWED, "redirect target does not parse"); + } + const verdict = this.check(method, target.href); + if (!verdict.ok) { + return this.refuse(REFUSALS.REDIRECT_NOT_ALLOWED, verdict.reason); + } + return verdict; + } +} diff --git a/nodes/extension/src/State.js b/nodes/extension/src/State.js new file mode 100644 index 00000000..9d7a84bf --- /dev/null +++ b/nodes/extension/src/State.js @@ -0,0 +1,37 @@ +const KEY = "state"; + +export const STATES = Object.freeze({ + OFF: "off", + CONNECTING: "connecting", + ONLINE: "online", + BACKOFF: "backoff", + HALTED: "halted", +}); + +export class State { + constructor(area, clock = () => Date.now()) { + this.area = area; + this.clock = clock; + this.data = { state: STATES.OFF, detail: "", since: clock(), deviceUid: "", lastError: "" }; + this.chain = Promise.resolve(); + } + + async load() { + const stored = (await this.area.get(KEY))[KEY]; + if (stored && typeof stored === "object") { + this.data = { ...this.data, ...stored }; + } + } + + snapshot() { + return { ...this.data }; + } + + set(patch) { + const changed = patch.state && patch.state !== this.data.state; + this.data = { ...this.data, ...patch, since: changed ? this.clock() : this.data.since }; + const payload = { ...this.data }; + this.chain = this.chain.then(() => this.area.set({ [KEY]: payload })); + return this.chain; + } +} diff --git a/nodes/extension/src/Usage.js b/nodes/extension/src/Usage.js new file mode 100644 index 00000000..d8558f5e --- /dev/null +++ b/nodes/extension/src/Usage.js @@ -0,0 +1,81 @@ +const KEY = "usage"; + +function dayKey(date) { + const month = String(date.getMonth() + 1).padStart(2, "0"); + const day = String(date.getDate()).padStart(2, "0"); + return `${date.getFullYear()}-${month}-${day}`; +} + +function empty(day) { + return { + day, + requests: 0, + failed: 0, + bytesUp: 0, + bytesDown: 0, + lifetimeRequests: 0, + lifetimeBytes: 0, + }; +} + +export class Usage { + constructor(area, clock = () => new Date()) { + this.area = area; + this.clock = clock; + this.data = empty(dayKey(clock())); + this.chain = Promise.resolve(); + } + + async load() { + const stored = (await this.area.get(KEY))[KEY]; + if (stored && typeof stored === "object") { + this.data = { ...empty(dayKey(this.clock())), ...stored }; + } + this.roll(); + } + + roll() { + const today = dayKey(this.clock()); + if (this.data.day !== today) { + this.data = { + ...empty(today), + lifetimeRequests: this.data.lifetimeRequests, + lifetimeBytes: this.data.lifetimeBytes, + }; + } + } + + snapshot() { + this.roll(); + return { ...this.data }; + } + + get usedBytes() { + this.roll(); + return this.data.bytesUp + this.data.bytesDown; + } + + record({ ok, up, down }) { + this.roll(); + const bytesUp = Math.max(0, Math.floor(up) || 0); + const bytesDown = Math.max(0, Math.floor(down) || 0); + this.data.requests += 1; + this.data.failed += ok ? 0 : 1; + this.data.bytesUp += bytesUp; + this.data.bytesDown += bytesDown; + this.data.lifetimeRequests += 1; + this.data.lifetimeBytes += bytesUp + bytesDown; + return this.persist(); + } + + persist() { + const payload = { ...this.data }; + this.chain = this.chain.then(() => this.area.set({ [KEY]: payload })); + return this.chain; + } + + async reset() { + this.data = empty(dayKey(this.clock())); + await this.persist(); + } +} diff --git a/nodes/extension/src/WsClient.js b/nodes/extension/src/WsClient.js new file mode 100644 index 00000000..4faec080 --- /dev/null +++ b/nodes/extension/src/WsClient.js @@ -0,0 +1,375 @@ +import { Policy } from "./Policy.js"; +import { STATES } from "./State.js"; +import { + CLIENT_KIND, + CLOSE_CAPACITY, + CLOSE_KICKED, + CLOSE_NORMAL, + CLOSE_PROTOCOL, + CLOSE_RETRY_OWNER, + DESTINATIONS, + FRAME_CEILING_BYTES, + HALTING_CLOSE_REASONS_EXEMPT, + MAX_BACKOFF_EXPONENT, + MODE_FETCH, + PROTOCOL_VERSION, + REFUSALS, + SERVER_FRAMES, +} from "./spec.js"; + +const OPEN = 1; +const POLICY_CLOSE = 1008; +const TICKET_PATH = "/nodes/ticket"; +const WS_PATH = "/nodes/ws"; +const HALT_STATUSES = [401, 403]; + +class FrameFault extends Error {} + +export class WsClient { + constructor(options) { + this.config = options.config; + this.limits = options.limits; + this.fetcher = options.fetcher; + this.state = options.state; + this.fetchImpl = options.fetchImpl || ((...args) => globalThis.fetch(...args)); + this.WebSocketImpl = options.WebSocketImpl || globalThis.WebSocket; + this.timers = options.timers || globalThis; + this.random = options.random || Math.random; + this.clientVersion = options.clientVersion || "0.0.0"; + this.platform = options.platform || "chrome"; + this.baked = options.baked || DESTINATIONS; + this.socket = null; + this.wanted = false; + this.connecting = false; + this.welcomed = false; + this.attempt = 0; + this.retryTimer = null; + this.pingTimer = null; + this.pending = new Set(); + this.generation = 0; + } + + async start() { + this.wanted = true; + await this.ensure(); + } + + async stop(detail = "") { + this.wanted = false; + this.clearTimers(); + this.generation += 1; + this.closeSocket(CLOSE_NORMAL, "stop"); + await this.state.set({ state: STATES.OFF, detail }); + } + + async ensure() { + if (!this.wanted || this.socket || this.connecting || this.retryTimer) { + return; + } + await this.connectOnce(); + } + + clearTimers() { + if (this.retryTimer) { + this.timers.clearTimeout(this.retryTimer); + this.retryTimer = null; + } + if (this.pingTimer) { + this.timers.clearInterval(this.pingTimer); + this.pingTimer = null; + } + } + + closeSocket(code, reason) { + const socket = this.socket; + this.socket = null; + if (socket && socket.readyState <= OPEN) { + socket.close(code, reason); + } + } + + backoffSeconds(fast) { + const low = this.limits.reconnectMin; + if (fast) { + return low; + } + const exponent = Math.min(this.attempt, MAX_BACKOFF_EXPONENT); + const ceiling = Math.min(this.limits.reconnectMax, low * 2 ** exponent); + return Math.max(low, this.random() * ceiling); + } + + async scheduleRetry(detail, fast = false) { + if (!this.wanted) { + return; + } + const seconds = this.backoffSeconds(fast); + this.attempt += 1; + this.clearTimers(); + this.retryTimer = this.timers.setTimeout(() => { + this.retryTimer = null; + this.ensure(); + }, seconds * 1000); + await this.state.set({ state: STATES.BACKOFF, detail, retryInSeconds: Math.round(seconds) }); + } + + async halt(detail) { + this.wanted = false; + this.clearTimers(); + this.generation += 1; + this.closeSocket(CLOSE_NORMAL, "halt"); + await this.state.set({ state: STATES.HALTED, detail, lastError: detail }); + } + + async fetchTicket() { + const { serverUrl, secret } = this.config.credentials(); + const response = await this.fetchImpl(serverUrl + TICKET_PATH, { + method: "POST", + headers: { Authorization: `Bearer ${secret}` }, + credentials: "omit", + cache: "no-store", + }); + if (!response.ok) { + let reason = ""; + try { + reason = (await response.json()).reason || ""; + } catch (error) { + reason = ""; + } + const error = new Error(`ticket refused with ${response.status} ${reason}`.trim()); + error.status = response.status; + throw error; + } + const body = await response.json(); + if (!body || typeof body.ticket !== "string" || !body.ticket) { + throw new Error("ticket response has no ticket"); + } + const target = new URL(typeof body.ws_path === "string" ? body.ws_path : WS_PATH, serverUrl); + if (target.origin !== new URL(serverUrl).origin) { + throw new Error("ticket response points at another origin"); + } + target.protocol = target.protocol === "https:" ? "wss:" : "ws:"; + target.searchParams.set("ticket", body.ticket); + return target.href; + } + + async connectOnce() { + this.connecting = true; + const generation = this.generation; + await this.state.set({ state: STATES.CONNECTING, detail: "" }); + let address; + try { + address = await this.fetchTicket(); + } catch (error) { + this.connecting = false; + if (generation !== this.generation) { + return; + } + if (HALT_STATUSES.includes(error.status)) { + await this.halt(error.message); + return; + } + await this.state.set({ lastError: error.message }); + await this.scheduleRetry(error.message); + return; + } + this.connecting = false; + if (generation !== this.generation) { + return; + } + this.open(address, generation); + } + + open(address, generation) { + this.welcomed = false; + const socket = new this.WebSocketImpl(address); + this.socket = socket; + socket.onopen = () => { + if (generation !== this.generation || this.socket !== socket) { + return; + } + this.sendFrame("hello", { + protocol: PROTOCOL_VERSION, + client_version: this.clientVersion, + kind: CLIENT_KIND, + platform: this.platform, + modes: [MODE_FETCH], + limits: this.limits.helloLimits(), + }); + }; + socket.onmessage = (event) => { + if (this.socket !== socket) { + return; + } + return this.handleMessage(event.data).catch((error) => this.protocolFault(error)); + }; + socket.onclose = (event) => { + if (this.socket !== socket) { + return; + } + this.socket = null; + this.onClosed(event.code, event.reason || ""); + }; + socket.onerror = () => { + this.state.set({ lastError: "websocket error" }); + }; + } + + sendFrame(type, fields = {}) { + if (!this.socket || this.socket.readyState !== OPEN) { + throw new Error(`cannot send ${type}, socket is not open`); + } + const text = JSON.stringify({ type, ...fields }); + const ceiling = FRAME_CEILING_BYTES[type] || FRAME_CEILING_BYTES.default; + if (new TextEncoder().encode(text).length > ceiling) { + throw new Error(`${type} frame exceeds its size cap`); + } + this.socket.send(text); + } + + async protocolFault(error) { + await this.state.set({ lastError: error.message }); + this.closeSocket(CLOSE_PROTOCOL, "protocol"); + this.clearTimers(); + await this.scheduleRetry(error.message); + } + + decode(raw) { + if (typeof raw !== "string") { + throw new FrameFault("binary frames are not part of the protocol"); + } + if (raw.length > this.limits.maxFrameBytes) { + throw new FrameFault("server frame exceeds the size cap"); + } + let frame; + try { + frame = JSON.parse(raw); + } catch (error) { + throw new FrameFault("server frame is not valid JSON"); + } + if (!frame || typeof frame !== "object" || Array.isArray(frame)) { + throw new FrameFault("server frame must be a JSON object"); + } + const shape = SERVER_FRAMES[frame.type]; + if (!shape) { + throw new FrameFault(`unknown server frame ${String(frame.type)}`); + } + for (const [name, kind] of Object.entries(shape)) { + const value = frame[name]; + const good = kind === "array" ? Array.isArray(value) : typeof value === kind && value !== null; + if (!good) { + throw new FrameFault(`frame ${frame.type} field ${name} must be ${kind}`); + } + } + return frame; + } + + async handleMessage(raw) { + const frame = this.decode(raw); + if (frame.type !== "welcome" && !this.welcomed && frame.type !== "kick") { + throw new FrameFault(`${frame.type} arrived before welcome`); + } + if (frame.type === "welcome") { + await this.onWelcome(frame); + } else if (frame.type === "ping") { + this.sendFrame("pong"); + } else if (frame.type === "job") { + this.runJob(frame); + } else if (frame.type === "settings_update") { + this.limits.applyServerSettings(frame.settings); + } else if (frame.type === "kick") { + await this.halt(`kicked by the server: ${frame.reason}`); + } else { + this.sendFrame("tunnel_opened", { + conn_id: frame.conn_id, + ok: false, + code: REFUSALS.HOST_NOT_ALLOWED, + }); + throw new FrameFault(`${frame.type} is a tunnel frame and this client is fetch-only`); + } + } + + async onWelcome(frame) { + if (this.welcomed) { + throw new FrameFault("a second welcome arrived"); + } + this.limits.applyWelcome(frame.limits, frame.settings); + const refreshed = Policy.fromRefreshed(this.baked, frame.destinations, this.limits.baked); + this.fetcher.setPolicy(refreshed.policy); + this.welcomed = true; + this.attempt = 0; + this.startKeepalive(); + const notes = refreshed.rejected.length ? `ignored: ${refreshed.rejected.join("; ")}` : ""; + await this.state.set({ + state: STATES.ONLINE, + detail: notes, + deviceUid: frame.device_uid, + lastError: notes, + }); + } + + startKeepalive() { + if (this.pingTimer) { + this.timers.clearInterval(this.pingTimer); + } + this.pingTimer = this.timers.setInterval(() => { + try { + this.sendFrame("pong"); + } catch (error) { + this.state.set({ lastError: error.message }); + } + }, this.limits.pingSeconds * 1000); + } + + runJob(frame) { + const task = this.fetcher + .execute(frame) + .then((result) => { + this.sendFrame(result.type, result); + return this.reportUsage(result); + }) + .catch((error) => this.state.set({ lastError: error.message })) + .finally(() => this.pending.delete(task)); + this.pending.add(task); + } + + reportUsage(result) { + const down = result.type === "job_result" ? Math.ceil((result.body_b64.length * 3) / 4) : 0; + this.sendFrame("usage", { bytes_up: 0, bytes_down: down }); + } + + async idle() { + while (this.pending.size) { + await Promise.all([...this.pending]); + } + } + + classifyClose(code, reason) { + if (code === CLOSE_KICKED) { + return { action: "halt", detail: `closed by the server: ${reason || "kicked"}` }; + } + if (code === CLOSE_RETRY_OWNER) { + return { action: "retry", fast: true, detail: "server asked for a retry" }; + } + if (code === POLICY_CLOSE && !HALTING_CLOSE_REASONS_EXEMPT.includes(reason)) { + return { action: "halt", detail: `refused by the server: ${reason || "policy"}` }; + } + if (code === CLOSE_CAPACITY) { + return { action: "retry", fast: false, detail: "server at capacity" }; + } + return { action: "retry", fast: false, detail: `closed ${code} ${reason}`.trim() }; + } + + async onClosed(code, reason) { + this.clearTimers(); + this.welcomed = false; + if (!this.wanted) { + return; + } + const verdict = this.classifyClose(code, reason); + if (verdict.action === "halt") { + await this.halt(verdict.detail); + return; + } + await this.scheduleRetry(verdict.detail, verdict.fast); + } +} diff --git a/nodes/extension/src/background.js b/nodes/extension/src/background.js new file mode 100644 index 00000000..ef23cc8b --- /dev/null +++ b/nodes/extension/src/background.js @@ -0,0 +1,135 @@ +import { Config, ConfigError } from "./Config.js"; +import { Fetcher } from "./Fetcher.js"; +import { Limits } from "./Limits.js"; +import { Policy } from "./Policy.js"; +import { State, STATES } from "./State.js"; +import { Usage } from "./Usage.js"; +import { WsClient } from "./WsClient.js"; + +const ALARM = "devplace-node-backstop"; +const ALARM_PERIOD_MINUTES = 1; +const DISCLOSURE_PAGE = "ui/disclosure.html"; + +const config = new Config(chrome.storage.local); +const usage = new Usage(chrome.storage.local); +const state = new State(chrome.storage.session); +const limits = new Limits(); +const fetcher = new Fetcher({ policy: new Policy(), limits, usage }); +const client = new WsClient({ + config, + limits, + fetcher, + state, + clientVersion: chrome.runtime.getManifest().version, + platform: navigator.userAgent.includes("Firefox") ? "firefox" : "chrome", +}); + +let ready = null; + +function boot() { + if (!ready) { + ready = (async () => { + await Promise.all([config.load(), usage.load(), state.load()]); + limits.setLocal(config.data); + })(); + } + return ready; +} + +async function reconcile() { + await boot(); + limits.setLocal(config.data); + if (config.data.enabled && config.data.disclosed && config.hasSecret) { + await client.start(); + } else if (client.wanted || state.data.state !== STATES.OFF) { + await client.stop(); + } +} + +async function status() { + await boot(); + return { + config: config.view(), + state: state.snapshot(), + usage: usage.snapshot(), + live: { concurrency: limits.concurrency, bandwidthBytes: limits.bandwidthBytes }, + }; +} + +const COMMANDS = { + status: async () => status(), + connect: async (message) => { + await config.connect(message); + await client.stop("credentials changed"); + await reconcile(); + return status(); + }, + accept: async () => { + await config.acceptDisclosure(); + return status(); + }, + enable: async (message) => { + await config.setEnabled(Boolean(message.enabled)); + await reconcile(); + return status(); + }, + settings: async (message) => { + await config.saveSettings(message.values || {}); + limits.setLocal(config.data); + return status(); + }, + forget: async () => { + await config.forget(); + await client.stop("device forgotten"); + return status(); + }, + resetUsage: async () => { + await usage.reset(); + return status(); + }, +}; + +async function dispatch(message) { + await boot(); + const command = COMMANDS[message && message.cmd]; + if (!command) { + return { error: `unknown command ${String(message && message.cmd)}` }; + } + try { + return await command(message); + } catch (error) { + if (error instanceof ConfigError) { + return { error: error.message }; + } + await state.set({ lastError: error.message }); + return { error: error.message }; + } +} + +chrome.runtime.onMessage.addListener((message, sender, respond) => { + if (sender.id !== chrome.runtime.id) { + return false; + } + dispatch(message).then(respond); + return true; +}); + +chrome.alarms.onAlarm.addListener((alarm) => { + if (alarm.name === ALARM) { + reconcile().catch((error) => state.set({ lastError: error.message })); + } +}); + +chrome.runtime.onInstalled.addListener((details) => { + chrome.alarms.create(ALARM, { periodInMinutes: ALARM_PERIOD_MINUTES }); + if (details.reason === "install") { + chrome.tabs.create({ url: chrome.runtime.getURL(DISCLOSURE_PAGE) }); + } +}); + +chrome.runtime.onStartup.addListener(() => { + chrome.alarms.create(ALARM, { periodInMinutes: ALARM_PERIOD_MINUTES }); + reconcile().catch((error) => state.set({ lastError: error.message })); +}); + +reconcile().catch((error) => state.set({ lastError: error.message })); diff --git a/nodes/extension/src/spec.js b/nodes/extension/src/spec.js new file mode 100644 index 00000000..346bb8c8 --- /dev/null +++ b/nodes/extension/src/spec.js @@ -0,0 +1,104 @@ +export const PROTOCOL_VERSION = 1; +export const MIN_SUPPORTED_PROTOCOL = 1; +export const CLIENT_KIND = "extension"; +export const MODE_FETCH = "fetch"; + +export const ALLOWED_PORTS = Object.freeze([80, 443]); +export const HTTPS_PORT = 443; + +export const CLOSE_NORMAL = 1000; +export const CLOSE_PROTOCOL = 4400; +export const CLOSE_RETRY_OWNER = 4013; +export const CLOSE_CAPACITY = 4029; +export const CLOSE_KICKED = 4403; +export const CLOSE_POLICY = 1008; + +export const MAX_REDIRECTS = 5; +export const MAX_RESPONSE_HEADERS = 64; +export const MAX_HEADER_VALUE_CHARS = 512; +export const MAX_BODY_CHUNKS = 65536; +export const MAX_BACKOFF_EXPONENT = 16; + +export const LIMITS = Object.freeze({ + max_frame_bytes: 256 * 1024, + max_body_bytes: 2 * 1024 * 1024, + max_concurrent_per_node: 2, + job_timeout_seconds: 25, + default_donor_mb_per_day: 500, + max_donor_mb_per_day: 20 * 1024, + ping_seconds: 20, + reconnect_min_seconds: 1, + reconnect_max_seconds: 60, + url_max_chars: 2048, +}); + +export const FRAME_CEILING_BYTES = Object.freeze({ + job_result: 3 * 1024 * 1024, + default: 256 * 1024, +}); + +export const DESTINATIONS = Object.freeze([ + Object.freeze({ + suffix: "reddit.com", + adapter: "reddit", + methods: Object.freeze(["GET"]), + ports: Object.freeze([80, 443]), + modes: Object.freeze(["tunnel", "fetch"]), + max_rps: 0.5, + }), + Object.freeze({ + suffix: "redd.it", + adapter: "reddit", + methods: Object.freeze(["GET"]), + ports: Object.freeze([80, 443]), + modes: Object.freeze(["tunnel", "fetch"]), + max_rps: 0.5, + }), +]); + +export const REFUSALS = Object.freeze({ + HOST_NOT_ALLOWED: "host_not_allowed", + PORT_NOT_ALLOWED: "port_not_allowed", + ADDRESS_NOT_PUBLIC: "address_not_public", + REDIRECT_NOT_ALLOWED: "redirect_not_allowed", + METHOD_NOT_ALLOWED: "method_not_allowed", + BANDWIDTH_EXHAUSTED: "bandwidth_exhausted", + CONCURRENCY_EXCEEDED: "concurrency_exceeded", + OUTSIDE_HOURS: "outside_hours", + BODY_TOO_LARGE: "body_too_large", + TIMEOUT: "timeout", +}); + +export const FETCH_FAILED = "fetch_failed"; + +export const SERVER_FRAMES = Object.freeze({ + welcome: Object.freeze({ + device_uid: "string", + limits: "object", + destinations: "array", + settings: "object", + }), + ping: Object.freeze({}), + job: Object.freeze({ + job_id: "string", + method: "string", + url: "string", + timeout: "number", + }), + tunnel_open: Object.freeze({ conn_id: "string", host: "string", port: "number" }), + tunnel_data: Object.freeze({ conn_id: "string", b64: "string" }), + tunnel_close: Object.freeze({ conn_id: "string" }), + settings_update: Object.freeze({ settings: "object" }), + kick: Object.freeze({ reason: "string" }), +}); + +export const DROPPED_RESPONSE_HEADERS = Object.freeze([ + "set-cookie", + "set-cookie2", + "authorization", + "proxy-authenticate", +]); + +export const FORWARDED_REQUEST_HEADERS = Object.freeze(["accept", "accept-language"]); + +export const HALTING_CLOSE_REASONS_EXEMPT = Object.freeze(["ticket", "heartbeat", "error", "capacity"]); diff --git a/nodes/extension/test/fetcher.test.mjs b/nodes/extension/test/fetcher.test.mjs new file mode 100644 index 00000000..d63ef12b --- /dev/null +++ b/nodes/extension/test/fetcher.test.mjs @@ -0,0 +1,197 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { Fetcher } from "../src/Fetcher.js"; +import { Limits } from "../src/Limits.js"; +import { Policy } from "../src/Policy.js"; +import { Usage } from "../src/Usage.js"; +import { MAX_REDIRECTS, REFUSALS } from "../src/spec.js"; + +function memoryArea() { + const store = {}; + return { + async get(key) { + return key in store ? { [key]: store[key] } : {}; + }, + async set(values) { + Object.assign(store, values); + }, + }; +} + +function build(fetchImpl, local = {}) { + const limits = new Limits({ local }); + const usage = new Usage(memoryArea()); + const fetcher = new Fetcher({ policy: new Policy(), limits, usage, fetchImpl }); + return { fetcher, limits, usage }; +} + +const job = (url = "https://www.reddit.com/r/x.json", extra = {}) => ({ + job_id: "j1", + method: "GET", + url, + timeout: 5, + ...extra, +}); + +function response(body, init = {}) { + return new Response(body, { status: 200, ...init }); +} + +test("success returns a job_result with base64 body and no donor credentials", async () => { + const calls = []; + const { fetcher, usage } = build(async (url, options) => { + calls.push({ url, options }); + return response("hello", { headers: { "content-type": "text/plain", "set-cookie": "a=b" } }); + }); + const result = await fetcher.execute( + job(undefined, { headers: { Cookie: "x=1", Authorization: "Bearer z", Accept: "text/html" } }), + ); + assert.equal(result.type, "job_result"); + assert.equal(result.status, 200); + assert.equal(Buffer.from(result.body_b64, "base64").toString(), "hello"); + assert.equal(result.truncated, false); + assert.equal(result.headers["content-type"], "text/plain"); + assert.equal("set-cookie" in result.headers, false); + assert.equal(calls[0].options.credentials, "omit"); + assert.equal(calls[0].options.redirect, "manual"); + assert.deepEqual(calls[0].options.headers, { accept: "text/html" }); + assert.equal(usage.snapshot().requests, 1); + assert.equal(usage.snapshot().bytesDown, 5); +}); + +test("a disallowed url never reaches fetch", async () => { + let called = 0; + const { fetcher, limits } = build(async () => { + called += 1; + return response("x"); + }); + for (const url of ["https://evilreddit.com/", "http://reddit.com/", "https://reddit.com@evil.net/"]) { + const result = await fetcher.execute(job(url)); + assert.equal(result.type, "job_error"); + assert.equal(result.code, REFUSALS.HOST_NOT_ALLOWED); + } + assert.equal(called, 0); + assert.equal(limits.inflight, 0); +}); + +test("body over the cap is truncated and flagged", async () => { + const big = "a".repeat(5000); + const { fetcher, limits } = build(async () => response(big)); + limits.caps.max_body_bytes = 1000; + const result = await fetcher.execute(job()); + assert.equal(result.type, "job_result"); + assert.equal(result.truncated, true); + assert.equal(Buffer.from(result.body_b64, "base64").length, 1000); +}); + +test("the local bandwidth budget also bounds the body", async () => { + const { fetcher, usage } = build(async () => response("b".repeat(3 * 1024 * 1024)), { bandwidthMb: 1 }); + const result = await fetcher.execute(job()); + assert.equal(result.truncated, true); + assert.ok(Buffer.from(result.body_b64, "base64").length < 1024 * 1024); + assert.ok(usage.usedBytes <= 1024 * 1024); +}); + +test("exhausted bandwidth refuses before fetching", async () => { + let called = 0; + const { fetcher, usage } = build( + async () => { + called += 1; + return response("x"); + }, + { bandwidthMb: 1 }, + ); + await usage.record({ ok: true, up: 0, down: 1024 * 1024 }); + const result = await fetcher.execute(job()); + assert.equal(result.code, REFUSALS.BANDWIDTH_EXHAUSTED); + assert.equal(called, 0); +}); + +test("concurrency cap refuses the extra job", async () => { + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const { fetcher } = build( + async () => { + await gate; + return response("x"); + }, + { maxConcurrent: 1 }, + ); + const first = fetcher.execute(job()); + const second = await fetcher.execute(job()); + assert.equal(second.code, REFUSALS.CONCURRENCY_EXCEEDED); + release(); + assert.equal((await first).type, "job_result"); +}); + +test("timeout aborts the request and reports timeout", async () => { + const { fetcher, limits } = build( + (url, options) => + new Promise((resolve, reject) => { + options.signal.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError"))); + }), + ); + limits.caps.job_timeout_seconds = 1; + const started = Date.now(); + const result = await fetcher.execute(job(undefined, { timeout: 1 })); + assert.equal(result.code, REFUSALS.TIMEOUT); + assert.ok(Date.now() - started < 3000); + assert.equal(limits.inflight, 0); +}); + +test("a redirect leaving the allowlist is refused and not followed", async () => { + const urls = []; + const { fetcher } = build(async (url) => { + urls.push(url); + return response("", { status: 302, headers: { location: "https://evil.net/steal" } }); + }); + const result = await fetcher.execute(job()); + assert.equal(result.type, "job_error"); + assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED); + assert.equal(urls.length, 1); +}); + +test("an opaque redirect fails closed", async () => { + const opaque = { type: "opaqueredirect", status: 0, headers: new Headers(), body: null }; + const { fetcher } = build(async () => opaque); + const result = await fetcher.execute(job()); + assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED); +}); + +test("a same-allowlist redirect is followed, with the policy checked on every hop", async () => { + const urls = []; + const { fetcher } = build(async (url) => { + urls.push(url); + if (urls.length === 1) { + return response("", { status: 301, headers: { location: "https://old.reddit.com/r/x" } }); + } + return response("final"); + }); + const result = await fetcher.execute(job()); + assert.equal(result.type, "job_result"); + assert.equal(Buffer.from(result.body_b64, "base64").toString(), "final"); + assert.deepEqual(urls, ["https://www.reddit.com/r/x.json", "https://old.reddit.com/r/x"]); +}); + +test("redirect loops are bounded", async () => { + let count = 0; + const { fetcher } = build(async () => { + count += 1; + return response("", { status: 302, headers: { location: "https://www.reddit.com/loop" } }); + }); + const result = await fetcher.execute(job()); + assert.equal(result.code, REFUSALS.REDIRECT_NOT_ALLOWED); + assert.equal(count, MAX_REDIRECTS + 1); +}); + +test("a network failure is a loud job_error", async () => { + const { fetcher, usage } = build(async () => { + throw new TypeError("network down"); + }); + const result = await fetcher.execute(job()); + assert.equal(result.type, "job_error"); + assert.equal(result.code, "fetch_failed"); + assert.equal(usage.snapshot().failed, 1); +}); diff --git a/nodes/extension/test/limits.test.mjs b/nodes/extension/test/limits.test.mjs new file mode 100644 index 00000000..21440aba --- /dev/null +++ b/nodes/extension/test/limits.test.mjs @@ -0,0 +1,91 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { Limits } from "../src/Limits.js"; +import { LIMITS, REFUSALS } from "../src/spec.js"; + +const MB = 1024 * 1024; + +function at(hour, minute = 0) { + return () => new Date(2026, 0, 15, hour, minute); +} + +test("defaults come from the baked spec", () => { + const limits = new Limits(); + assert.equal(limits.concurrency, LIMITS.max_concurrent_per_node); + assert.equal(limits.bandwidthBytes, LIMITS.default_donor_mb_per_day * MB); + assert.equal(limits.maxBodyBytes, LIMITS.max_body_bytes); +}); + +test("local settings are clamped", () => { + const limits = new Limits({ local: { bandwidthMb: 10 ** 9, maxConcurrent: 99 } }); + assert.equal(limits.local.bandwidthMb, LIMITS.max_donor_mb_per_day); + assert.equal(limits.local.maxConcurrent, LIMITS.max_concurrent_per_node); + limits.setLocal({ bandwidthMb: -5, maxConcurrent: 0 }); + assert.equal(limits.local.bandwidthMb, LIMITS.default_donor_mb_per_day); + assert.equal(limits.local.maxConcurrent, LIMITS.max_concurrent_per_node); + limits.setLocal({ activeFrom: "25:99", activeTo: "10:00" }); + assert.equal(limits.local.activeFrom, ""); +}); + +test("a welcome can lower caps and never raise them", () => { + const limits = new Limits(); + limits.applyWelcome( + { + max_body_bytes: 10 ** 9, + max_concurrent_per_node: 1, + job_timeout_seconds: 5, + ping_seconds: 999, + reconnect_min_seconds: 2, + reconnect_max_seconds: 10 ** 6, + }, + { bandwidth_mb_day: 5, max_concurrent: 99 }, + ); + assert.equal(limits.maxBodyBytes, LIMITS.max_body_bytes); + assert.equal(limits.concurrency, 1); + assert.equal(limits.timeoutSeconds(25), 5); + assert.equal(limits.pingSeconds, LIMITS.ping_seconds); + assert.equal(limits.bandwidthBytes, 5 * MB); + assert.equal(limits.reconnectMin, 2); + assert.equal(limits.reconnectMax, LIMITS.reconnect_max_seconds); +}); + +test("concurrency cap admits and releases", () => { + const limits = new Limits({ local: { maxConcurrent: 1 } }); + assert.equal(limits.admit(0), null); + assert.equal(limits.admit(0), REFUSALS.CONCURRENCY_EXCEEDED); + limits.release(); + assert.equal(limits.admit(0), null); + limits.release(); + limits.release(); + assert.equal(limits.inflight, 0); +}); + +test("bandwidth cap refuses when used up", () => { + const limits = new Limits({ local: { bandwidthMb: 1 } }); + assert.equal(limits.admit(MB - 1), null); + limits.release(); + assert.equal(limits.admit(MB), REFUSALS.BANDWIDTH_EXHAUSTED); + assert.equal(limits.remainingBytes(2 * MB), 0); +}); + +test("active hours, including a window that wraps midnight", () => { + const day = new Limits({ clock: at(12), local: { activeFrom: "09:00", activeTo: "17:00" } }); + assert.equal(day.admit(0), null); + const early = new Limits({ clock: at(8, 59), local: { activeFrom: "09:00", activeTo: "17:00" } }); + assert.equal(early.admit(0), REFUSALS.OUTSIDE_HOURS); + const end = new Limits({ clock: at(17), local: { activeFrom: "09:00", activeTo: "17:00" } }); + assert.equal(end.admit(0), REFUSALS.OUTSIDE_HOURS); + const wrapIn = new Limits({ clock: at(23, 30), local: { activeFrom: "22:00", activeTo: "06:00" } }); + assert.equal(wrapIn.inActiveHours(), true); + const wrapEarly = new Limits({ clock: at(3), local: { activeFrom: "22:00", activeTo: "06:00" } }); + assert.equal(wrapEarly.inActiveHours(), true); + const wrapOut = new Limits({ clock: at(12), local: { activeFrom: "22:00", activeTo: "06:00" } }); + assert.equal(wrapOut.inActiveHours(), false); + const always = new Limits({ clock: at(12) }); + assert.equal(always.inActiveHours(), true); +}); + +test("hello limits advertise the effective concurrency", () => { + const limits = new Limits({ local: { maxConcurrent: 1 } }); + assert.deepEqual(limits.helloLimits(), { max_concurrent: 1 }); +}); diff --git a/nodes/extension/test/policy.test.mjs b/nodes/extension/test/policy.test.mjs new file mode 100644 index 00000000..ac121b57 --- /dev/null +++ b/nodes/extension/test/policy.test.mjs @@ -0,0 +1,114 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { Policy } from "../src/Policy.js"; +import { DESTINATIONS, REFUSALS } from "../src/spec.js"; + +const policy = new Policy(); + +test("allows https on the allowlisted hosts and subdomains", () => { + for (const url of [ + "https://reddit.com/r/python.json", + "https://www.reddit.com/r/python.json", + "https://old.reddit.com/", + "https://redd.it/abc", + "https://REDDIT.COM./x", + ]) { + assert.equal(policy.check("GET", url).ok, true, url); + } +}); + +test("suffix matching is dot safe", () => { + for (const url of [ + "https://evilreddit.com/", + "https://reddit.com.evil.net/", + "https://notredd.it/", + "https://reddit.co/", + ]) { + const verdict = policy.check("GET", url); + assert.equal(verdict.ok, false, url); + assert.equal(verdict.code, REFUSALS.HOST_NOT_ALLOWED); + } +}); + +test("userinfo tricks are refused", () => { + for (const url of [ + "https://reddit.com@evil.net/", + "https://user:pass@reddit.com/", + "https://evil.net\\@reddit.com/", + ]) { + assert.equal(policy.check("GET", url).ok, false, url); + } +}); + +test("non https schemes are refused", () => { + for (const url of ["http://reddit.com/", "ftp://reddit.com/", "ws://reddit.com/", "file:///etc/passwd"]) { + const verdict = policy.check("GET", url); + assert.equal(verdict.ok, false, url); + } +}); + +test("only port 443 passes over https", () => { + assert.equal(policy.check("GET", "https://reddit.com:443/").ok, true); + for (const url of ["https://reddit.com:80/", "https://reddit.com:8443/", "https://reddit.com:22/"]) { + const verdict = policy.check("GET", url); + assert.equal(verdict.ok, false, url); + assert.equal(verdict.code, REFUSALS.PORT_NOT_ALLOWED); + } +}); + +test("method must be permitted", () => { + const verdict = policy.check("POST", "https://reddit.com/"); + assert.equal(verdict.code, REFUSALS.METHOD_NOT_ALLOWED); + assert.equal(policy.check("get", "https://reddit.com/").ok, true); +}); + +test("ip literals, junk and oversize urls are refused", () => { + for (const url of ["https://127.0.0.1/", "https://[::1]/", "https://10.0.0.1/", "nonsense", "", null, 5]) { + assert.equal(policy.check("GET", url).ok, false, String(url)); + } + assert.equal(policy.check("GET", "https://reddit.com/" + "a".repeat(3000)).ok, false); +}); + +test("a redirect leaving the allowlist is refused with redirect_not_allowed", () => { + const base = new URL("https://www.reddit.com/r/x"); + const bad = policy.checkRedirect("https://evil.net/landing", base, "GET"); + assert.equal(bad.ok, false); + assert.equal(bad.code, REFUSALS.REDIRECT_NOT_ALLOWED); + assert.equal(policy.checkRedirect("http://reddit.com/x", base, "GET").code, REFUSALS.REDIRECT_NOT_ALLOWED); + assert.equal(policy.checkRedirect("//evilreddit.com/x", base, "GET").code, REFUSALS.REDIRECT_NOT_ALLOWED); + assert.equal(policy.checkRedirect("/r/y", base, "GET").ok, true); + assert.equal(policy.checkRedirect("https://old.reddit.com/r/y", base, "GET").ok, true); +}); + +test("a refreshed allowlist may only be equal or narrower", () => { + const narrower = [{ ...DESTINATIONS[0], suffix: "old.reddit.com" }]; + const kept = Policy.narrow(DESTINATIONS, narrower); + assert.equal(kept.destinations.length, 1); + assert.equal(kept.rejected.length, 0); + + const equal = Policy.narrow(DESTINATIONS, DESTINATIONS.map((item) => ({ ...item }))); + assert.equal(equal.destinations.length, DESTINATIONS.length); + + const widened = Policy.narrow(DESTINATIONS, [ + { suffix: "evil.net", adapter: "x", methods: ["GET"], ports: [443], modes: ["fetch"] }, + { suffix: "reddit.com", adapter: "reddit", methods: ["GET", "POST"], ports: [443], modes: ["fetch"] }, + { suffix: "reddit.com", adapter: "reddit", methods: ["GET"], ports: [443, 8080], modes: ["fetch"] }, + { suffix: "reddit.com", adapter: "reddit", methods: ["GET"], ports: [443], modes: ["tunnel"] }, + { suffix: "evilreddit.com", adapter: "reddit", methods: ["GET"], ports: [443], modes: ["fetch"] }, + ]); + assert.equal(widened.destinations.length, 0); + assert.equal(widened.rejected.length, 5); + + const empty = Policy.narrow(DESTINATIONS, []); + assert.equal(empty.destinations.length, 0); + assert.equal(Policy.narrow(DESTINATIONS, "x").destinations.length, 0); +}); + +test("a narrowed policy refuses what the baked policy would allow", () => { + const { policy: narrow } = Policy.fromRefreshed(DESTINATIONS, [ + { ...DESTINATIONS[0], suffix: "old.reddit.com" }, + ]); + assert.equal(narrow.check("GET", "https://old.reddit.com/").ok, true); + assert.equal(narrow.check("GET", "https://www.reddit.com/").ok, false); + assert.equal(narrow.check("GET", "https://redd.it/x").ok, false); +}); diff --git a/nodes/extension/test/spec.test.mjs b/nodes/extension/test/spec.test.mjs new file mode 100644 index 00000000..066a2805 --- /dev/null +++ b/nodes/extension/test/spec.test.mjs @@ -0,0 +1,38 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { DESTINATIONS, LIMITS, REFUSALS, ALLOWED_PORTS, PROTOCOL_VERSION } from "../src/spec.js"; + +const source = readFileSync( + fileURLToPath(new URL("../../../devplacepy/services/nodes/spec.py", import.meta.url)), + "utf8", +); + +function evaluate(expression) { + const clean = expression.replaceAll("_", "").trim(); + assert.match(clean, /^[\d\s*]+$/, expression); + return clean.split("*").reduce((product, part) => product * Number(part), 1); +} + +test("baked allowlist matches the server specification", () => { + const suffixes = [...source.matchAll(/DestinationSpec\("([^"]+)"/g)].map((match) => match[1]); + assert.deepEqual(DESTINATIONS.map((item) => item.suffix), suffixes); +}); + +test("baked ports, protocol and refusal codes match the server specification", () => { + const ports = /ALLOWED_PORTS = \(([^)]*)\)/.exec(source)[1].split(",").filter(Boolean).map(Number); + assert.deepEqual([...ALLOWED_PORTS], ports); + assert.equal(PROTOCOL_VERSION, Number(/PROTOCOL_VERSION = (\d+)/.exec(source)[1])); + const codes = [...source.matchAll(/RefusalSpec\("([a-z_]+)"/g)].map((match) => match[1]); + assert.deepEqual(Object.values(REFUSALS).sort(), codes.sort()); +}); + +test("baked limits match the server LimitSpec defaults", () => { + for (const [key, value] of Object.entries(LIMITS)) { + const pattern = new RegExp(`^\\s+${key}: int = ([\\d_ *]+)$`, "m"); + const match = pattern.exec(source); + assert.ok(match, `${key} missing from spec.py`); + assert.equal(value, evaluate(match[1]), key); + } +}); diff --git a/nodes/extension/test/wsclient.test.mjs b/nodes/extension/test/wsclient.test.mjs new file mode 100644 index 00000000..8e828149 --- /dev/null +++ b/nodes/extension/test/wsclient.test.mjs @@ -0,0 +1,323 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { Fetcher } from "../src/Fetcher.js"; +import { Limits } from "../src/Limits.js"; +import { Policy } from "../src/Policy.js"; +import { State, STATES } from "../src/State.js"; +import { Usage } from "../src/Usage.js"; +import { WsClient } from "../src/WsClient.js"; +import { DESTINATIONS } from "../src/spec.js"; + +function memoryArea() { + const store = {}; + return { + async get(key) { + return key in store ? { [key]: store[key] } : {}; + }, + async set(values) { + Object.assign(store, values); + }, + }; +} + +class FakeSocket { + static instances = []; + + constructor(address) { + this.address = address; + this.readyState = 0; + this.sent = []; + this.closed = null; + FakeSocket.instances.push(this); + } + + open() { + this.readyState = 1; + this.onopen(); + } + + send(text) { + this.sent.push(JSON.parse(text)); + } + + close(code, reason) { + this.readyState = 3; + this.closed = { code, reason }; + } + + receive(frame) { + return this.onmessage({ data: typeof frame === "string" ? frame : JSON.stringify(frame) }); + } + + serverClose(code, reason = "") { + this.readyState = 3; + return this.onclose({ code, reason }); + } +} + +class FakeTimers { + constructor() { + this.timeouts = []; + this.intervals = []; + } + setTimeout(fn, ms) { + const handle = { fn, ms, cleared: false }; + this.timeouts.push(handle); + return handle; + } + clearTimeout(handle) { + handle.cleared = true; + } + setInterval(fn, ms) { + const handle = { fn, ms, cleared: false }; + this.intervals.push(handle); + return handle; + } + clearInterval(handle) { + handle.cleared = true; + } +} + +function build(overrides = {}) { + FakeSocket.instances = []; + const timers = new FakeTimers(); + const limits = new Limits(); + const usage = new Usage(memoryArea()); + const state = new State(memoryArea()); + const fetchLog = []; + const jobFetch = overrides.jobFetch || (async () => new Response("page", { status: 200 })); + const fetcher = new Fetcher({ policy: new Policy(), limits, usage, fetchImpl: jobFetch }); + const ticketFetch = + overrides.ticketFetch || + (async (url, options) => { + fetchLog.push({ url, options }); + return new Response(JSON.stringify({ ticket: "tk1", expires_in: 30, ws_path: "/nodes/ws" }), { + status: 200, + }); + }); + const client = new WsClient({ + config: { credentials: () => ({ serverUrl: "https://devplace.net", secret: "dpn_secretsecretsecret" }) }, + limits, + fetcher, + state, + fetchImpl: ticketFetch, + WebSocketImpl: FakeSocket, + timers, + random: overrides.random || (() => 0.5), + clientVersion: "9.9.9", + platform: "firefox", + }); + return { client, timers, state, fetcher, limits, usage, fetchLog }; +} + +const welcome = (extra = {}) => ({ + type: "welcome", + device_uid: "dev1", + limits: { max_concurrent_per_node: 2, ping_seconds: 20 }, + destinations: DESTINATIONS.map((item) => ({ ...item })), + settings: { bandwidth_mb_day: 500, max_concurrent: 2 }, + ...extra, +}); + +async function online(context) { + await context.client.start(); + const socket = FakeSocket.instances.at(-1); + socket.open(); + await socket.receive(welcome()); + return socket; +} + +test("start fetches a ticket with the bearer secret and opens the socket", async () => { + const context = build(); + await context.client.start(); + assert.equal(context.fetchLog[0].url, "https://devplace.net/nodes/ticket"); + assert.equal(context.fetchLog[0].options.headers.Authorization, "Bearer dpn_secretsecretsecret"); + assert.equal(context.fetchLog[0].options.credentials, "omit"); + const socket = FakeSocket.instances[0]; + assert.equal(socket.address, "wss://devplace.net/nodes/ws?ticket=tk1"); +}); + +test("hello is sent on open with extension kind and fetch mode only", async () => { + const context = build(); + await context.client.start(); + const socket = FakeSocket.instances[0]; + socket.open(); + const hello = socket.sent[0]; + assert.equal(hello.type, "hello"); + assert.equal(hello.protocol, 1); + assert.equal(hello.kind, "extension"); + assert.deepEqual(hello.modes, ["fetch"]); + assert.equal(hello.client_version, "9.9.9"); + assert.equal(hello.platform, "firefox"); + assert.deepEqual(hello.limits, { max_concurrent: 2 }); +}); + +test("welcome puts the client online and starts the keepalive", async () => { + const context = build(); + await online(context); + assert.equal(context.state.snapshot().state, STATES.ONLINE); + assert.equal(context.state.snapshot().deviceUid, "dev1"); + assert.equal(context.timers.intervals[0].ms, 20000); +}); + +test("keepalive timer sends pong and a server ping is answered with pong", async () => { + const context = build(); + const socket = await online(context); + context.timers.intervals[0].fn(); + await socket.receive({ type: "ping" }); + assert.deepEqual( + socket.sent.filter((frame) => frame.type === "pong"), + [{ type: "pong" }, { type: "pong" }], + ); +}); + +test("a job is executed and answered with job_result then usage", async () => { + const context = build(); + const socket = await online(context); + await socket.receive({ + type: "job", + job_id: "j9", + method: "GET", + url: "https://www.reddit.com/r/x.json", + timeout: 5, + }); + await context.client.idle(); + const result = socket.sent.find((frame) => frame.type === "job_result"); + assert.equal(result.job_id, "j9"); + assert.equal(Buffer.from(result.body_b64, "base64").toString(), "page"); + assert.equal(socket.sent.some((frame) => frame.type === "usage"), true); +}); + +test("a job for a forbidden host is answered with job_error", async () => { + const context = build(); + const socket = await online(context); + await socket.receive({ type: "job", job_id: "j2", method: "GET", url: "https://evilreddit.com/", timeout: 5 }); + await context.client.idle(); + const error = socket.sent.find((frame) => frame.type === "job_error"); + assert.deepEqual(error, { type: "job_error", job_id: "j2", code: "host_not_allowed" }); +}); + +test("a welcome that widens the allowlist is ignored for the widened entries", async () => { + const context = build(); + await context.client.start(); + const socket = FakeSocket.instances[0]; + socket.open(); + await socket.receive( + welcome({ + destinations: [ + ...DESTINATIONS.map((item) => ({ ...item })), + { suffix: "evil.net", adapter: "x", methods: ["GET"], ports: [443], modes: ["fetch"] }, + ], + }), + ); + assert.match(context.state.snapshot().lastError, /evil\.net/); + await socket.receive({ type: "job", job_id: "j3", method: "GET", url: "https://evil.net/", timeout: 5 }); + await context.client.idle(); + assert.equal(socket.sent.find((frame) => frame.type === "job_error").code, "host_not_allowed"); +}); + +test("a job before welcome is a protocol fault and closes with 4400", async () => { + const context = build(); + await context.client.start(); + const socket = FakeSocket.instances[0]; + socket.open(); + await socket.receive({ type: "job", job_id: "j", method: "GET", url: "https://reddit.com/", timeout: 5 }); + assert.equal(socket.closed.code, 4400); + assert.equal(context.state.snapshot().state, STATES.BACKOFF); +}); + +test("malformed frames are rejected loudly", async () => { + const context = build(); + const socket = await online(context); + for (const bad of ["not json", "[]", JSON.stringify({ type: "nope" }), JSON.stringify({ type: "job", job_id: "x" })]) { + await assert.rejects(context.client.handleMessage(bad)); + } + await assert.rejects(context.client.handleMessage(new ArrayBuffer(4))); + assert.equal(socket.sent.filter((frame) => frame.type === "job_result").length, 0); +}); + +test("kick halts without reconnecting", async () => { + const context = build(); + const socket = await online(context); + await socket.receive({ type: "kick", reason: "replaced" }); + assert.equal(context.state.snapshot().state, STATES.HALTED); + assert.match(context.state.snapshot().detail, /replaced/); + assert.equal(context.timers.timeouts.filter((handle) => !handle.cleared).length, 0); + await context.client.ensure(); + assert.equal(FakeSocket.instances.length, 1); +}); + +test("a normal close schedules a full jitter reconnect within the bounds", async () => { + const context = build({ random: () => 0.999 }); + const socket = await online(context); + await socket.serverClose(1006); + const timer = context.timers.timeouts.at(-1); + assert.ok(timer.ms >= 1000 && timer.ms <= 60000, String(timer.ms)); + assert.equal(context.state.snapshot().state, STATES.BACKOFF); + timer.fn(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(FakeSocket.instances.length, 2); +}); + +test("backoff grows with attempts and never exceeds reconnect_max", async () => { + const context = build({ random: () => 1 }); + await context.client.start(); + const delays = []; + for (let index = 0; index < 12; index += 1) { + const socket = FakeSocket.instances.at(-1); + await socket.serverClose(1006); + const timer = context.timers.timeouts.at(-1); + delays.push(timer.ms); + timer.fn(); + await new Promise((resolve) => setImmediate(resolve)); + } + assert.ok(delays[1] > delays[0]); + assert.ok(Math.max(...delays) <= 60000); + assert.equal(delays.at(-1), 60000); +}); + +test("retry owner close reconnects at the minimum delay", async () => { + const context = build({ random: () => 1 }); + const socket = await online(context); + await socket.serverClose(4013); + assert.equal(context.timers.timeouts.at(-1).ms, 1000); +}); + +test("a consent or device refusal halts instead of hammering the server", async () => { + const context = build(); + await context.client.start(); + const socket = FakeSocket.instances[0]; + await socket.serverClose(1008, "no_consent"); + assert.equal(context.state.snapshot().state, STATES.HALTED); + assert.match(context.state.snapshot().detail, /no_consent/); +}); + +test("a ticket 401 halts, a ticket 500 retries", async () => { + const halted = build({ + ticketFetch: async () => new Response(JSON.stringify({ error: "x" }), { status: 401 }), + }); + await halted.client.start(); + assert.equal(halted.state.snapshot().state, STATES.HALTED); + assert.equal(FakeSocket.instances.length, 0); + + const retried = build({ ticketFetch: async () => new Response("boom", { status: 500 }) }); + await retried.client.start(); + assert.equal(retried.state.snapshot().state, STATES.BACKOFF); +}); + +test("stop closes the socket and clears timers", async () => { + const context = build(); + const socket = await online(context); + await context.client.stop("user"); + assert.equal(socket.closed.code, 1000); + assert.equal(context.timers.intervals[0].cleared, true); + assert.equal(context.state.snapshot().state, STATES.OFF); +}); + +test("settings_update can only lower the effective limits", async () => { + const context = build(); + const socket = await online(context); + await socket.receive({ type: "settings_update", settings: { bandwidth_mb_day: 10, max_concurrent: 1 } }); + assert.equal(context.limits.concurrency, 1); + assert.equal(context.limits.bandwidthBytes, 10 * 1024 * 1024); +}); diff --git a/nodes/extension/ui/disclosure.html b/nodes/extension/ui/disclosure.html new file mode 100644 index 00000000..ee212bae --- /dev/null +++ b/nodes/extension/ui/disclosure.html @@ -0,0 +1,55 @@ + + + + + + DevPlace Node: what is shared + + + +
+

Before you share anything

+

DevPlace Node lets the community use a little of your internet connection to read public pages from sites that block data centers. Read this once. Nothing runs until you accept and switch it on.

+ +
+

What passes through your browser

+
    +
  • The address of the page that is requested (host and full URL), for example a public Reddit thread.
    • +
  • The page content that comes back. It is read by this extension and sent to the DevPlace server.
    • +
  • Your public IP address is what the destination site sees. That is the point of donating.
    • +
  • How many requests and how many bytes you moved, counted on this device and on the server.
    • +
+
+ +
+

What is never used

+
    +
  • Your cookies, logins and accounts. Every request is sent without credentials, so you are never signed in as yourself.
    • +
  • Your browsing history, open tabs and other sites.
    • +
  • Any host outside the built-in list, any port other than the standard secure one, any address inside your own network.
    • +
+
+ +
+

You stay in control

+
    +
  • One on/off switch in the popup. Off means disconnected.
    • +
  • A daily data cap, a parallel request cap and optional active hours, enforced on this device even if the server asks for more.
    • +
  • Live counters of requests and bytes in the popup.
    • +
  • You can forget the device secret at any time, and revoke the device on your DevPlace profile.
    • +
+

Pages fetched through your connection are public pages. Do not donate from a connection whose address you cannot afford to have associated with that traffic.

+
+ +
+
+ + +
+ +

+
+
+ + + diff --git a/nodes/extension/ui/disclosure.js b/nodes/extension/ui/disclosure.js new file mode 100644 index 00000000..cf89853a --- /dev/null +++ b/nodes/extension/ui/disclosure.js @@ -0,0 +1,31 @@ +class Disclosure { + constructor(root) { + this.root = root; + this.$ = (id) => root.getElementById(id); + } + + async accept() { + try { + const reply = await chrome.runtime.sendMessage({ cmd: "accept" }); + if (!reply || reply.error) { + throw new Error(reply ? reply.error : "The background worker did not answer"); + } + this.$("error").textContent = ""; + this.$("done").textContent = + "Accepted. Open the extension popup, connect your device secret and switch sharing on."; + } catch (error) { + this.$("error").textContent = error.message; + } + } + + decline() { + this.$("done").textContent = "Nothing was enabled. You can close this tab."; + } + + start() { + this.$("accept").addEventListener("click", () => this.accept()); + this.$("decline").addEventListener("click", () => this.decline()); + } +} + +new Disclosure(document).start(); diff --git a/nodes/extension/ui/popup.css b/nodes/extension/ui/popup.css new file mode 100644 index 00000000..8f8e7fd3 --- /dev/null +++ b/nodes/extension/ui/popup.css @@ -0,0 +1,226 @@ +:root { + color-scheme: light dark; + --bg: #f6f7f9; + --card: #ffffff; + --text: #1c2128; + --muted: #5b6672; + --border: #d5dae0; + --accent: #2f6fed; + --accent-text: #ffffff; + --ok: #1a8a4a; + --warn: #b7791f; + --bad: #c0392b; + --radius: 8px; + --gap: 12px; + --font: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; +} + +@media (prefers-color-scheme: dark) { + :root { + --bg: #14181d; + --card: #1d232a; + --text: #e8ecf0; + --muted: #97a3b0; + --border: #323b45; + --accent: #5b92ff; + --accent-text: #0b1220; + --ok: #4cc27f; + --warn: #e0b04a; + --bad: #ee7b6e; + } +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + background: var(--bg); + color: var(--text); + font: 14px/1.45 var(--font); + width: 100%; + min-width: 260px; + max-width: 420px; +} + +body.page { + max-width: 640px; + margin: 0 auto; +} + +main { + padding: var(--gap); + display: grid; + gap: var(--gap); +} + +h1 { + font-size: 1.15rem; + margin: 0; +} + +h2 { + font-size: 0.95rem; + margin: 0 0 6px; +} + +p { + margin: 0 0 8px; +} + +.card { + background: var(--card); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: var(--gap); +} + +.muted { + color: var(--muted); + font-size: 0.85rem; +} + +.row { + display: flex; + flex-wrap: wrap; + align-items: center; + justify-content: space-between; + gap: 8px; +} + +.status { + display: inline-flex; + align-items: center; + gap: 6px; + font-weight: 600; +} + +.dot { + width: 10px; + height: 10px; + border-radius: 50%; + background: var(--muted); +} + +.dot.online { + background: var(--ok); +} + +.dot.connecting, +.dot.backoff { + background: var(--warn); +} + +.dot.halted { + background: var(--bad); +} + +.counters { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(110px, 1fr)); + gap: 8px; +} + +.counter { + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 8px; +} + +.counter strong { + display: block; + font-size: 1.1rem; +} + +label { + display: grid; + gap: 4px; + margin-bottom: 8px; + font-size: 0.85rem; + color: var(--muted); +} + +input { + width: 100%; + font: inherit; + color: var(--text); + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 7px 9px; +} + +button, +.button { + font: inherit; + cursor: pointer; + color: var(--text); + background: var(--card); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 7px 12px; + text-decoration: none; +} + +button.primary { + background: var(--accent); + color: var(--accent-text); + border-color: var(--accent); +} + +button.danger { + color: var(--bad); +} + +button:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.switch { + display: inline-flex; + align-items: center; + gap: 8px; + margin: 0; + color: var(--text); + font-size: 1rem; + font-weight: 600; +} + +.switch input { + width: 20px; + height: 20px; +} + +.notice { + border-left: 3px solid var(--warn); + padding-left: 10px; +} + +.error { + color: var(--bad); + min-height: 1.2em; + word-break: break-word; +} + +.pair { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(110px, 1fr)); + gap: 8px; +} + +details > summary { + cursor: pointer; + font-weight: 600; + margin-bottom: 8px; +} + +ul { + margin: 0 0 8px; + padding-left: 20px; +} + +li { + margin-bottom: 4px; +} diff --git a/nodes/extension/ui/popup.html b/nodes/extension/ui/popup.html new file mode 100644 index 00000000..2f4da44e --- /dev/null +++ b/nodes/extension/ui/popup.html @@ -0,0 +1,77 @@ + + + + + + DevPlace Node + + + +
+
+

DevPlace Node

+ off +
+ + + +
+
+ + +
+

+ +
+ +
+

Today

+
+
Requests0
+
Downloaded0 B
+
Uploaded0 B
+
Failed0
+
+

+
+ +
+
+ Limits on this device +
+
+ + +
+
+ + +
+

Leave the hours empty to share at any time. These limits are enforced here, whatever the server asks.

+ +
+
+
+ +
+
+ Connect to a server +
+ + + + +
+

+ +
+
+ +

What is shared

+
+ + + diff --git a/nodes/extension/ui/popup.js b/nodes/extension/ui/popup.js new file mode 100644 index 00000000..c7e4d567 --- /dev/null +++ b/nodes/extension/ui/popup.js @@ -0,0 +1,155 @@ +const POLL_MS = 2000; +const UNITS = ["B", "KB", "MB", "GB", "TB"]; +const STATE_LABELS = { + off: "Off", + connecting: "Connecting", + online: "Online", + backoff: "Reconnecting", + halted: "Stopped", +}; + +function formatBytes(value) { + let size = Math.max(0, Number(value) || 0); + let unit = 0; + while (size >= 1024 && unit < UNITS.length - 1) { + size /= 1024; + unit += 1; + } + return `${unit === 0 ? size : size.toFixed(1)} ${UNITS[unit]}`; +} + +class Popup { + constructor(root) { + this.root = root; + this.timer = null; + this.view = null; + this.$ = (id) => root.getElementById(id); + } + + async send(message) { + const reply = await chrome.runtime.sendMessage(message); + if (!reply) { + throw new Error("The background worker did not answer"); + } + if (reply.error) { + throw new Error(reply.error); + } + return reply; + } + + showError(error) { + this.$("error").textContent = error ? error.message || String(error) : ""; + } + + async run(message) { + try { + this.render(await this.send(message)); + this.showError(null); + } catch (error) { + this.showError(error); + } + } + + render(view) { + this.view = view; + const { config, state, usage } = view; + this.$("dot").className = `dot ${state.state}`; + this.$("state").textContent = STATE_LABELS[state.state] || state.state; + this.$("detail").textContent = state.detail || state.lastError || ""; + this.$("label").textContent = config.label; + this.$("disclosure-card").hidden = config.disclosed; + const toggle = this.$("toggle"); + toggle.checked = config.enabled; + toggle.disabled = !config.disclosed || !config.hasSecret; + this.$("toggle-label").textContent = config.enabled ? "Sharing on" : "Sharing off"; + this.$("requests").textContent = usage.requests; + this.$("failed").textContent = usage.failed; + this.$("down").textContent = formatBytes(usage.bytesDown); + this.$("up").textContent = formatBytes(usage.bytesUp); + this.$("lifetime").textContent = + `All time: ${usage.lifetimeRequests} requests, ${formatBytes(usage.lifetimeBytes)}`; + this.renderSettings(config); + this.renderConnection(config); + } + + renderSettings(config) { + if (this.root.activeElement && this.$("settings").contains(this.root.activeElement)) { + return; + } + this.$("bandwidth").value = config.bandwidthMb; + this.$("concurrency").value = config.maxConcurrent; + this.$("from").value = config.activeFrom; + this.$("to").value = config.activeTo; + } + + renderConnection(config) { + this.$("forget").hidden = !config.hasSecret; + this.$("connect-summary").textContent = config.hasSecret ? "Connection" : "Connect to a server"; + this.$("saved").textContent = config.hasSecret + ? `Connected to ${config.serverUrl} with secret ${config.secretPrefix}` + : ""; + if (!config.hasSecret) { + this.$("connect-box").open = true; + } + } + + async requestOrigin(serverUrl) { + const origin = new URL(serverUrl).origin; + const granted = await chrome.permissions.request({ origins: [origin + "/*"] }); + if (!granted) { + throw new Error("Permission to talk to that server was not granted"); + } + } + + async onConnect(event) { + event.preventDefault(); + const secretField = this.$("secret"); + try { + await this.requestOrigin(this.$("server").value); + this.render( + await this.send({ + cmd: "connect", + serverUrl: this.$("server").value, + secret: secretField.value, + label: this.$("device-label").value, + }), + ); + secretField.value = ""; + this.showError(null); + } catch (error) { + this.showError(error); + } + } + + onSettings(event) { + event.preventDefault(); + this.run({ + cmd: "settings", + values: { + bandwidthMb: Number(this.$("bandwidth").value), + maxConcurrent: Number(this.$("concurrency").value), + activeFrom: this.$("from").value, + activeTo: this.$("to").value, + }, + }); + } + + onForget() { + if (this.root.defaultView.confirm("Forget this device secret on this browser?")) { + this.run({ cmd: "forget" }); + } + } + + start() { + this.$("connect").addEventListener("submit", (event) => this.onConnect(event)); + this.$("settings").addEventListener("submit", (event) => this.onSettings(event)); + this.$("forget").addEventListener("click", () => this.onForget()); + this.$("toggle").addEventListener("change", (event) => + this.run({ cmd: "enable", enabled: event.target.checked }), + ); + this.run({ cmd: "status" }); + this.timer = setInterval(() => this.run({ cmd: "status" }), POLL_MS); + } +} + +new Popup(document).start(); diff --git a/pyproject.toml b/pyproject.toml index 17aa352c..f7b9d2a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "devplacepy" -version = "1.1.1" +version = "1.1.2" description = "DevPlace - The Developer Social Network" requires-python = ">=3.12" dependencies = [ diff --git a/tests/CLAUDE.md b/tests/CLAUDE.md index d850e611..3f14f95a 100644 --- a/tests/CLAUDE.md +++ b/tests/CLAUDE.md @@ -24,7 +24,7 @@ The suite does NOT stop at the first failure. `-rf` lives in `pyproject.toml` `a | `make test-slowest` | all three tiers + the 40 slowest tests | full | | `make test-cache-clean` | drops `.pytest_cache/` when `--last-failed` misbehaves | instant | -**Server-side display caches are disabled for the suite, never waited out.** A test that changes state the server serves from a TTL cache must not poll until the TTL expires - that burns the TTL in wall-clock on every run (the leaderboard ranking test cost 60s and the Code Farm saturation test 30s exactly this way). Each such cache reads its TTL from an env var with the production value as the default, and `conftest.py` sets it to `0` before the server subprocess starts: `DEVPLACE_SITEMAP_TTL` (`seo.SITEMAP_TTL`), `DEVPLACE_HOME_CACHE_TTL` (`main._home_cache`), `DEVPLACE_RANKING_TTL` (`database/ranking.py` `_authors_cache`), `DEVPLACE_MARKET_SATURATION_TTL` (`services/game/store/market.py` `_saturation_cache`). Add a new display cache to that list rather than sleeping in the test. **Correctness caches are different**: the cross-worker `cache_state` version read (`_cache_version_cache`, 1s) is load-bearing and stays on, so a test that flips the admin set from the test process must wait for the server to agree before asserting (see `tests/e2e/admin/containers/manage.py` `_await_instance_visible`), never assume the change is instant. +**Server-side display caches are disabled for the suite, never waited out.** A test that changes state the server serves from a TTL cache must not poll until the TTL expires - that burns the TTL in wall-clock on every run (the leaderboard ranking test cost 60s and the Code Farm saturation test 30s exactly this way). Each such cache reads its TTL from an env var with the production value as the default, and `conftest.py` sets it to `0` before the server subprocess starts: `DEVPLACE_SITEMAP_TTL` (`seo.SITEMAP_TTL`), `DEVPLACE_HOME_CACHE_TTL` (`main._home_cache`), `DEVPLACE_RANKING_TTL` (`database/ranking.py` `_authors_cache`), `DEVPLACE_NODES_STATS_TTL` (`routers/nodes/me.py` `_public_cache`), `DEVPLACE_MARKET_SATURATION_TTL` (`services/game/store/market.py` `_saturation_cache`). Add a new display cache to that list rather than sleeping in the test. **Correctness caches are different**: the cross-worker `cache_state` version read (`_cache_version_cache`, 1s) is load-bearing and stays on, so a test that flips the admin set from the test process must wait for the server to agree before asserting (see `tests/e2e/admin/containers/manage.py` `_await_instance_visible`), never assume the change is instant. The loop: `make test-fast` -> fix -> `make test-failed` until it is empty -> `make test` as the final gate. `make test-first-failure` is only for the rare case where one early failure poisons every test after it (a corrupted session fixture, a stopped service). Pytest's cache lives in `.pytest_cache/` (git-ignored), so `--last-failed` works with no extra setup; do not pass `-p no:cacheprovider` in a run whose failures you intend to re-run. **If `make test-failed` runs the whole suite instead of just the failures, the cache is polluted**: pytest keeps `lastfailed` entries for tests it did not collect, so node ids from deleted files (or from a stray `pytest` invocation that walked `data/`/`var/`) linger forever and make `--last-failed` give up and select everything. `make test-cache-clean` drops `.pytest_cache/` and fixes it; `make clean` does it too. diff --git a/tests/api/admin/communities/__init__.py b/tests/api/admin/communities/__init__.py new file mode 100644 index 00000000..95ee7c3e --- /dev/null +++ b/tests/api/admin/communities/__init__.py @@ -0,0 +1 @@ +# retoor diff --git a/tests/api/admin/communities/index.py b/tests/api/admin/communities/index.py new file mode 100644 index 00000000..101570e9 --- /dev/null +++ b/tests/api/admin/communities/index.py @@ -0,0 +1,247 @@ +# retoor + +import time + +import pytest +from fastapi.testclient import TestClient + +from devplacepy.database import get_setting, get_table, set_setting +from devplacepy.main import app +from devplacepy.routers.admin import communities as router_module +from devplacepy.services.community.runner import CommunityRunner +from devplacepy.services.community.store import store +from devplacepy.utils import register_account +from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep + +_counter = [0] + + +def unique(prefix: str) -> str: + _counter[0] += 1 + return f"{prefix}{int(time.time() * 1000)}{_counter[0]}" + + +def make_account(role: str = "Member") -> dict: + name = unique("cmty") + uid, _, _ = register_account(name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True) + if role != "Member": + get_table("users").update({"uid": uid, "role": role}, ["uid"]) + row = get_table("users").find_one(uid=uid) + return {"username": name, "headers": {"X-API-KEY": row["api_key"], "Accept": "application/json"}} + + +@pytest.fixture(scope="module") +def client(): + previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")} + set_setting("sync_enabled", "1") + set_setting("sync_origins", "") + with TestClient(app) as test_client: + yield test_client + for key, value in previous.items(): + set_setting(key, value) + + +@pytest.fixture +def fake_world(monkeypatch): + routes = json_community() + monkeypatch.setattr( + router_module, + "CommunityRunner", + lambda: CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep), + ) + get_table("communities").delete() + get_table("community_runs").delete() + get_table("sync_links").delete(origin="forum.test") + return routes + + +@pytest.fixture +def admin(client): + return make_account("Admin") + + +def wait_for(predicate, seconds: float = 10.0): + deadline = time.time() + seconds + while time.time() < deadline: + value = predicate() + if value: + return value + time.sleep(0.1) + raise AssertionError("condition never became true") + + +def add_ready(client, admin) -> dict: + response = client.post("/admin/communities", data={"query": "https://forum.test/api/posts"}, headers=admin["headers"]) + assert response.status_code == 200, response.text + return wait_for(lambda: (row := store.get("forum-test")) and row["status"] == "ready" and row) + + +def test_member_is_refused(client, admin, fake_world): + member = make_account() + assert client.get("/admin/communities", headers=member["headers"], follow_redirects=False).status_code in (303, 403) + assert client.post("/admin/communities", data={"query": "x.test"}, headers=member["headers"], follow_redirects=False).status_code in (303, 403) + + +def test_add_learns_and_shows_detail(client, admin, fake_world): + row = add_ready(client, admin) + assert row["origin"] == "forum.test" + assert row["schema"]["feeds"][0]["type"] == "json_api" + assert row["description"] == "A friendly forum about code." + assert row["user_count"] == 1234 + listing = client.get("/admin/communities", headers=admin["headers"]) + assert listing.status_code == 200 + assert listing.json()["items"][0]["slug"] == "forum-test" + detail = client.get("/admin/communities/forum-test", headers=admin["headers"]) + assert detail.status_code == 200 + body = detail.json() + assert body["feeds"][0]["pagination"] == "page" + assert body["collectors_lines"] > 100 + html = client.get("/admin/communities/forum-test", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]}) + assert html.status_code == 200 and "Learned feeds" in html.text + + +def test_add_is_idempotent(client, admin, fake_world): + add_ready(client, admin) + again = client.post("/admin/communities", data={"query": "https://forum.test/api/posts"}, headers=admin["headers"]) + assert again.status_code == 200 + assert len(store.list()) == 1 + + +def test_dry_run_writes_nothing_and_keeps_watermark(client, admin, fake_world): + row = add_ready(client, admin) + users_before = get_table("users").count() + response = client.post("/admin/communities/forum-test/run", data={"dry_run": "true"}, headers=admin["headers"]) + assert response.status_code == 200 + run = wait_for(lambda: next((item for item in store.runs(row["uid"]) if item["status"] == "dry_run"), None)) + assert run["dry_run"] == 1 + assert run["pushed"]["posts"]["created"] == 10 + assert "DRY RUN" in run["log"] + assert get_table("users").count() == users_before + after = store.get("forum-test") + assert not after["watermark"] + assert after["run_count"] == 0 + + +def test_live_run_creates_then_is_unchanged(client, admin, fake_world): + row = add_ready(client, admin) + client.post("/admin/communities/forum-test/run", data={}, headers=admin["headers"]) + first = wait_for(lambda: next((item for item in store.runs(row["uid"]) if item["status"] in ("ok", "partial", "failed")), None)) + assert first["status"] == "ok", first["errors"] + assert first["pushed"]["posts"]["created"] == 10 + assert get_table("sync_links").count(origin="forum.test", kind="posts") == 10 + done = store.get("forum-test") + assert done["run_count"] == 1 and done["watermark"]["feeds"]["feed1"]["since"] + assert done["next_run_at"] + client.post("/admin/communities/forum-test/run", data={}, headers=admin["headers"]) + second = wait_for(lambda: len([item for item in store.runs(row["uid"]) if item["status"] != "running"]) >= 2 and store.runs(row["uid"])[0]) + assert second["pushed"].get("posts", {}).get("created", 0) == 0 + + +def test_live_run_refused_when_sync_disabled(client, admin, fake_world): + row = add_ready(client, admin) + set_setting("sync_enabled", "0") + try: + client.post("/admin/communities/forum-test/run", data={}, headers=admin["headers"]) + run = wait_for(lambda: next((item for item in store.runs(row["uid"]) if item["status"] != "running"), None)) + assert run["status"] == "failed" + assert any("disabled" in error for error in run["errors"]) + finally: + set_setting("sync_enabled", "1") + + +def test_update_and_delete(client, admin, fake_world): + add_ready(client, admin) + client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "enabled": "on"}, headers=admin["headers"]) + assert store.get("forum-test")["interval_seconds"] == 7200 + bad = client.post("/admin/communities/forum-test/update", data={"interval_seconds": "10"}, headers=admin["headers"]) + assert bad.status_code in (400, 422) + client.post("/admin/communities/forum-test/delete", headers=admin["headers"]) + assert store.get("forum-test") is None + assert get_table("communities").find_one(slug="forum-test")["deleted_at"] + + +def test_unknown_community_is_404(client, admin, fake_world): + assert client.get("/admin/communities/nope", headers=admin["headers"]).status_code == 404 + + +GATED = [ + ("get", "/admin/communities", {}), + ("post", "/admin/communities", {"query": "https://forum.test/api/posts"}), + ("get", "/admin/communities/forum-test", {}), + ("post", "/admin/communities/forum-test/run", {}), + ("post", "/admin/communities/forum-test/relearn", {}), + ("post", "/admin/communities/forum-test/update", {"interval_seconds": "7200"}), + ("post", "/admin/communities/forum-test/delete", {}), +] + + +@pytest.mark.parametrize("method,path,data", GATED) +def test_every_endpoint_refuses_guests_and_members_and_changes_nothing(client, admin, fake_world, method, path, data): + add_ready(client, admin) + before = store.get("forum-test") + member = make_account() + for headers in ({"Accept": "application/json"}, member["headers"]): + response = client.request(method.upper(), path, data=data or None, headers=headers, follow_redirects=False) + assert response.status_code in (303, 401, 403), (headers, response.status_code) + after = store.get("forum-test") + assert after is not None + assert after["interval_seconds"] == before["interval_seconds"] and len(store.list()) == 1 + + +def test_password_is_stored_but_never_returned(client, admin, fake_world): + client.post( + "/admin/communities", + data={"query": "https://forum.test/api/posts", "login_username": "bot", "login_password": "s3cret-pw"}, + headers=admin["headers"], + ) + row = wait_for(lambda: (found := store.get("forum-test")) and found["status"] == "ready" and found) + assert row["login_password"] == "s3cret-pw" + for url in ("/admin/communities", "/admin/communities/forum-test"): + body = client.get(url, headers=admin["headers"]) + assert "s3cret-pw" not in body.text and body.json() + page = client.get("/admin/communities/forum-test", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]}) + assert "s3cret-pw" not in page.text and "(stored)" in page.text + client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "login_username": "other"}, headers=admin["headers"]) + assert store.get("forum-test")["login_password"] == "s3cret-pw" + client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "clear_login": "on"}, headers=admin["headers"]) + assert store.get("forum-test")["login_password"] == "" + + +def test_devii_tools_are_admin_only_and_delete_needs_confirmation(): + from devplacepy.services.devii.actions.catalog.community import COMMUNITY_ACTIONS + from devplacepy.services.devii.actions.dispatcher import CONFIRM_REQUIRED + from devplacepy.docs_api.groups import community + + assert {action.name for action in COMMUNITY_ACTIONS} >= {"community_add", "community_run", "community_delete"} + assert all(action.requires_admin for action in COMMUNITY_ACTIONS) + assert "community_delete" in CONFIRM_REQUIRED + assert next(action for action in COMMUNITY_ACTIONS if action.name == "community_delete").params[-1].name == "confirm" + assert community.GROUP["admin"] is True + + +def test_update_saves_sync_options_and_keeps_unsent_ones(client, admin, fake_world): + add_ready(client, admin) + body = {"interval_seconds": 86400, "enabled": "true", "alias": "true", "sample_percent": 40, "topics": "devlog, rant"} + assert client.post("/admin/communities/forum-test/update", data=body, headers=admin["headers"]).status_code == 200 + options = store.get("forum-test")["sync_options"] + assert options["alias"] is True and options["sample_percent"] == 40 and options["topics"] == ["devlog", "rant"] + client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "jitter_percent": 10}, headers=admin["headers"]) + kept = store.get("forum-test")["sync_options"] + assert kept["alias"] is True and kept["jitter_percent"] == 10 + + +def test_update_rejects_bad_options_and_late_aliasing(client, admin, fake_world): + add_ready(client, admin) + bad = client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "topics": "nonsense"}, headers=admin["headers"]) + assert bad.status_code == 400 + get_table("sync_links").insert({"uid": "late1", "origin": "forum.test", "kind": "users", "origin_uid": "1", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""}) + late = client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "alias": "true"}, headers=admin["headers"]) + assert late.status_code == 400 + assert not store.get("forum-test")["sync_options"].get("alias") + + +def test_sync_options_are_admin_only(client, fake_world): + member = make_account("Member") + add_ready(client, make_account("Admin")) + assert client.get("/admin/communities/forum-test", headers=member["headers"]).status_code in (401, 403) + assert client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "alias": "true"}, headers=member["headers"]).status_code in (401, 403) diff --git a/tests/api/admin/nodes/__init__.py b/tests/api/admin/nodes/__init__.py new file mode 100644 index 00000000..95ee7c3e --- /dev/null +++ b/tests/api/admin/nodes/__init__.py @@ -0,0 +1 @@ +# retoor diff --git a/tests/api/admin/nodes/conftest.py b/tests/api/admin/nodes/conftest.py new file mode 100644 index 00000000..7a6dcb97 --- /dev/null +++ b/tests/api/admin/nodes/conftest.py @@ -0,0 +1,36 @@ +# retoor + +import pytest +from fastapi.testclient import TestClient + +from devplacepy.database import get_setting, set_setting +from devplacepy.database.nodes import create_device +from devplacepy.main import app +from tests.api.nodes.conftest import make_member + +JSON = {"Accept": "application/json"} + + +def admin_headers(who: dict) -> dict: + return {**who["headers"], **JSON} + + +def make_admin() -> dict: + return make_member(consent=False, enabled=False, role="Admin") + + +def make_device(owner_uid: str, label: str = "Home laptop") -> dict: + return create_device(owner_uid, label=label, kind="native")[0] + + +@pytest.fixture(scope="module") +def client(): + previous = get_setting("nodes_enabled", "0") + with TestClient(app) as test_client: + yield test_client + set_setting("nodes_enabled", previous) + + +@pytest.fixture +def admin(): + return make_admin() diff --git a/tests/api/admin/nodes/flags.py b/tests/api/admin/nodes/flags.py new file mode 100644 index 00000000..35223da6 --- /dev/null +++ b/tests/api/admin/nodes/flags.py @@ -0,0 +1,34 @@ +# retoor + +from devplacepy.database.nodes import list_flags, raise_flag +from tests.api.admin.nodes.conftest import admin_headers, make_admin, make_device +from tests.api.nodes.conftest import make_member + + +def test_member_cannot_resolve(client): + member = make_member() + assert client.post("/admin/nodes/flags/x/resolve", headers=admin_headers(member)).status_code == 403 + + +def test_resolve_closes_the_flag_once(client, admin): + owner = make_member() + device = make_device(owner["uid"]) + flag = raise_flag("refusal_mismatch", device_uid=device["uid"], user_uid=owner["uid"], detail="test") + listed = client.get("/admin/nodes", headers=admin_headers(admin)).json()["flags"] + assert flag["uid"] in {item["uid"] for item in listed} + assert client.post(f"/admin/nodes/flags/{flag['uid']}/resolve", headers=admin_headers(admin)).status_code == 200 + assert flag["uid"] not in {item["uid"] for item in list_flags()} + assert client.post(f"/admin/nodes/flags/{flag['uid']}/resolve", headers=admin_headers(admin)).status_code == 409 + + +def test_unknown_flag_is_404(client, admin): + assert client.post("/admin/nodes/flags/nope/resolve", headers=admin_headers(admin)).status_code == 404 + + +def test_junior_admin_cannot_resolve_a_senior_admins_flag(client): + senior = make_admin() + junior = make_admin() + device = make_device(senior["uid"]) + flag = raise_flag("integrity_failure", device_uid=device["uid"], user_uid=senior["uid"], detail="test") + client.post(f"/admin/nodes/flags/{flag['uid']}/resolve", headers=admin_headers(junior)) + assert flag["uid"] in {item["uid"] for item in list_flags()} diff --git a/tests/api/admin/nodes/index.py b/tests/api/admin/nodes/index.py new file mode 100644 index 00000000..6af25760 --- /dev/null +++ b/tests/api/admin/nodes/index.py @@ -0,0 +1,42 @@ +# retoor + +from devplacepy.database.nodes import LIMITS +from tests.api.admin.nodes.conftest import admin_headers, make_device +from tests.api.nodes.conftest import make_member + + +def test_guest_is_refused(client): + assert client.get("/admin/nodes", headers={"Accept": "application/json"}).status_code == 401 + + +def test_member_is_refused(client): + member = make_member() + assert client.get("/admin/nodes", headers=admin_headers(member)).status_code == 403 + + +def test_admin_overview_shape(client, admin): + owner = make_member() + device = make_device(owner["uid"], "Overview laptop") + body = client.get("/admin/nodes", headers=admin_headers(admin)).json() + assert set(body["pool"]) >= {"online", "connected", "capacity", "open_circuits"} + assert body["queue"]["max_depth"] == LIMITS.max_queue_depth + assert body["limits"]["max_connected_nodes"] == LIMITS.max_connected_nodes + assert {item["key"] for item in body["settings"]} == { + "nodes_max_total", + "nodes_daily_quota_user", + "nodes_daily_quota_guest", + "nodes_cache_ttl_seconds", + "nodes_daily_social_user", + "nodes_social_via_nodes", + } + listed = {row["uid"]: row for row in body["devices"]} + assert listed[device["uid"]]["owner_username"] == owner["username"] + assert len(body["devices"]) <= 100 + assert "secret_hash" not in listed[device["uid"]] + + +def test_html_page_renders(client, admin): + response = client.get("/admin/nodes", headers=admin["headers"]) + assert response.status_code == 200 + assert "Community nodes" in response.text + assert "data-live=\"summary\"" in response.text diff --git a/tests/api/admin/nodes/killswitch.py b/tests/api/admin/nodes/killswitch.py new file mode 100644 index 00000000..87e71322 --- /dev/null +++ b/tests/api/admin/nodes/killswitch.py @@ -0,0 +1,24 @@ +# retoor + +from devplacepy.database import get_setting, set_setting +from tests.api.admin.nodes.conftest import admin_headers +from tests.api.nodes.conftest import make_member + + +def test_member_cannot_flip_the_switch(client): + member = make_member() + assert client.post("/admin/nodes/killswitch", json={"enabled": True}, headers=admin_headers(member)).status_code == 403 + + +def test_admin_sets_and_flips_the_switch(client, admin): + previous = get_setting("nodes_enabled", "0") + try: + set_setting("nodes_enabled", "0") + on = client.post("/admin/nodes/killswitch", json={"enabled": True}, headers=admin_headers(admin)) + assert on.json()["data"] == {"enabled": True} + assert get_setting("nodes_enabled", "0") == "1" + flipped = client.post("/admin/nodes/killswitch", json={}, headers=admin_headers(admin)) + assert flipped.json()["data"] == {"enabled": False} + assert get_setting("nodes_enabled", "1") == "0" + finally: + set_setting("nodes_enabled", previous) diff --git a/tests/api/admin/nodes/settings.py b/tests/api/admin/nodes/settings.py new file mode 100644 index 00000000..98fff63f --- /dev/null +++ b/tests/api/admin/nodes/settings.py @@ -0,0 +1,36 @@ +# retoor + +from devplacepy.database import get_setting, set_setting +from devplacepy.database.nodes import LIMITS +from tests.api.admin.nodes.conftest import admin_headers +from tests.api.nodes.conftest import make_member + +KEYS = ("nodes_max_total", "nodes_daily_quota_user", "nodes_social_via_nodes") + + +def test_member_cannot_change_settings(client): + member = make_member() + assert client.post("/admin/nodes/settings", json={"nodes_max_total": 1}, headers=admin_headers(member)).status_code == 403 + + +def test_only_submitted_fields_change_and_ceilings_clamp(client, admin): + previous = {key: get_setting(key, "") for key in KEYS} + try: + response = client.post( + "/admin/nodes/settings", + json={"nodes_max_total": LIMITS.max_connected_nodes + 500, "nodes_social_via_nodes": "1"}, + headers=admin_headers(admin), + ) + data = response.json()["data"] + assert "nodes_max_total" in data["clamped"] + assert get_setting("nodes_max_total", "") == str(LIMITS.max_connected_nodes) + assert get_setting("nodes_social_via_nodes", "") == "1" + assert get_setting("nodes_daily_quota_user", "") == previous["nodes_daily_quota_user"] + finally: + for key, value in previous.items(): + set_setting(key, value) + + +def test_negative_and_unknown_values_are_rejected(client, admin): + assert client.post("/admin/nodes/settings", json={"nodes_max_total": -1}, headers=admin_headers(admin)).status_code in (400, 422) + assert client.post("/admin/nodes/settings", json={"nodes_social_via_nodes": "2"}, headers=admin_headers(admin)).status_code in (400, 422) diff --git a/tests/api/admin/nodes/suspend.py b/tests/api/admin/nodes/suspend.py new file mode 100644 index 00000000..2fa63b10 --- /dev/null +++ b/tests/api/admin/nodes/suspend.py @@ -0,0 +1,47 @@ +# retoor + +from devplacepy.database.nodes import get_device +from tests.api.admin.nodes.conftest import admin_headers, make_admin, make_device +from tests.api.nodes.conftest import make_member + + +def test_member_cannot_suspend(client): + member = make_member() + device = make_device(member["uid"]) + response = client.post(f"/admin/nodes/devices/{device['uid']}/suspend", json={"minutes": 5}, headers=admin_headers(member)) + assert response.status_code == 403 + + +def test_suspend_and_unsuspend_round_trip(client, admin): + owner = make_member() + device = make_device(owner["uid"]) + url = f"/admin/nodes/devices/{device['uid']}" + response = client.post(f"{url}/suspend", json={"minutes": 30, "reason": "noisy"}, headers=admin_headers(admin)) + assert response.json()["data"] == {"uid": device["uid"], "minutes": 30} + row = get_device(device["uid"]) + assert row["status"] == "suspended" + assert row["suspend_reason"] == "noisy" + assert client.post(f"{url}/unsuspend", headers=admin_headers(admin)).status_code == 200 + assert get_device(device["uid"])["status"] == "active" + assert client.post(f"{url}/unsuspend", headers=admin_headers(admin)).status_code == 409 + + +def test_minutes_are_bounded(client, admin): + owner = make_member() + device = make_device(owner["uid"]) + url = f"/admin/nodes/devices/{device['uid']}/suspend" + for minutes in (0, 10081): + assert client.post(url, json={"minutes": minutes}, headers=admin_headers(admin)).status_code in (400, 422) + + +def test_unknown_device_is_404(client, admin): + response = client.post("/admin/nodes/devices/nope/suspend", json={"minutes": 5}, headers=admin_headers(admin)) + assert response.status_code == 404 + + +def test_junior_admin_cannot_suspend_a_senior_admins_device(client): + senior = make_admin() + junior = make_admin() + device = make_device(senior["uid"]) + client.post(f"/admin/nodes/devices/{device['uid']}/suspend", json={"minutes": 5}, headers=admin_headers(junior)) + assert get_device(device["uid"])["status"] == "active" diff --git a/tests/api/nodes/clients.py b/tests/api/nodes/clients.py new file mode 100644 index 00000000..77d3174f --- /dev/null +++ b/tests/api/nodes/clients.py @@ -0,0 +1,47 @@ +# retoor + +import hashlib + +import pytest + +from devplacepy import config + + +@pytest.fixture +def artifacts(tmp_path, monkeypatch): + monkeypatch.setattr(config, "NODE_CLIENTS_DIR", tmp_path) + (tmp_path / "devplace-node-linux-amd64").write_bytes(b"binary-bytes") + (tmp_path / "manifest.json").write_text('[{"version": "0.3.0"}]') + (tmp_path / "bad name.txt").write_text("x") + return tmp_path + + +def test_client_version_lists_artifacts_with_checksums(client, artifacts): + body = client.get("/nodes/client-version").json() + assert body["version"] == "0.3.0" and body["protocol"] >= 1 + names = [item["name"] for item in body["artifacts"]] + assert names == ["devplace-node-linux-amd64"] + assert body["artifacts"][0]["sha256"] == hashlib.sha256(b"binary-bytes").hexdigest() + + +def test_download_serves_the_file_with_its_checksum(client, artifacts): + response = client.get("/nodes/download/devplace-node-linux-amd64") + assert response.status_code == 200 and response.content == b"binary-bytes" + assert response.headers["x-content-sha256"] == hashlib.sha256(b"binary-bytes").hexdigest() + + +@pytest.mark.parametrize("name", ["manifest.json", "missing", "..%2Fsecret", "bad%20name.txt", ".hidden"]) +def test_download_refuses_everything_else(client, artifacts, name): + assert client.get(f"/nodes/download/{name}").status_code in (404, 422) + + +def test_download_refuses_a_symlink(client, artifacts): + outside = artifacts.parent / "outside.bin" + outside.write_bytes(b"secret") + (artifacts / "link").symlink_to(outside) + assert client.get("/nodes/download/link").status_code == 404 + + +def test_empty_directory_is_an_empty_list(client, tmp_path, monkeypatch): + monkeypatch.setattr(config, "NODE_CLIENTS_DIR", tmp_path / "absent") + assert client.get("/nodes/client-version").json()["artifacts"] == [] diff --git a/tests/api/nodes/jobs.py b/tests/api/nodes/jobs.py new file mode 100644 index 00000000..5040f5ae --- /dev/null +++ b/tests/api/nodes/jobs.py @@ -0,0 +1,187 @@ +# retoor + +import base64 +import json +import time + +import pytest +from starlette.websockets import WebSocketDisconnect + +from devplacepy.database import get_table +from devplacepy.database import nodes as store +from devplacepy.services.manager import service_manager +from devplacepy.services.nodes.broker import JobBroker +from devplacepy.services.nodes.policy import DestinationPolicy +from devplacepy.services.nodes.spec import CLOSE_PROTOCOL, SPEC +from tests.api.nodes.conftest import make_member +from tests.api.nodes.ws import hello, ticket_for + +URL = "https://www.reddit.com/r/python.json" +BROKER = JobBroker() + + +@pytest.fixture +def relaxed(client): + service = service_manager.get_service("nodes") + original = service.scheduler.destination_ready + service.scheduler.destination_ready = lambda target: True + get_table("node_requests").delete() + yield service + service.scheduler.destination_ready = original + get_table("node_requests").delete() + + +def enqueue(caller="caller-1", kind="user", url=URL): + target = DestinationPolicy().check(url) + return BROKER.enqueue(kind, caller, target, "any") + + +def connect(client, member): + device_uid, _, ticket = ticket_for(client, member) + return device_uid, client.websocket_connect(f"/nodes/ws?ticket={ticket}") + + +def next_job(ws, timeout=5.0): + deadline = time.time() + timeout + while time.time() < deadline: + frame = json.loads(ws.receive_text()) + if frame["type"] == "job": + return frame + if frame["type"] == "ping": + ws.send_text(json.dumps({"type": "pong"})) + raise AssertionError("no job arrived") + + +def wait_for(uid, status, timeout=5.0): + deadline = time.time() + timeout + while time.time() < deadline: + row = store.get_request(uid) + if row["status"] == status: + return row + time.sleep(0.05) + raise AssertionError(f"request stayed {store.get_request(uid)['status']}") + + +def test_a_queued_request_reaches_the_node_and_the_result_comes_back(client, relaxed, member): + device_uid, connection = connect(client, member) + with connection as ws: + ws.send_text(hello(modes=["fetch"])) + assert json.loads(ws.receive_text())["type"] == "welcome" + uid = enqueue() + job = next_job(ws) + assert job["job_id"] == uid and job["url"] == URL and job["method"] == "GET" + assert job["timeout"] == SPEC.limits.job_timeout_seconds + ws.send_text( + json.dumps( + { + "type": "job_result", + "job_id": uid, + "status": 200, + "headers": {"Content-Type": "application/json"}, + "body_b64": base64.b64encode(b'{"ok": true}').decode(), + } + ) + ) + wait_for(uid, "done") + result = BROKER.read(uid) + assert result.http_status == 200 and result.body == b'{"ok": true}' + assert result.headers == {"content-type": "application/json"} + today = store.usage_today(device_uid) + assert today["requests_ok"] == 1 and today["bytes_down"] == 12 + assert store.get_donor(member["uid"])["requests_ok"] == 1 + + +def test_a_node_that_speaks_no_known_transport_gets_nothing(client, relaxed, member): + _, connection = connect(client, member) + with connection as ws: + ws.send_text(hello(modes=["any"])) + ws.receive_text() + uid = enqueue() + time.sleep(0.7) + assert store.get_request(uid)["status"] == "queued" + + +def test_a_donor_never_receives_its_own_request(client, relaxed, member): + _, connection = connect(client, member) + with connection as ws: + ws.send_text(hello(modes=["fetch"])) + ws.receive_text() + uid = enqueue(caller=member["uid"]) + time.sleep(0.7) + assert store.get_request(uid)["status"] == "queued" + + +def test_a_donor_opt_out_of_the_platform_is_respected_live(client, relaxed, member): + store.save_node_settings(member["uid"], {"allowed_platforms": []}) + _, connection = connect(client, member) + with connection as ws: + ws.send_text(hello(modes=["fetch"])) + assert json.loads(ws.receive_text())["settings"]["allowed_platforms"] == [] + uid = enqueue() + time.sleep(0.7) + assert store.get_request(uid)["status"] == "queued" + + +def test_a_job_error_marks_the_request_failed(client, relaxed, member): + _, connection = connect(client, member) + with connection as ws: + ws.send_text(hello(modes=["fetch"])) + ws.receive_text() + uid = enqueue() + job = next_job(ws) + ws.send_text(json.dumps({"type": "job_error", "job_id": job["job_id"], "code": "timeout"})) + row = wait_for(uid, "failed") + assert row["error"] == "timeout" + + +def test_disconnecting_with_a_job_in_flight_fails_it(client, relaxed, member): + _, connection = connect(client, member) + with connection as ws: + ws.send_text(hello(modes=["fetch"])) + ws.receive_text() + uid = enqueue() + next_job(ws) + row = wait_for(uid, "failed") + assert row["error"] == "node_disconnected" + + +def test_a_forged_result_is_an_integrity_failure_and_a_flag(client, relaxed, member): + _, connection = connect(client, member) + with connection as ws: + ws.send_text(hello(modes=["fetch"])) + ws.receive_text() + uid = enqueue() + next_job(ws) + ws.send_text( + json.dumps({"type": "job_result", "job_id": uid, "status": 200, "body_b64": "***"}) + ) + row = wait_for(uid, "failed") + assert row["error"] == "bad_base64" + assert any(flag["kind"] == "integrity" for flag in store.list_flags()) + + +def test_answering_a_job_that_was_never_assigned_closes_the_socket(client, relaxed, member): + _, connection = connect(client, member) + with connection as ws: + ws.send_text(hello(modes=["fetch"])) + ws.receive_text() + ws.send_text( + json.dumps({"type": "job_result", "job_id": "made-up", "status": 200, "body_b64": ""}) + ) + with pytest.raises(WebSocketDisconnect) as info: + ws.receive_text() + assert info.value.code == CLOSE_PROTOCOL + + +def test_two_donors_split_two_requests(client, relaxed): + first, second = make_member(), make_member() + _, one = connect(client, first) + _, two = connect(client, second) + with one as ws_one, two as ws_two: + for ws in (ws_one, ws_two): + ws.send_text(hello(modes=["fetch"])) + ws.receive_text() + a = enqueue(caller="x") + b = enqueue(caller="y", url="https://old.reddit.com/r/rust.json") + jobs = {next_job(ws_one)["job_id"], next_job(ws_two)["job_id"]} + assert jobs == {a, b} diff --git a/tests/api/nodes/me.py b/tests/api/nodes/me.py new file mode 100644 index 00000000..fa79fd46 --- /dev/null +++ b/tests/api/nodes/me.py @@ -0,0 +1,84 @@ +# retoor + +from devplacepy.database import set_consent +from devplacepy.database.nodes import record_job, save_node_settings +from tests.api.nodes.conftest import make_member + + +def test_guests_get_401(client): + assert client.get("/nodes/me").status_code == 401 + assert client.post("/nodes/settings", json={"enabled": False}).status_code == 401 + + +def test_me_reports_status_and_never_caches(client, member): + response = client.get("/nodes/me", headers=member["headers"]) + assert response.status_code == 200 + assert response.headers["cache-control"] == "no-store" + body = response.json() + assert body["consent"] is True and body["can_donate"] is True + assert body["settings"]["enabled"] is True + assert body["devices"] == [] and body["max_devices"] > 0 + assert body["donated"]["lifetime_requests"] == 0 + assert "reddit" in body["platforms"] + + +def test_me_explains_why_a_member_cannot_donate(client): + who = make_member(consent=False, enabled=False) + body = client.get("/nodes/me", headers=who["headers"]).json() + assert body["can_donate"] is False + assert body["reason"] and body["message"] + + +def test_settings_are_clamped_and_unsent_fields_keep_their_value(client, member): + response = client.post( + "/nodes/settings", + json={"bandwidth_mb_day": 100, "active_from": "08:00", "active_to": "22:00", "timezone": "Europe/Amsterdam"}, + headers=member["headers"], + ) + assert response.status_code == 200 + settings = response.json()["settings"] + assert settings["bandwidth_mb_day"] == 100 and settings["active_from"] == "08:00" + assert settings["enabled"] is True + again = client.post("/nodes/settings", json={"max_concurrent": 1}, headers=member["headers"]).json() + assert again["settings"]["bandwidth_mb_day"] == 100 and again["settings"]["max_concurrent"] == 1 + too_big = client.post("/nodes/settings", json={"bandwidth_mb_day": 10**9}, headers=member["headers"]) + assert too_big.status_code in (400, 422) + + +def test_switching_on_needs_the_consent(client): + who = make_member(consent=False, enabled=False) + refused = client.post("/nodes/settings", json={"enabled": True}, headers=who["headers"]) + assert refused.status_code == 403 + assert refused.json()["error"]["reason"] == "no_consent" + set_consent("user", who["uid"], "bandwidth_sharing", True) + assert client.post("/nodes/settings", json={"enabled": True}, headers=who["headers"]).status_code == 200 + + +def test_the_assistant_can_switch_off_but_never_on(client): + who = make_member(enabled=False) + headers = {**who["headers"], "X-Devii-Agent": "1"} + refused = client.post("/nodes/settings", json={"enabled": True}, headers=headers) + assert refused.status_code == 403 + save_node_settings(who["uid"], {"enabled": True}) + off = client.post("/nodes/settings", json={"enabled": False}, headers=headers) + assert off.status_code == 200 and off.json()["settings"]["enabled"] is False + + +def test_stats_are_aggregate_only(client, member): + device = client.post("/nodes/devices", json={"label": "x"}, headers=member["headers"]).json()["device"] + record_job(device["uid"], member["uid"], ok=True, bytes_up=10, bytes_down=90) + body = client.get("/nodes/stats").json() + assert set(body) == {"requests", "bytes", "donors"} + assert body["requests"] >= 1 and body["bytes"] >= 100 and body["donors"] >= 1 + assert member["username"] not in str(body) + + +def test_leaderboard_lists_only_members_who_opted_in(client): + hidden = make_member() + shown = make_member() + for who, public in ((hidden, False), (shown, True)): + save_node_settings(who["uid"], {"public_display": public}) + device = client.post("/nodes/devices", json={"label": "x"}, headers=who["headers"]).json()["device"] + record_job(device["uid"], who["uid"], ok=True, bytes_up=1, bytes_down=5) + names = [item["username"] for item in client.get("/nodes/leaderboard").json()["donors"]] + assert shown["username"] in names and hidden["username"] not in names diff --git a/tests/api/nodes/social.py b/tests/api/nodes/social.py new file mode 100644 index 00000000..9bb84bc8 --- /dev/null +++ b/tests/api/nodes/social.py @@ -0,0 +1,65 @@ +# retoor + +import json + +from devplacepy.database import get_setting, set_setting +from devplacepy.services.nodes import social as social_module + +HN = json.dumps({"hits": [{"objectID": "9", "title": "A story", "points": 4}]}).encode() + + +def test_guests_get_401(client): + assert client.get("/nodes/social/search?q=x").status_code == 401 + assert client.post("/nodes/internal/fetch", json={"url": "https://www.reddit.com/"}).status_code == 401 + + +def test_query_is_required(client, member): + assert client.get("/nodes/social/search", headers=member["headers"]).status_code == 400 + + +def test_search_returns_hits_and_reasons(client, member, monkeypatch): + async def getter(url): + return HN + + monkeypatch.setattr(social_module, "direct_get", getter) + body = client.get( + "/nodes/social/search?q=sqlite&platforms=hackernews,reddit", headers=member["headers"] + ).json() + assert body["hits"][0]["platform"] == "hackernews" + assert body["unavailable"] == {"reddit": "nodes_not_used"} + assert body["platforms"] == ["hackernews", "reddit"] + + +def test_search_has_a_daily_quota(client, member, monkeypatch): + async def getter(url): + return HN + + monkeypatch.setattr(social_module, "direct_get", getter) + previous = get_setting("nodes_daily_social_user", "300") + set_setting("nodes_daily_social_user", "2") + try: + codes = [ + client.get("/nodes/social/search?q=a&platforms=hackernews", headers=member["headers"]).status_code + for _ in range(3) + ] + finally: + set_setting("nodes_daily_social_user", previous) + assert codes == [200, 200, 429] + + +def test_internal_fetch_refuses_hosts_off_the_allowlist(client, member): + response = client.post( + "/nodes/internal/fetch", json={"url": "https://example.com/"}, headers=member["headers"] + ) + assert response.status_code == 403 + assert response.json()["error"]["reason"] == "host_not_allowed" + + +def test_internal_fetch_without_donors_is_503(client, member): + response = client.post( + "/nodes/internal/fetch", + json={"url": "https://www.reddit.com/r/python.json", "cache": False}, + headers=member["headers"], + ) + assert response.status_code == 503 + assert response.json()["error"]["reason"] in ("no_nodes", "kill_switch") diff --git a/tests/api/sync/__init__.py b/tests/api/sync/__init__.py new file mode 100644 index 00000000..95ee7c3e --- /dev/null +++ b/tests/api/sync/__init__.py @@ -0,0 +1 @@ +# retoor diff --git a/tests/api/sync/comments.py b/tests/api/sync/comments.py new file mode 100644 index 00000000..8824f744 --- /dev/null +++ b/tests/api/sync/comments.py @@ -0,0 +1,70 @@ +# retoor + +from devplacepy.database import get_table +from tests.api.sync.conftest import comment_record, post_record, sync_ok, unique, user_record + + +def seed_thread(client, admin, origin): + sync_ok(client, admin, "users", origin, [user_record("u1", unique("cu1")), user_record("u2", unique("cu2"))]) + sync_ok(client, admin, "posts", origin, [post_record("p1", "u1"), post_record("p2", "u1")]) + + +def test_unknown_references_are_rejected_with_specific_reasons(client, admin, origin): + seed_thread(client, admin, origin) + body = sync_ok( + client, admin, "comments", origin, + [ + comment_record("c1", author="nobody"), + comment_record("c2", post="nopost"), + comment_record("c3", parent_uid="noparent"), + ], + ) + assert [r["reason"] for r in body["results"]] == ["unknown_author", "unknown_post", "unknown_parent"] + + +def test_create_with_nested_reply_in_one_batch(client, admin, origin): + seed_thread(client, admin, origin) + body = sync_ok( + client, admin, "comments", origin, + [comment_record("c1", author="u2"), comment_record("c2", author="u1", parent_uid="c1")], + ) + assert [r["status"] for r in body["results"]] == ["created", "created"] + parent, reply = (get_table("comments").find_one(uid=r["local_uid"]) for r in body["results"]) + post = get_table("posts").find_one(uid=reply["post_uid"]) + assert reply["parent_uid"] == parent["uid"] + assert reply["target_type"] == "post" and reply["target_uid"] == post["uid"] + assert parent["deleted_at"] is None + + +def test_parent_on_a_different_post_is_rejected(client, admin, origin): + seed_thread(client, admin, origin) + sync_ok(client, admin, "comments", origin, [comment_record("c1", post="p1")]) + body = sync_ok(client, admin, "comments", origin, [comment_record("c2", post="p2", parent_uid="c1")]) + assert body["results"][0]["reason"] == "parent_mismatch" + + +def test_edit_replay_and_tombstone(client, admin, origin): + seed_thread(client, admin, origin) + created = sync_ok(client, admin, "comments", origin, [comment_record("c1")])["results"][0] + assert sync_ok(client, admin, "comments", origin, [comment_record("c1")])["results"][0]["status"] == "unchanged" + edited = sync_ok( + client, admin, "comments", origin, + [comment_record("c1", content="edited comment", updated_at="2025-02-01T00:00:00Z")], + )["results"][0] + assert edited["status"] == "updated" + assert get_table("comments").find_one(uid=created["local_uid"])["content"] == "edited comment" + sync_ok( + client, admin, "comments", origin, + [comment_record("c1", content="edited comment", updated_at="2025-03-01T00:00:00Z", deleted=True)], + ) + row = get_table("comments").find_one(uid=created["local_uid"]) + assert row["deleted_at"] and row["deleted_by"] == "sync" + + +def test_synced_comment_renders_on_the_post_page(client, admin, origin): + seed_thread(client, admin, origin) + sync_ok(client, admin, "comments", origin, [comment_record("c1", content="comment visible on the page")]) + link = get_table("sync_links").find_one(origin=origin, kind="posts", origin_uid="p1") + slug = get_table("posts").find_one(uid=link["local_uid"])["slug"] + page = client.get(f"/posts/{slug}") + assert page.status_code == 200 and "comment visible on the page" in page.text diff --git a/tests/api/sync/conftest.py b/tests/api/sync/conftest.py new file mode 100644 index 00000000..9c9f6695 --- /dev/null +++ b/tests/api/sync/conftest.py @@ -0,0 +1,98 @@ +# retoor + +import time + +import pytest +from fastapi.testclient import TestClient + +from devplacepy.database import get_setting, get_table, set_setting +from devplacepy.main import app +from devplacepy.utils import register_account + +_counter = [0] + + +def unique(prefix: str) -> str: + _counter[0] += 1 + return f"{prefix}{int(time.time() * 1000)}{_counter[0]}" + + +def make_account(role: str = "Member") -> dict: + name = unique("syncmb") + uid, _, _ = register_account( + name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True + ) + if role != "Member": + get_table("users").update({"uid": uid, "role": role}, ["uid"]) + row = get_table("users").find_one(uid=uid) + return { + "uid": uid, + "username": name, + "headers": {"X-API-KEY": row["api_key"], "Accept": "application/json"}, + } + + +@pytest.fixture(scope="module") +def client(): + previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")} + set_setting("sync_enabled", "1") + set_setting("sync_origins", "") + with TestClient(app) as test_client: + yield test_client + for key, value in previous.items(): + set_setting(key, value) + + +@pytest.fixture +def admin(client): + return make_account("Admin") + + +@pytest.fixture +def origin(): + return f"{unique('o')}.example.org" + + +def sync(client, admin, kind, origin, records, **extra): + response = client.post( + f"/sync/{kind}", + json={"origin": origin, "records": records, **extra}, + headers=admin["headers"], + ) + return response + + +def sync_ok(client, admin, kind, origin, records, **extra) -> dict: + response = sync(client, admin, kind, origin, records, **extra) + assert response.status_code == 200, response.text + return response.json() + + +def user_record(uid="u1", username="syncuser", **extra) -> dict: + return { + "origin_uid": uid, + "username": username, + "created_at": "2025-01-01T00:00:00Z", + **extra, + } + + +def post_record(uid="p1", author="u1", **extra) -> dict: + return { + "origin_uid": uid, + "author_uid": author, + "content": "a synced post with enough content", + "created_at": "2025-01-02T00:00:00Z", + **extra, + } + + +def comment_record(uid="c1", post="p1", author="u1", **extra) -> dict: + return { + "origin_uid": uid, + "post_uid": post, + "author_uid": author, + "content": "a synced comment", + "created_at": "2025-01-03T00:00:00Z", + **extra, + } diff --git a/tests/api/sync/posts.py b/tests/api/sync/posts.py new file mode 100644 index 00000000..f1db1850 --- /dev/null +++ b/tests/api/sync/posts.py @@ -0,0 +1,118 @@ +# retoor + +from devplacepy.database import get_table +from tests.api.sync.conftest import post_record, sync_ok, unique, user_record + + +def seed_author(client, admin, origin, uid="u1"): + body = sync_ok(client, admin, "users", origin, [user_record(uid, unique("author"))]) + return body["results"][0]["local_uid"] + + +def test_unknown_author_is_rejected_with_reason(client, admin, origin): + body = sync_ok(client, admin, "posts", origin, [post_record(author="missing")]) + result = body["results"][0] + assert result["status"] == "rejected" and result["reason"] == "unknown_author" + assert get_table("sync_links").count(origin=origin) == 0 + + +def test_create_normalizes_and_has_no_side_effects(client, admin, origin): + author = seed_author(client, admin, origin) + notifications_before = get_table("notifications").count() + body = sync_ok( + client, admin, "posts", origin, + [post_record("p1", title=" Hello World ", content="line one\r\nline two ok", topic="nonsense", stars=50)], + ) + result = body["results"][0] + assert result["status"] == "created" and result["ignored"] == ["stars"] + row = get_table("posts").find_one(uid=result["local_uid"]) + assert row["user_uid"] == author + assert row["title"] == "Hello World" + assert row["content"] == "line one\nline two ok" + assert row["topic"] == "random" + assert row["stars"] == 0 + assert row["created_at"] == "2025-01-02T00:00:00+00:00" + assert row["slug"].endswith("hello-world") + assert row["deleted_at"] is None + assert get_table("notifications").count() == notifications_before + assert get_table("users").find_one(uid=author)["xp"] == 0 + + +def test_replay_update_stale_and_tombstone(client, admin, origin): + seed_author(client, admin, origin) + created = sync_ok(client, admin, "posts", origin, [post_record("p1")])["results"][0] + assert sync_ok(client, admin, "posts", origin, [post_record("p1")])["results"][0]["status"] == "unchanged" + edited = sync_ok( + client, admin, "posts", origin, + [post_record("p1", content="edited synced content", updated_at="2025-03-01T00:00:00Z")], + )["results"][0] + assert edited["status"] == "updated" and edited["local_uid"] == created["local_uid"] + assert get_table("posts").find_one(uid=created["local_uid"])["content"] == "edited synced content" + stale = sync_ok( + client, admin, "posts", origin, + [post_record("p1", content="older content here", updated_at="2025-01-05T00:00:00Z")], + )["results"][0] + assert stale["status"] == "stale" + deleted = sync_ok( + client, admin, "posts", origin, + [post_record("p1", content="edited synced content", updated_at="2025-04-01T00:00:00Z", deleted=True)], + )["results"][0] + assert deleted["status"] == "updated" + row = get_table("posts").find_one(uid=created["local_uid"]) + assert row["deleted_at"] and row["deleted_by"] == "sync" + revived = sync_ok( + client, admin, "posts", origin, + [post_record("p1", content="edited synced content", updated_at="2025-05-01T00:00:00Z")], + )["results"][0] + assert revived["status"] == "updated" + assert get_table("posts").find_one(uid=created["local_uid"])["deleted_at"] is None + + +def test_locally_moderated_post_is_not_resurrected(client, admin, origin): + seed_author(client, admin, origin) + created = sync_ok(client, admin, "posts", origin, [post_record("p1")])["results"][0] + get_table("posts").update( + {"uid": created["local_uid"], "deleted_at": "2025-02-01T00:00:00+00:00", "deleted_by": admin["uid"]}, + ["uid"], + ) + again = sync_ok( + client, admin, "posts", origin, + [post_record("p1", content="remote edit arrives", updated_at="2025-03-01T00:00:00Z")], + )["results"][0] + assert again["status"] == "unchanged" and again["reason"] == "locally_deleted" + assert get_table("posts").find_one(uid=created["local_uid"])["content"] != "remote edit arrives" + + +def test_purged_local_copy_is_not_recreated(client, admin, origin): + seed_author(client, admin, origin) + created = sync_ok(client, admin, "posts", origin, [post_record("p1")])["results"][0] + get_table("posts").delete(uid=created["local_uid"]) + again = sync_ok( + client, admin, "posts", origin, + [post_record("p1", content="changed after purge", updated_at="2025-03-01T00:00:00Z")], + )["results"][0] + assert again["status"] == "rejected" and again["reason"] == "purged_locally" + assert get_table("posts").count(uid=created["local_uid"]) == 0 + + +def test_invalid_post_reports_field_errors(client, admin, origin): + seed_author(client, admin, origin) + body = sync_ok(client, admin, "posts", origin, [post_record("p1", content="short")]) + result = body["results"][0] + assert result["status"] == "rejected" and result["reason"] == "invalid" + assert any(error.startswith("content:") for error in result["errors"]) + + +def test_future_timestamps_are_clamped(client, admin, origin): + seed_author(client, admin, origin) + body = sync_ok(client, admin, "posts", origin, [post_record("p1", created_at="2999-01-01T00:00:00Z")]) + row = get_table("posts").find_one(uid=body["results"][0]["local_uid"]) + assert row["created_at"] < "2100" + + +def test_post_shows_up_on_its_detail_page(client, admin, origin): + seed_author(client, admin, origin) + body = sync_ok(client, admin, "posts", origin, [post_record("p1", title="Visible synced post")]) + slug = get_table("posts").find_one(uid=body["results"][0]["local_uid"])["slug"] + page = client.get(f"/posts/{slug}") + assert page.status_code == 200 and "Visible synced post" in page.text diff --git a/tests/api/sync/settings.py b/tests/api/sync/settings.py new file mode 100644 index 00000000..dc1215ac --- /dev/null +++ b/tests/api/sync/settings.py @@ -0,0 +1,39 @@ +# retoor + +from devplacepy.database import get_setting, set_setting +from tests.api.sync.conftest import sync_ok, unique, user_record + + +def test_admin_settings_save_normalizes_sync_origins(client, admin): + previous = get_setting("sync_origins", "") + try: + response = client.post( + "/admin/settings", + json={"sync_origins": "A.example.org, b.example.org\nA.example.org"}, + headers=admin["headers"], + ) + assert response.status_code == 200, response.text + set_setting_cache_free = get_setting("sync_origins", "") + assert set_setting_cache_free == "a.example.org,b.example.org" + refused = client.post( + "/admin/settings", json={"sync_origins": "bad origin!"}, headers=admin["headers"] + ) + assert refused.status_code in (400, 422) + finally: + set_setting("sync_origins", previous) + + +def test_admin_settings_page_shows_sync_controls(client, admin): + page = client.get("/admin/settings", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]}) + assert page.status_code == 200 + assert 'name="sync_enabled"' in page.text and 'name="sync_origins"' in page.text + + +def test_clearing_sync_origins_removes_the_allowlist(client, admin, origin): + set_setting("sync_origins", "only.example.org") + try: + client.post("/admin/settings", json={"sync_origins": ""}, headers=admin["headers"]) + assert get_setting("sync_origins", "x") == "" + sync_ok(client, admin, "users", origin, [user_record("u1", unique("any"))]) + finally: + set_setting("sync_origins", "") diff --git a/tests/api/sync/spec.py b/tests/api/sync/spec.py new file mode 100644 index 00000000..9f2fb260 --- /dev/null +++ b/tests/api/sync/spec.py @@ -0,0 +1,15 @@ +# retoor + +from devplacepy.services.sync.spec import SPEC + + +def test_spec_is_public_and_matches_code(client): + response = client.get("/sync/spec.json") + assert response.status_code == 200 + body = response.json() + assert body["kinds"] == list(SPEC.kinds) + assert body["statuses"] == list(SPEC.statuses) + assert body["limits"]["max_batch_records"] == SPEC.limits.max_batch_records + assert set(body["records"]) == set(SPEC.kinds) + assert "username" in body["records"]["users"]["properties"] + assert "stars" not in body["records"]["posts"]["properties"] diff --git a/tests/api/sync/users.py b/tests/api/sync/users.py new file mode 100644 index 00000000..5ed8e4ec --- /dev/null +++ b/tests/api/sync/users.py @@ -0,0 +1,173 @@ +# retoor + +from devplacepy.database import get_table, set_setting +from tests.api.sync.conftest import make_account, sync, sync_ok, unique, user_record + + +def test_requires_authentication(client, origin): + response = client.post( + "/sync/users", json={"origin": origin, "records": [user_record()]} + ) + assert response.status_code == 401 + + +def test_member_is_refused(client, origin): + member = make_account() + response = sync(client, member, "users", origin, [user_record()]) + assert response.status_code == 403 + + +def test_kill_switch_refuses_when_disabled(client, admin, origin): + set_setting("sync_enabled", "0") + try: + response = sync(client, admin, "users", origin, [user_record()]) + assert response.status_code == 503 + finally: + set_setting("sync_enabled", "1") + + +def test_origin_allowlist(client, admin, origin): + set_setting("sync_origins", "trusted.example.org") + try: + assert sync(client, admin, "users", origin, [user_record()]).status_code == 403 + allowed = sync(client, admin, "users", "trusted.example.org", [user_record(unique("a"), unique("allow"))]) + assert allowed.status_code == 200 + finally: + set_setting("sync_origins", "") + + +def test_invalid_envelope_is_422(client, admin, origin): + assert sync(client, admin, "users", origin, []).status_code == 422 + assert sync(client, admin, "users", "bad origin!", [user_record()]).status_code == 422 + + +def test_create_then_replay_is_idempotent(client, admin, origin): + name = unique("bob") + first = sync_ok(client, admin, "users", origin, [user_record("u1", name, bio="hi")]) + result = first["results"][0] + assert result["status"] == "created" and result["local_uid"] + again = sync_ok(client, admin, "users", origin, [user_record("u1", name, bio="hi")]) + assert again["results"][0]["status"] == "unchanged" + assert again["results"][0]["local_uid"] == result["local_uid"] + assert get_table("users").count(username=name) == 1 + + +def test_update_changes_profile_and_keeps_local_uid(client, admin, origin): + name = unique("upd") + created = sync_ok(client, admin, "users", origin, [user_record("u1", name)]) + uid = created["results"][0]["local_uid"] + updated = sync_ok( + client, + admin, + "users", + origin, + [user_record("u1", name, bio="new bio", updated_at="2025-06-01T00:00:00Z")], + ) + assert updated["results"][0]["status"] == "updated" + assert get_table("users").find_one(uid=uid)["bio"] == "new bio" + + +def test_older_update_is_stale(client, admin, origin): + name = unique("stale") + sync_ok(client, admin, "users", origin, [user_record("u1", name, updated_at="2025-06-01T00:00:00Z")]) + older = sync_ok( + client, admin, "users", origin, + [user_record("u1", name, bio="old", updated_at="2025-02-01T00:00:00Z")], + ) + assert older["results"][0]["status"] == "stale" + + +def test_username_is_normalized_and_collisions_get_a_suffix(client, admin, origin): + taken = unique("taken") + make_account_row = get_table("users") + sync_ok(client, admin, "users", origin, [user_record("u0", taken)]) + other = sync_ok(client, admin, "users", origin, [user_record("u1", taken)]) + local = make_account_row.find_one(uid=other["results"][0]["local_uid"]) + assert local["username"] != taken and local["username"].startswith(taken[:10]) + odd = sync_ok(client, admin, "users", origin, [user_record("u2", "Ünï cøde!")]) + created = make_account_row.find_one(uid=odd["results"][0]["local_uid"]) + assert created["username"].replace("_", "").replace("-", "").isalnum() + + +def test_counters_credentials_and_role_are_ignored_and_reported(client, admin, origin): + body = sync_ok( + client, admin, "users", origin, + [user_record("u1", unique("priv"), xp=999, level=50, stars=7, role="Admin", email="x@y.z", password_hash="p")], + ) + result = body["results"][0] + assert set(result["ignored"]) == {"xp", "level", "stars", "role", "email", "password_hash"} + row = get_table("users").find_one(uid=result["local_uid"]) + assert row["role"] == "Member" and row["xp"] == 0 and row["level"] == 1 + assert row["email"].endswith("@sync.invalid") + + +def test_synced_account_cannot_log_in(client, admin, origin): + body = sync_ok(client, admin, "users", origin, [user_record("u1", unique("nologin"))]) + row = get_table("users").find_one(uid=body["results"][0]["local_uid"]) + response = client.post( + "/auth/login", + json={"email": row["email"], "password": "anything1"}, + headers={"Accept": "application/json"}, + ) + assert response.status_code == 401 + + +def test_dry_run_writes_nothing(client, admin, origin): + name = unique("dry") + body = sync_ok(client, admin, "users", origin, [user_record("u1", name)], dry_run=True) + assert body["dry_run"] is True and body["results"][0]["status"] == "created" + assert get_table("users").count(username=name) == 0 + assert get_table("sync_links").count(origin=origin) == 0 + + +def test_one_bad_record_does_not_fail_the_batch(client, admin, origin): + body = sync_ok( + client, admin, "users", origin, + [user_record("u1", unique("good")), {"origin_uid": "", "username": "x"}, user_record("u3", unique("good"), created_at="nonsense")], + ) + assert [r["status"] for r in body["results"]] == ["created", "rejected", "rejected"] + assert body["counts"]["created"] == 1 and body["counts"]["rejected"] == 2 + assert all(r["errors"] for r in body["results"][1:]) + + +def test_deleted_flag_deactivates_account(client, admin, origin): + name = unique("gone") + created = sync_ok(client, admin, "users", origin, [user_record("u1", name)]) + sync_ok(client, admin, "users", origin, [user_record("u1", name, deleted=True, updated_at="2025-05-01T00:00:00Z")]) + assert not get_table("users").find_one(uid=created["results"][0]["local_uid"])["is_active"] + + +def test_tombstone_for_unknown_record_creates_nothing(client, admin, origin): + body = sync_ok(client, admin, "users", origin, [user_record("u1", unique("ghost"), deleted=True)]) + assert body["results"][0]["status"] == "unchanged" + assert body["results"][0]["reason"] == "tombstone_for_unknown" + assert get_table("sync_links").count(origin=origin) == 0 + + +def test_same_origin_uid_in_different_origins_are_distinct(client, admin, origin): + first = sync_ok(client, admin, "users", origin, [user_record("u1", unique("one"))]) + second = sync_ok(client, admin, "users", f"b.{origin}", [user_record("u1", unique("two"))]) + assert first["results"][0]["local_uid"] != second["results"][0]["local_uid"] + + +def test_form_encoded_body_with_json_records_is_accepted(client, admin, origin): + import json + + name = unique("form") + response = client.post( + "/sync/users", + data={"origin": origin, "records": json.dumps([user_record("u1", name)]), "dry_run": "true"}, + headers=admin["headers"], + ) + assert response.status_code == 200, response.text + assert response.json()["results"][0]["status"] == "created" + assert get_table("users").count(username=name) == 0 + + +def test_malformed_records_string_is_422(client, admin, origin): + response = client.post( + "/sync/users", + data={"origin": origin, "records": "not json"}, + headers=admin["headers"], + ) + assert response.status_code in (400, 422) diff --git a/tests/conftest.py b/tests/conftest.py index 6c3a239c..e97c9b9c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,6 +38,7 @@ os.environ["DEVPLACE_DISABLE_RATE_LIMIT"] = "1" os.environ["DEVPLACE_SITEMAP_TTL"] = "0" os.environ["DEVPLACE_HOME_CACHE_TTL"] = "0" os.environ["DEVPLACE_RANKING_TTL"] = "0" +os.environ["DEVPLACE_NODES_STATS_TTL"] = "0" os.environ["DEVPLACE_MARKET_SATURATION_TTL"] = "0" os.environ["DEVPLACE_HAPPY_404_POOL_TTL"] = "0" os.environ["DEVPLACE_FEATURED_TOPICS_POOL_TTL"] = "0" diff --git a/tests/unit/cli/nodes.py b/tests/unit/cli/nodes.py new file mode 100644 index 00000000..26ca3448 --- /dev/null +++ b/tests/unit/cli/nodes.py @@ -0,0 +1,44 @@ +# retoor + +import argparse + +from devplacepy.cli import nodes as cli +from devplacepy.database import get_table +from devplacepy.database import nodes as store +from devplacepy.utils import generate_uid + + +def parser(): + root = argparse.ArgumentParser() + cli.register_nodes(root.add_subparsers(dest="command")) + return root + + +def device(local_db): + row, _ = store.create_device(generate_uid(), "cli", "native") + return row["uid"] + + +def test_suspend_and_unsuspend(local_db, capsys): + uid = device(local_db) + args = parser().parse_args(["nodes", "suspend", uid, "--minutes", "5"]) + args.func(args) + assert store.effective_status(get_table("node_devices").find_one(uid=uid)) == "suspended" + args = parser().parse_args(["nodes", "unsuspend", uid]) + args.func(args) + assert store.effective_status(get_table("node_devices").find_one(uid=uid)) == "active" + + +def test_clear_is_a_dry_run_without_force(local_db, capsys): + get_table("node_events").insert({"uid": generate_uid(), "kind": "x", "detail": "", "created_at": "2026-01-01T00:00:00+00:00"}) + before = store.clear_ephemeral(dry_run=True)["node_events"] + args = parser().parse_args(["nodes", "clear"]) + args.func(args) + assert "pass --force" in capsys.readouterr().out + assert store.clear_ephemeral(dry_run=True)["node_events"] == before + + +def test_prune_dry_run_reports_counts(local_db, capsys): + args = parser().parse_args(["nodes", "prune", "--dry-run"]) + args.func(args) + assert "would remove" in capsys.readouterr().out diff --git a/tests/unit/database/nodes.py b/tests/unit/database/nodes.py index 115dc845..b8ad604d 100644 --- a/tests/unit/database/nodes.py +++ b/tests/unit/database/nodes.py @@ -225,3 +225,170 @@ def test_ip_is_only_ever_stored_hashed(): assert "203" not in digest and len(digest) == 32 assert store.hash_ip("203.0.113.9") == digest assert store.hash_ip("") == "" + + +def queue_one(url="https://reddit.com/r/x", caller="c1"): + return store.enqueue_request("user", caller, url, "GET", "reddit", "any") + + +def test_the_queue_is_bounded(local_db): + get_table("node_requests").delete() + for _ in range(LIMITS.max_queue_depth): + queue_one() + with pytest.raises(store.QueueFull): + queue_one() + assert store.count_open_requests() == LIMITS.max_queue_depth + first = store.list_queued(1)[0] + store.fail_request(first["uid"], "x") + queue_one() + get_table("node_requests").delete() + + +def test_a_request_is_claimed_by_exactly_one_assignment(local_db): + get_table("node_requests").delete() + uid = queue_one() + assert store.claim_request(uid, "device-a") is True + assert store.claim_request(uid, "device-b") is False + assert store.get_request(uid)["device_uid"] == "device-a" + get_table("node_requests").delete() + + +def test_a_result_can_only_land_on_an_assigned_request(local_db): + get_table("node_requests").delete() + uid = queue_one() + done = dict(result_status=200, result_headers="{}", result_body="eA==", result_bytes=1, truncated=False) + assert store.finish_request(uid, **done) is False + store.claim_request(uid, "d") + assert store.finish_request(uid, **done) is True + assert store.finish_request(uid, **done) is False + assert store.fail_request(uid, "late") is False + assert store.get_request(uid)["status"] == "done" + get_table("node_requests").delete() + + +def test_expiry_separates_waiting_from_stuck_requests(local_db): + get_table("node_requests").delete() + waiting, stuck, fresh = queue_one(), queue_one(), queue_one() + store.claim_request(stuck, "d") + later = datetime.now(timezone.utc) + timedelta( + seconds=LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds + LIMITS.queue_max_wait_seconds + 1 + ) + counts = store.expire_requests(later) + assert counts == {"queued": 2, "assigned": 1} + assert store.get_request(waiting)["status"] == "expired" + assert store.get_request(stuck)["error"] == "timeout" + assert store.get_request(fresh)["status"] == "expired" + assert store.expire_requests(later) == {"queued": 0, "assigned": 0} + get_table("node_requests").delete() + + +def test_requests_are_listed_oldest_first_and_limited(local_db): + get_table("node_requests").delete() + uids = [queue_one() for _ in range(5)] + assert [row["uid"] for row in store.list_queued(3)] == uids[:3] + get_table("node_requests").delete() + + +def test_request_garbage_collection_leaves_recent_rows(local_db): + get_table("node_requests").delete() + queue_one() + assert store.prune_requests() == 0 + far = datetime.now(timezone.utc) + timedelta(seconds=LIMITS.request_retention_seconds + 5) + assert store.prune_requests(dry_run=True, moment=far) == 1 + assert store.prune_requests(moment=far) == 1 + + +def test_caller_quota_counts_exactly_to_the_limit(local_db): + caller = generate_uid() + results = [store.reserve_caller_request("user", caller, 5) for _ in range(9)] + assert results.count(True) == 5 + assert store.caller_requests_today("user", caller) == 5 + + +def test_a_zero_quota_or_anonymous_caller_never_reserves(local_db): + assert not store.reserve_caller_request("guest", "someone", 0) + assert not store.reserve_caller_request("user", "", 10) + assert store.caller_requests_today("guest", "someone") == 0 + + +def test_caller_quota_is_per_caller_and_per_day(local_db): + a, b = generate_uid(), generate_uid() + for _ in range(3): + assert store.reserve_caller_request("user", a, 3) + assert not store.reserve_caller_request("user", a, 3) + assert store.reserve_caller_request("user", b, 3) + tomorrow = datetime.now(timezone.utc) + timedelta(days=1) + assert store.reserve_caller_request("user", a, 3, moment=tomorrow) + assert store.reserve_caller_request("guest", a, 3) + + +def test_caller_counters_are_pruned_with_the_usage_buckets(local_db): + caller = generate_uid() + store.reserve_caller_request("user", caller, 5) + far = datetime.now(timezone.utc) + timedelta(days=LIMITS.usage_daily_retention_days + 2) + assert store.prune_usage_daily(dry_run=True, moment=far) >= 1 + assert store.caller_requests_today("user", caller) == 1 + store.prune_usage_daily(moment=far) + assert store.caller_requests_today("user", caller) == 0 + + +def test_batch_helpers_match_the_single_lookups(local_db): + owner = generate_uid() + row, _ = store.create_device(owner, "a", "native") + store.save_node_settings(owner, {"enabled": True, "bandwidth_mb_day": 7}) + assert store.get_devices_by_uids([row["uid"], "missing", ""])[row["uid"]]["user_uid"] == owner + assert store.get_enabled_by_users([owner, "nobody"]) == {owner: True} + assert store.get_settings_by_users([owner])[owner]["bandwidth_mb_day"] == 7 + assert store.get_devices_by_uids([]) == {} + + +def test_batch_helpers_ignore_revoked_devices(local_db): + owner = generate_uid() + row, _ = store.create_device(owner, "a", "native") + store.revoke_device(row["uid"], owner) + assert store.get_devices_by_uids([row["uid"]]) == {} + + +def test_device_breakdown_buckets_blank_values_as_unknown(local_db): + owner = generate_uid() + store.create_device(owner, "a", "native") + store.create_device(owner, "b", "extension") + kinds = store.device_breakdown("kind") + assert kinds["native"] >= 1 and kinds["extension"] >= 1 + assert "unknown" in store.device_breakdown("country") + with pytest.raises(ValueError): + store.device_breakdown("secret_hash") + + +def test_device_breakdown_ignores_revoked_devices(local_db): + owner = generate_uid() + row, _ = store.create_device(owner, "a", "native") + before = store.device_breakdown("status").get("active", 0) + store.revoke_device(row["uid"], owner) + assert store.device_breakdown("status").get("active", 0) == before - 1 + + +def test_request_status_counts_always_lists_every_status(local_db): + counts = store.request_status_counts() + assert set(store.REQUEST_STATUSES) <= set(counts) + assert all(value >= 0 for value in counts.values()) + + +def test_usage_day_totals_sums_the_day(local_db): + owner = generate_uid() + row, _ = store.create_device(owner, "a", "native") + before = store.usage_day_totals() + store.record_job(row["uid"], owner, ok=True, bytes_up=10, bytes_down=90) + after = store.usage_day_totals() + assert after["requests_ok"] == before["requests_ok"] + 1 + assert after["bytes_down"] == before["bytes_down"] + 90 + + +def test_recent_devices_is_capped_and_excludes_deleted(local_db): + owner = generate_uid() + row, _ = store.create_device(owner, "a", "native") + assert row["uid"] in {item["uid"] for item in store.recent_devices(store.ADMIN_LIST_LIMIT)} + assert len(store.recent_devices(1)) == 1 + assert len(store.recent_devices(10_000)) <= store.ADMIN_LIST_LIMIT + store.revoke_device(row["uid"], owner) + assert row["uid"] not in {item["uid"] for item in store.recent_devices(store.ADMIN_LIST_LIMIT)} diff --git a/tests/unit/services/community/__init__.py b/tests/unit/services/community/__init__.py new file mode 100644 index 00000000..95ee7c3e --- /dev/null +++ b/tests/unit/services/community/__init__.py @@ -0,0 +1 @@ +# retoor diff --git a/tests/unit/services/community/ai.py b/tests/unit/services/community/ai.py new file mode 100644 index 00000000..bfd93d8d --- /dev/null +++ b/tests/unit/services/community/ai.py @@ -0,0 +1,67 @@ +# retoor + +from tests.conftest import run_async + +from devplacepy.services.community.ai import AiAssistant, outline, parse_json, to_html +from devplacepy.services.community.extract import parse_html, select_nodes +from tests.unit.services.community.fakes import HTML_ITEM + + +class Canned(AiAssistant): + def __init__(self, answer): + self.answer = answer + self.asked = [] + + def enabled(self): + return True + + async def ask(self, system, user): + self.asked.append((system, user)) + return self.answer + + +def nodes(): + root = parse_html("
" + "".join(HTML_ITEM.format(n=n) for n in range(1, 5)) + "
") + return root, select_nodes(root, "article.post") + + +def test_json_is_extracted_from_fenced_or_chatty_replies(): + assert parse_json('```json\n{"a": 1}\n```') == {"a": 1} + assert parse_json('Sure! Here you go: {"a": {"b": 2}} hope it helps') == {"a": {"b": 2}} + assert parse_json("no json") is None + + +def test_only_proposals_that_match_the_sample_are_accepted(): + _, items = nodes() + assistant = Canned({"title": "h2", "link": "h2 a::attr(href)", "author": ".nonexistent", "created_at": "time::attr(datetime)"}) + accepted = run_async(assistant.post_fields(items)) + assert accepted == {"title": "h2", "link": "h2 a::attr(href)", "created_at": "time::attr(datetime)"} + + +def test_item_selector_must_match_at_least_three_elements(): + root, _ = nodes() + assert run_async(Canned({"item_selector": "article.post"}).item_selector(root)) == "article.post" + assert run_async(Canned({"item_selector": "article.missing"}).item_selector(root)) is None + assert run_async(Canned({"item_selector": "!!"}).item_selector(root)) is None + + +def test_comment_structure_needs_content(): + root = parse_html('

hello there

ann

second text

bob
') + good = run_async(Canned({"item_selector": "div.c", "id": "::attr(id)", "content": "p", "author": "b"}).comment_structure(root)) + assert good == {"item_selector": "div.c", "id": "::attr(id)", "content": "p", "author": "b"} + assert run_async(Canned({"item_selector": "div.c", "content": "span"}).comment_structure(root)) is None + + +def test_disabled_assistant_never_asks(): + class Off(AiAssistant): + def enabled(self): + return False + + assert run_async(Off().ask("s", "u")) is None + + +def test_outline_and_html_are_bounded(): + root, items = nodes() + assert "article.post" in outline(root) + assert len(outline(root, 3).splitlines()) <= 3 + assert len(to_html(items[0], 100)) <= 100 diff --git a/tests/unit/services/community/alias.py b/tests/unit/services/community/alias.py new file mode 100644 index 00000000..b36b96a1 --- /dev/null +++ b/tests/unit/services/community/alias.py @@ -0,0 +1,133 @@ +# retoor + +import pytest + +from devplacepy.database import get_setting, get_table, init_db, set_setting +from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner +from devplacepy.services.community.store import store +from devplacepy.services.sync import ai +from devplacepy.services.sync.options import ALIAS_KEY, SyncOptions +from tests.conftest import run_async +from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep + +ORIGINALS = ("writer0", "writer1", "replier0", "replier1") + + +@pytest.fixture +def world(monkeypatch): + init_db() + keys = ("sync_enabled", "sync_origins", ALIAS_KEY) + previous = {key: get_setting(key, "") for key in keys} + set_setting("sync_enabled", "1") + set_setting("sync_origins", "") + set_setting(ALIAS_KEY, "1") + for table in ("communities", "community_runs"): + get_table(table).delete() + get_table("sync_links").delete(origin="forum.test") + get_table("sync_identities").delete(origin="forum.test") + + async def unreachable(system, text, timeout): + return None + + monkeypatch.setattr(ai, "complete", unreachable) + runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(json_community()), sleep=no_sleep) + store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum") + run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test")) + yield runner + for key, value in previous.items(): + set_setting(key, value) + get_table("sync_identities").delete(origin="forum.test") + + +def go(runner, dry_run=False): + return run_async(runner.run(store.get("forum-test"), dry_run=dry_run, trigger=TRIGGER_CLI)) + + +def synced(kind: str) -> list[dict]: + links = get_table("sync_links").find(origin="forum.test", kind=kind) + return [row for row in (get_table(kind).find_one(uid=link["local_uid"]) for link in links) if row] + + +def local_text() -> str: + names = [row["username"] for row in synced("users")] + posts = [f"{row['title']} {row['content']}" for row in synced("posts")] + comments = [row["content"] for row in synced("comments")] + return " ".join(names + posts + comments).lower() + + +def enable(**changes): + row = store.get("forum-test") + store.update(row["uid"], sync_options=SyncOptions.from_row(row).merged(**changes).to_dict()) + + +def test_aliasing_off_by_default_keeps_original_names(world): + report = go(world) + assert report.status == "ok", report.errors + assert "writer0" in local_text() + assert get_table("sync_identities").count(origin="forum.test") == 0 + + +def test_alias_run_never_stores_an_original_username(world): + enable(alias=True) + report = go(world) + assert report.status == "ok", report.errors + text = local_text() + assert not [name for name in ORIGINALS if name in text] + assert get_table("sync_identities").count(origin="forum.test") == 4 + again = go(world) + assert again.status == "ok" + assert again.pushed["users"].get("created", 0) == 0 + assert get_table("sync_identities").count(origin="forum.test") == 4 + + +def test_kill_switch_stops_aliased_runs_and_creates_nothing(world): + enable(alias=True) + set_setting(ALIAS_KEY, "0") + report = go(world) + assert any("switched off" in error for error in report.errors) + assert not synced("posts") and not synced("users") + assert get_table("sync_identities").count(origin="forum.test") == 0 + + +def test_dry_run_creates_no_identities_and_no_rows(world): + enable(alias=True) + users = get_table("users").count() + report = go(world, dry_run=True) + assert report.status == "dry_run", report.errors + assert get_table("sync_identities").count(origin="forum.test") == 0 + assert get_table("users").count() == users + + +def test_enabling_alias_after_unaliased_sync_is_refused_at_run(world): + assert go(world).status == "ok" + enable(alias=True) + report = go(world) + assert any("already has synced records" in error for error in report.errors) + assert get_table("sync_identities").count(origin="forum.test") == 0 + + +def test_selection_skips_are_reported_and_sampling_is_stable(world): + enable(sample_percent=30) + first = go(world) + count = len(synced("posts")) + assert first.pushed["skipped"]["sampled_out"] + count >= 10 + assert 0 < count < 10 + go(world) + assert len(synced("posts")) == count + + +def test_translation_failure_holds_watermark_and_stores_nothing(world): + enable(alias=True, translate_to="nl") + row = store.get("forum-test") + store.update(row["uid"], language="en") + report = go(world) + assert not synced("posts") + assert report.status == "partial" + assert report.pushed["skipped"]["translation_failed"] >= 1 + assert not store.get("forum-test")["watermark"] + + +def test_jitter_moves_next_run_within_bounds(world): + row = store.get("forum-test") + moments = {store.schedule_next(row["uid"], 86400, 40) for _ in range(5)} + assert len(moments) > 1 diff --git a/tests/unit/services/community/conftest.py b/tests/unit/services/community/conftest.py new file mode 100644 index 00000000..69397f9b --- /dev/null +++ b/tests/unit/services/community/conftest.py @@ -0,0 +1,13 @@ +# retoor + +import pytest + +from devplacepy.services.community.ai import AiAssistant + + +@pytest.fixture(autouse=True) +def silent_ai(monkeypatch): + async def never(self, system, user): + return None + + monkeypatch.setattr(AiAssistant, "ask", never) diff --git a/tests/unit/services/community/discoverer.py b/tests/unit/services/community/discoverer.py new file mode 100644 index 00000000..9d443ec7 --- /dev/null +++ b/tests/unit/services/community/discoverer.py @@ -0,0 +1,133 @@ +# retoor + +from tests.conftest import run_async + +from devplacepy.services.community.discoverer import CommunityDiscoverer +from devplacepy.services.community.spec import STATUS_FAILED, STATUS_LOGIN, STATUS_READY +from tests.unit.services.community.fakes import ( + FakeFetcher, + club_community, + html_community, + json_community, + rss_community, + spa_community, +) + + +def discover(routes: dict, query: str, credentials: dict | None = None): + fetcher = FakeFetcher(routes) + return run_async(CommunityDiscoverer(fetcher, credentials=credentials).discover(query)), fetcher + + +def test_json_api_community_is_learned(): + result, _ = discover(json_community(), "https://forum.test/api/posts") + assert result.status == STATUS_READY, result.error + feed = result.schema["feeds"][0] + assert feed["type"] == "json_api" + assert feed["list_path"] == "$.items" + assert feed["post"]["fields"]["id"] == ["$.id"] + assert feed["post"]["fields"]["content"][0] == "$.content_html" + assert feed["post"]["fields"]["created_at"] == ["$.created_at"] + assert feed["author"]["fields"]["username"] == ["$.author.username"] + assert feed["author"]["fields"]["id"] == ["$.author.id"] + assert feed["pagination"] == {"style": "page", "param": "page", "start": 1} + assert feed["comments"]["mode"] == "inline" + assert feed["comments"]["fields"]["content"] == ["$.body"] + assert feed["comments"]["author"]["fields"]["username"] == ["$.user.username"] + records = result.sample["records"] + assert len(records["posts"]) == 8 + assert len(records["comments"]) == 6 + assert {user["username"] for user in records["users"]} >= {"writer0", "writer1", "replier0"} + + +def test_rss_community_is_learned_from_advertised_feed(): + result, _ = discover(rss_community(), "https://blog.test/") + assert result.status == STATUS_READY, result.error + feed = result.schema["feeds"][0] + assert feed["type"] == "rss" + assert feed["post"]["fields"]["created_at"] == ["$.pubDate"] + assert result.sample["records"]["users"][0]["username"] == "Alice" + assert result.sample["records"]["posts"][0]["content"].startswith("Full body of entry") + assert feed["comments"]["mode"] == "page" + assert feed["comments"]["item_selector"] == "div.comment" + assert feed["post"]["detail"]["content"] == ["article"] + comments = result.sample["records"]["comments"] + assert len(comments) == 9 + assert sum(1 for comment in comments if comment.get("parent_uid")) == 3 + + +def test_html_list_community_is_learned(): + result, _ = discover(html_community(), "https://board.test/") + assert result.status == STATUS_READY, result.error + feed = result.schema["feeds"][0] + assert feed["type"] == "html_list" + assert feed["item_selector"] == "article.post" + assert len(result.sample["records"]["posts"]) == 6 + assert result.sample["records"]["posts"][0]["title"].startswith("Thread 1") + assert feed["comments"]["mode"] == "page" + assert feed["comments"]["fields"]["id"] == ["::attr(id)"] + assert feed["comments"]["author"]["fields"]["username"] == [".author"] + assert len(result.sample["records"]["comments"]) == 9 + + +def test_unreachable_community_fails_with_reason(): + result, _ = discover({}, "https://nothing.test/") + assert result.status == STATUS_FAILED + assert result.error + + +def test_javascript_only_site_is_rendered(): + result, _ = discover(spa_community(), "https://spa.test/") + assert result.status == STATUS_READY, result.error + feed = result.schema["feeds"][0] + assert feed["render"] is True + assert len(result.sample["records"]["posts"]) == 6 + + +def test_login_wall_without_credentials_is_flagged_not_failed(): + state = {} + result, _ = discover(club_community(state), "https://club.test/") + assert result.status == STATUS_LOGIN + assert result.schema["login"]["mode"] == "form" + assert result.schema["login"]["username_field"] == "user" + assert result.schema["login_required"] is True + + +def test_login_wall_with_credentials_is_learned_behind_the_wall(): + state = {} + result, fetcher = discover(club_community(state), "https://club.test/", {"username": "bob", "password": "pw"}) + assert result.status == STATUS_READY, result.error + assert result.schema["login_required"] is True + assert result.schema["feeds"][0]["type"] == "rss" + assert fetcher.posts[0] == ("https://club.test/login", {"csrf": "t1", "user": "bob", "pass": "pw"}) + + +def test_wrong_credentials_fail_loudly(): + result, _ = discover(club_community({}), "https://club.test/", {"username": "bob", "password": "nope"}) + assert result.status == STATUS_FAILED + assert "rejected the stored credentials" in result.error + + +def test_feeds_listing_the_same_posts_are_learned_once_and_comment_feeds_are_skipped(): + routes = rss_community() + routes["https://blog.test/"] = ( + '' + '' + 'Latest posts' + ) + routes["https://blog.test/comments.xml"] = routes["https://blog.test/feed.xml"] + items = "".join( + f'" + for n in range(1, 7) + ) + routes["https://blog.test/latest"] = f"
{items}
" + result, fetcher = discover(routes, "https://blog.test/") + assert result.status == STATUS_READY, result.error + assert [feed["type"] for feed in result.schema["feeds"]] == ["rss"] + assert not any("comments.xml" in request for request in fetcher.requests) + + +def test_generic_probe_paths_are_skipped_when_a_feed_is_advertised(): + _, fetcher = discover(rss_community(), "https://blog.test/") + assert not any(request.endswith("/api/posts") or request.endswith("/rss.xml") for request in fetcher.requests) diff --git a/tests/unit/services/community/extract.py b/tests/unit/services/community/extract.py new file mode 100644 index 00000000..a526ec6c --- /dev/null +++ b/tests/unit/services/community/extract.py @@ -0,0 +1,60 @@ +# retoor + +import pytest + +from devplacepy.services.community import extract + + +def test_select_walks_keys_indexes_and_wildcards(): + data = {"a": [{"b": 1}, {"b": 2}], "c": {"d": "x"}} + assert extract.select(data, "$.a[*].b") == [1, 2] + assert extract.select(data, "$.a[1].b") == [2] + assert extract.select(data, "$.c.d") == ["x"] + assert extract.select(data, "$.missing") == [] + + +@pytest.mark.parametrize("path", ["a.b", "$.a[", "$..", "$" + ".x" * 40]) +def test_bad_paths_raise(path): + with pytest.raises(ValueError): + extract.parse_path(path) + + +def test_instants_accept_iso_rfc822_and_epochs(): + assert extract.normalize_instant("2026-01-01T00:00:00Z") == "2026-01-01T00:00:00+00:00" + assert extract.normalize_instant("Tue, 14 Oct 2026 10:00:00 GMT") == "2026-10-14T10:00:00+00:00" + assert extract.normalize_instant(1700000000) == extract.normalize_instant(1700000000000) + assert extract.normalize_instant("garbage") is None + assert extract.normalize_instant("") is None + + +def test_atom_feed_is_parsed_with_links_and_authors(): + atom = ( + 'TE' + 'Ann' + "2026-01-02T00:00:00Z" + ) + kind, _, items = extract.parse_feed(atom) + assert kind == "atom" + assert extract.text_at(items[0], "$.link.@href") == "http://x/1" + assert extract.text_at(items[0], "$.author.name") == "Ann" + + +def test_css_selectors_support_classes_ids_attributes_and_combinators(): + root = extract.parse_html( + '
' + ) + assert len(extract.select_nodes(root, "li.a")) == 2 + assert len(extract.select_nodes(root, "li.a.b")) == 1 + assert len(extract.select_nodes(root, "#m > ul > li")) == 2 + assert len(extract.select_nodes(root, "div li")) == 2 + assert extract.html_field(extract.select_nodes(root, "li.b")[0], "a[rel=next]::attr(href)") == "/x" + + +def test_strip_tags_keeps_paragraph_breaks_and_drops_scripts(): + assert extract.strip_tags("

a

b
c

") == "a\n\nb\n\nc" + assert extract.strip_tags("plain text") == "plain text" + + +def test_field_text_falls_through_alternates_and_now_token(): + assert extract.field_text({"a": "", "b": "yes"}, ["$.a", "$.b"]) == "yes" + assert extract.field_text({}, [extract.NOW_TOKEN]).startswith("20") diff --git a/tests/unit/services/community/fakes.py b/tests/unit/services/community/fakes.py new file mode 100644 index 00000000..7c6b16fa --- /dev/null +++ b/tests/unit/services/community/fakes.py @@ -0,0 +1,228 @@ +# retoor + +import json +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + + +class FakeResponse: + def __init__(self, status: int, body): + self.status = status + self.body = body if isinstance(body, str) else json.dumps(body) + self.headers = {} + + async def text(self): + return self.body + + async def json(self): + return json.loads(self.body) + + +class FakeFetcher: + def __init__(self, routes: dict): + self.routes = routes + self.requests: list[str] = [] + self.session = None + self.posts: list[tuple] = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return None + + @staticmethod + def key(url: str, params: dict | None) -> str: + parts = urlsplit(url) + query = dict(parse_qsl(parts.query)) + query.update({name: str(value) for name, value in (params or {}).items()}) + return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(sorted(query.items())), "")) + + async def get(self, url, params=None, timeout=None, **kwargs): + key = self.key(url, params) + self.requests.append(key) + route = self.routes.get(key) + if route is None: + return FakeResponse(404, "not found") + if callable(route): + route = route() + status, body = route if isinstance(route, tuple) else (200, route) + return FakeResponse(status, body) + + async def post(self, url, data=None, timeout=None, **kwargs): + self.posts.append((url, dict(data or {}))) + route = self.routes.get("post:" + url) + if callable(route): + route = route(dict(data or {})) + status, body = route if isinstance(route, tuple) else (200, route or "") + return FakeResponse(status, body) + + async def render_html(self, url, session=None): + route = self.routes.get("render:" + url) + if route is None: + response = await self.get(url) + return response.status, response.body + return 200, route + + def page(self, session=None): + return FakePage(self) + + async def render(self, url, collect_links=False): + response = await self.get(url) + return {"status": response.status, "text": response.body, "links": [], "rendered": False} + + +def json_post(number: int) -> dict: + return { + "id": 1000 + number, + "title": f"Post number {number} about asyncio", + "content_html": f"

This is the body of post {number}, long enough to sync.

", + "created_at": f"2026-10-{number:02d}T10:00:00Z", + "author": {"id": 7 + number % 2, "username": f"writer{number % 2}", "url": f"https://forum.test/u/{number % 2}"}, + "url": f"https://forum.test/posts/{1000 + number}", + "comments": [ + { + "id": 5000 + number * 10 + index, + "body": f"Comment {index} on post {number}", + "created_at": f"2026-10-{number:02d}T11:0{index}:00Z", + "user": {"id": 20 + index, "username": f"replier{index}"}, + } + for index in range(2) + ], + } + + +def json_community() -> dict: + posts = [json_post(number) for number in range(1, 9)] + return { + "https://forum.test/": 'Forum Test' + '' + "

Join 1,234 members discussing code every single day on this community site.

", + "https://forum.test/api/posts": {"items": posts, "total": 8}, + "https://forum.test/api/posts?page=2": {"items": [json_post(9), json_post(10)], "total": 8}, + "https://forum.test/about": "

About: 2,500 members strong.

", + } + + +RSS = """ + +Blog TestA blog +{items} +""" + +RSS_ITEM = """Entry {n} on testinghttps://blog.test/p/{n} +https://blog.test/p/{n}Tue, {n:02d} Oct 2026 10:00:00 GMT +Alice<p>Body of entry {n} with enough text.</p>""" + + +def detail_page(kind: str, n: int, comments: int = 2) -> str: + rows = "".join( + f'
replier{index}' + f'' + f'
Comment {index} on {kind} {n} with words
' + + ( + f'
deep{index}' + f'
Nested reply to comment {index} on {n}
' + if index == 0 + else "" + ) + + "
" + for index in range(comments) + ) + body = f"Full body of {kind} {n}. " + "It goes on at length about the topic. " * 6 + return f"
{body}
{rows}
" + + +def rss_community() -> dict: + items = "".join(RSS_ITEM.format(n=n) for n in range(1, 7)) + pages = {f"https://blog.test/p/{n}": detail_page("entry", n) for n in range(1, 7)} + return { + **pages, + "https://blog.test/": '' + "Blog Testhi", + "https://blog.test/feed.xml": RSS.format(items=items), + } + + +HTML_ITEM = """""" + + +def html_community() -> dict: + items = "".join(HTML_ITEM.format(n=n) for n in range(1, 7)) + pages = {f"https://board.test/t/{n}": detail_page("thread", n) for n in range(1, 7)} + return { + **pages, + "https://board.test/": f"Board
{items}
", + } + + +async def no_sleep(_seconds): + return None + + +class FakePage: + def __init__(self, fetcher): + self.fetcher = fetcher + self.filled: dict[str, str] = {} + self.url = "" + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return None + + async def goto(self, url, **kwargs): + self.url = url + status, self.html = await self.fetcher.render_html(url) + return type("Resp", (), {"status": status})() + + async def wait_for_timeout(self, _ms): + return None + + async def content(self): + return self.html + + async def fill(self, selector, value): + self.filled[selector] = value + + async def click(self, selector): + self.fetcher.posts.append(("click", dict(self.filled))) + + +SPA_SHELL = "Spa
" + + +def spa_community() -> dict: + items = "".join(HTML_ITEM.format(n=n) for n in range(1, 7)) + return { + "https://spa.test/": SPA_SHELL, + "render:https://spa.test/": f"
{items}
", + } + + +WALL = ( + '
' + '
' +) +CLUB_HOME = ( + 'Club' + "

Members only area with plenty of text on the page for the club.

" +) + + +def club_community(state: dict) -> dict: + items = "".join(RSS_ITEM.format(n=n).replace("blog.test", "club.test") for n in range(1, 7)) + feed = RSS.format(items=items) + + def login(data): + state["in"] = data.get("user") == "bob" and data.get("pass") == "pw" and data.get("csrf") == "t1" + return (200, "welcome" if state["in"] else WALL) + + return { + "https://club.test/": lambda: CLUB_HOME if state.get("in") else WALL, + "https://club.test/login": WALL, + "https://club.test/feed.xml": lambda: feed if state.get("in") else (401, "login"), + "post:https://club.test/login": login, + } diff --git a/tests/unit/services/community/generator.py b/tests/unit/services/community/generator.py new file mode 100644 index 00000000..ee29e68e --- /dev/null +++ b/tests/unit/services/community/generator.py @@ -0,0 +1,168 @@ +# retoor + +from tests.conftest import run_async +import importlib.util +import json +import py_compile + +import pytest + +from devplacepy.services.community.discoverer import CommunityDiscoverer +from devplacepy.services.community.generator import CodeGenerator +from devplacepy.services.community.runtime import Collector, LoginRequired +from devplacepy.services.sync.records import RECORD_MODELS +from tests.unit.services.community.fakes import ( + FakeFetcher, + FakeResponse, + html_community, + json_community, + rss_community, +) + + +async def no_sleep(_seconds): + return None + + +def learn(routes: dict, query: str) -> dict: + result = run_async(CommunityDiscoverer(FakeFetcher(routes)).discover(query)) + assert result.schema, result.error + return result.schema + + +def load(path): + spec = importlib.util.spec_from_file_location("generated_collectors", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + "routes,query", + [ + (json_community(), "https://forum.test/api/posts"), + (rss_community(), "https://blog.test/"), + (html_community(), "https://board.test/"), + ], +) +def test_generated_file_compiles_and_collects_valid_sync_records(tmp_path, routes, query): + schema = learn(routes, query) + target = CodeGenerator(schema).write(tmp_path) + py_compile.compile(str(target), doraise=True) + assert json.loads((tmp_path / "schema.json").read_text()) == schema + module = load(target) + result = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep)) + assert result["ok"], result["errors"] + assert result["records"]["posts"] + for kind, rows in result["records"].items(): + for row in rows: + RECORD_MODELS[kind].model_validate(row) + uids = {user["origin_uid"] for user in result["records"]["users"]} + assert {post["author_uid"] for post in result["records"]["posts"]} <= uids + + +def test_generated_wrappers_exist_per_feed(tmp_path): + schema = learn(json_community(), "https://forum.test/api/posts") + module = load(CodeGenerator(schema).write(tmp_path)) + assert callable(module.collect_feed1) + assert module.SCHEMA == schema + + +def test_generated_output_matches_in_process_collector(tmp_path): + routes = json_community() + schema = learn(routes, "https://forum.test/api/posts") + module = load(CodeGenerator(schema).write(tmp_path)) + generated = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep)) + direct = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run()) + assert generated["records"] == direct["records"] + + +def test_incremental_run_collects_only_new_posts(tmp_path): + routes = json_community() + schema = learn(routes, "https://forum.test/api/posts") + first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run()) + again = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run()) + assert len(first["records"]["posts"]) == 10 + assert again["records"]["posts"] == [] + assert again["state"]["feeds"]["feed1"]["since"] == first["state"]["feeds"]["feed1"]["since"] + + +def test_incremental_run_picks_up_newer_post(): + routes = json_community() + schema = learn(routes, "https://forum.test/api/posts") + first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run()) + newer = { + "id": 2000, + "title": "A brand new post", + "content_html": "

Fresh content that arrived later.

", + "created_at": "2026-11-01T10:00:00Z", + "author": {"id": 7, "username": "writer1"}, + "comments": [], + } + routes["https://forum.test/api/posts"]["items"].insert(0, newer) + result = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run()) + assert [post["origin_uid"] for post in result["records"]["posts"]] == ["2000"] + + +def test_failed_requests_retry_with_backoff(): + calls = {"count": 0} + delays = [] + + class Flaky: + async def get(self, url, params=None, timeout=None, **kwargs): + calls["count"] += 1 + return FakeResponse(503 if calls["count"] < 3 else 200, "[]") + + async def record(seconds): + delays.append(seconds) + + collector = Collector(Flaky(), {"feeds": [], "limits": {"delay_seconds": 0, "backoff_seconds": 2, "retries": 3}}, sleep=record) + assert run_async(collector.fetch("https://x.test/")) == "[]" + assert calls["count"] == 3 + assert [delay for delay in delays if delay] == [2.0, 4.0] + + +def test_login_required_schema_raises_clear_error(): + schema = {"community": "closed", "origin": "closed.test", "login_required": True, "feeds": []} + with pytest.raises(LoginRequired, match="requires a login"): + run_async(Collector(FakeFetcher({}), schema).run()) + + +def test_unauthorized_response_is_login_required(): + schema = { + "community": "x", + "origin": "x.test", + "feeds": [{"id": "feed1", "url": "https://x.test/f", "type": "json_api", "post": {"fields": {"id": ["$.id"]}}, "author": {}}], + "limits": {"delay_seconds": 0, "retries": 1}, + } + with pytest.raises(LoginRequired): + run_async(Collector(FakeFetcher({"https://x.test/f": (401, "no")}), schema, sleep=no_sleep).run()) + + +def test_generated_collectors_log_in_with_credentials(tmp_path): + from tests.unit.services.community.fakes import club_community + + state = {} + routes = club_community(state) + result = run_async(CommunityDiscoverer(FakeFetcher(routes), credentials={"username": "bob", "password": "pw"}).discover("https://club.test/")) + module = load(CodeGenerator(result.schema).write(tmp_path)) + state.clear() + with pytest.raises(module.LoginRequired, match="requires a login"): + run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep)) + collected = run_async( + module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "pw"}) + ) + assert collected["ok"] and collected["records"]["posts"] + state.clear() + with pytest.raises(module.LoginRequired, match="rejected the stored credentials"): + run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "bad"})) + + +def test_javascript_feed_is_collected_through_the_browser(tmp_path): + from tests.unit.services.community.fakes import spa_community + + routes = spa_community() + schema = learn(routes, "https://spa.test/") + module = load(CodeGenerator(schema).write(tmp_path)) + collected = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep)) + assert len(collected["records"]["posts"]) == 6 diff --git a/tests/unit/services/community/search.py b/tests/unit/services/community/search.py new file mode 100644 index 00000000..b04bc0bf --- /dev/null +++ b/tests/unit/services/community/search.py @@ -0,0 +1,39 @@ +# retoor + +from tests.conftest import run_async + +import pytest + +from devplacepy.services.community.search import SearchPhase, direct_url, rank +from tests.unit.services.community.fakes import FakeFetcher + + +def test_direct_addresses_skip_search(): + assert direct_url("https://x.test/forum") == "https://x.test/forum" + assert direct_url("x.test") == "https://x.test" + assert direct_url("devplace") is None + + +def test_rank_prefers_domain_match_and_drops_social_noise(): + results = [ + {"title": "DevPlace on LinkedIn", "url": "https://www.linkedin.com/company/devplace", "description": ""}, + {"title": "Other", "url": "https://other.test/page", "description": "mentions devplace"}, + {"title": "DevPlace", "url": "https://devplace.example/", "description": ""}, + ] + ranked = rank("devplace", results) + assert [candidate.host for candidate in ranked] == ["devplace.example", "other.test"] + + +def test_search_uses_rsearch_and_caps_candidates(): + results = [{"title": f"t{n}", "url": f"https://site{n}.test/", "description": ""} for n in range(9)] + key = "https://rsearch.app.molodetz.nl/search?content=false&count=15&query=forum" + fetcher = FakeFetcher({key: {"success": True, "results": results}}) + found = run_async(SearchPhase(fetcher).run("forum")) + assert len(found) == 5 + + +def test_search_failure_raises(): + key = "https://rsearch.app.molodetz.nl/search?content=false&count=15&query=forum" + fetcher = FakeFetcher({key: (500, "boom")}) + with pytest.raises(RuntimeError): + run_async(SearchPhase(fetcher).run("forum")) diff --git a/tests/unit/services/community/service.py b/tests/unit/services/community/service.py new file mode 100644 index 00000000..5c8e6394 --- /dev/null +++ b/tests/unit/services/community/service.py @@ -0,0 +1,81 @@ +# retoor + +from tests.conftest import run_async + +import pytest + +from devplacepy.database import get_setting, get_table, init_db, set_setting +from devplacepy.services.community.runner import CommunityRunner +from devplacepy.services.community.service import DRY_RUN_KEY, CommunityService +from devplacepy.services.community.spec import STATUS_FAILING, STATUS_READY +from devplacepy.services.community.store import store +from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep + + +@pytest.fixture +def world(): + init_db() + previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins", DRY_RUN_KEY)} + set_setting("sync_enabled", "1") + set_setting("sync_origins", "") + get_table("communities").delete() + get_table("community_runs").delete() + get_table("sync_links").delete(origin="forum.test") + routes = json_community() + runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep) + yield routes, runner + for key, value in previous.items(): + set_setting(key, value) + + +def seed(runner) -> dict: + store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum") + run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test")) + return store.get("forum-test") + + +def test_dry_run_service_tick_learns_nothing_and_runs_nothing(world): + _, runner = world + store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum") + set_setting(DRY_RUN_KEY, "1") + service = CommunityService(runner) + run_async(service.run_once()) + assert store.get("forum-test")["status"] == "pending" + assert any("DRY RUN" in line for line in service.log_buffer) + + +def test_live_tick_learns_pending_then_runs_due(world): + _, runner = world + store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum") + set_setting(DRY_RUN_KEY, "0") + service = CommunityService(runner) + run_async(service.run_once()) + assert store.get("forum-test")["status"] == STATUS_READY + run_async(service.run_once()) + row = store.get("forum-test") + assert row["run_count"] == 1 + assert get_table("sync_links").count(origin="forum.test", kind="posts") == 10 + run_async(service.run_once()) + assert store.get("forum-test")["run_count"] == 1 + + +def test_repeated_failures_disable_and_flag_the_community(world): + routes, runner = world + row = seed(runner) + routes["https://forum.test/api/posts"] = (500, "down") + failing = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep) + for _ in range(6): + current = store.get("forum-test") + report = run_async(failing.run(current, dry_run=False, trigger="test")) + assert report.status == "failed" + final = store.get("forum-test") + assert final["status"] == STATUS_FAILING and not final["enabled"] + assert "relearn" in "".join(store.runs(row["uid"])[0]["log"]).lower() or final["failure_count"] >= 5 + + +def test_run_history_is_bounded(world): + _, runner = world + row = seed(runner) + for _ in range(35): + run_async(runner.run(store.get("forum-test"), dry_run=True, trigger="test")) + assert len(store.runs(row["uid"], limit=100)) == 30 diff --git a/tests/unit/services/devii/actions/catalog.py b/tests/unit/services/devii/actions/catalog.py index c70e14c1..d532cf55 100644 --- a/tests/unit/services/devii/actions/catalog.py +++ b/tests/unit/services/devii/actions/catalog.py @@ -147,3 +147,17 @@ def test_scheduling_tools_are_hidden_from_guests(): } for name in ("create_task", "update_task", "run_task_now"): assert name not in guest + + +def test_sync_tools_are_admin_only_and_match_the_routes(): + from devplacepy.main import app + from devplacepy.services.sync.spec import KINDS + + paths = set(app.openapi()["paths"]) + for kind in KINDS: + action = BY_NAME[f"sync_{kind}"] + assert action.method == "POST" and action.path == f"/sync/{kind}" + assert action.requires_admin is True + assert action.path in paths + assert {param.name for param in action.params} == {"origin", "records", "dry_run"} + assert _records_mutation(action.name) is True diff --git a/tests/unit/services/nodes/adapters.py b/tests/unit/services/nodes/adapters.py new file mode 100644 index 00000000..5418950a --- /dev/null +++ b/tests/unit/services/nodes/adapters.py @@ -0,0 +1,91 @@ +# retoor + +import json + +import pytest + +from devplacepy.services.nodes.adapters.base import AdapterError, clean, tag_for +from devplacepy.services.nodes.adapters.direct import DevTo, GitHub, HackerNews, Mastodon, StackExchange +from devplacepy.services.nodes.adapters.reddit import Reddit +from devplacepy.services.nodes.adapters.registry import ADAPTERS +from devplacepy.services.nodes.spec import SPEC + + +def body(value) -> bytes: + return json.dumps(value).encode() + + +def test_clean_strips_tags_entities_and_caps(): + assert clean("

a & b

c", 50) == "a & b c" + assert len(clean("x" * 900, 100)) == 100 + assert clean(None, 10) == "" + + +def test_tag_for_picks_longest_word(): + assert tag_for("sqlite wal mode") == "sqlite" + assert tag_for("!!") == "" + + +def test_hackernews_parse(): + data = { + "hits": [ + {"objectID": "7", "title": "T", "points": 5, "num_comments": 2, "author": "a", "created_at": "2026"}, + {"objectID": "", "title": "no id"}, + "junk", + ] + } + hits = HackerNews().parse(body(data), 8) + assert [item.url for item in hits] == ["https://news.ycombinator.com/item?id=7"] + assert hits[0].score == 5 and hits[0].via == "direct" + + +def test_stackexchange_parse(): + data = {"items": [{"title": "Q & A", "link": "https://stackoverflow.com/q/1", "score": 3, "answer_count": 2, "tags": ["c"], "owner": {"display_name": "bob"}}]} + hit = StackExchange().parse(body(data), 8)[0] + assert hit.title == "Q & A" and hit.author == "bob" and hit.comments == 2 + + +def test_github_parse_requires_http_url(): + data = {"items": [{"title": "I", "html_url": "javascript:alert(1)"}, {"title": "J", "html_url": "https://github.com/a/b/issues/1", "user": {"login": "u"}}]} + hits = GitHub().parse(body(data), 8) + assert [item.title for item in hits] == ["J"] + + +def test_devto_and_mastodon_need_a_tag(): + with pytest.raises(AdapterError) as error: + DevTo().request("!!", 5) + assert error.value.code == "no_tag" + with pytest.raises(AdapterError): + Mastodon().request("", 5) + assert "tag=sqlite" in DevTo().request("sqlite", 5) + assert "/tag/sqlite" in Mastodon().request("sqlite", 5) + + +def test_mastodon_parse_strips_html(): + data = [{"url": "https://m.example/1", "content": "

hello world

", "account": {"acct": "x@y"}, "favourites_count": 2, "reblogs_count": 1}] + hit = Mastodon().parse(body(data), 5)[0] + assert hit.snippet == "hello world" and hit.score == 3 + + +def test_reddit_parse_and_via(): + data = {"data": {"children": [{"data": {"title": "T", "permalink": "/r/x/comments/1/t/", "score": 9, "num_comments": 4, "author": "a", "selftext": "body"}}, {"data": {"title": "bad", "permalink": "http://evil"}}]}} + hits = Reddit().parse(body(data), 5) + assert len(hits) == 1 and hits[0].url == "https://www.reddit.com/r/x/comments/1/t/" and hits[0].via == "fetch" + + +@pytest.mark.parametrize("adapter", list(ADAPTERS.values())) +def test_forged_structure_is_rejected(adapter): + for forged in (b"not json", body("string"), body({"unexpected": 1}), body(42)): + with pytest.raises(AdapterError): + adapter.parse(forged, 5) + + +def test_oversize_body_is_rejected(): + with pytest.raises(AdapterError) as error: + HackerNews().parse(b" " * (2 * 1024 * 1024 + 1), 5) + assert error.value.code == "body_too_large" + + +def test_node_adapters_are_exactly_the_spec_adapters(): + node_backed = {name for name, adapter in ADAPTERS.items() if not adapter.direct} + assert node_backed == {item.adapter for item in SPEC.destinations} diff --git a/tests/unit/services/nodes/cache.py b/tests/unit/services/nodes/cache.py new file mode 100644 index 00000000..4f202162 --- /dev/null +++ b/tests/unit/services/nodes/cache.py @@ -0,0 +1,109 @@ +# retoor + +import random + +from devplacepy.services.nodes.cache import ResultCache, cache_key +from devplacepy.services.nodes.spec import SPEC + +LIMITS = SPEC.limits + + +class Clock: + def __init__(self): + self.now = 100.0 + + def __call__(self): + return self.now + + +def make(**kwargs): + clock = Clock() + return ResultCache(clock=clock, **kwargs), clock + + +def test_hit_and_miss_counters(): + cache, _ = make() + assert cache.get("GET", "u") is None + assert cache.put("GET", "u", 200, {"a": "b"}, b"xyz", False) + hit = cache.get("GET", "u") + assert hit.body == b"xyz" and hit.headers == {"a": "b"} + assert (cache.hits, cache.misses) == (1, 1) + + +def test_method_is_part_of_the_key(): + assert cache_key("GET", "u") != cache_key("HEAD", "u") + assert cache_key("get", "u") == cache_key("GET", "u") + + +def test_entries_expire_after_the_ttl(): + cache, clock = make(ttl_seconds=10) + cache.put("GET", "u", 200, {}, b"x", False) + clock.now += 9.9 + assert cache.get("GET", "u") is not None + clock.now += 0.2 + assert cache.get("GET", "u") is None + assert len(cache) == 0 and cache.size_bytes == 0 + + +def test_ttl_is_clamped_to_the_spec_ceiling_and_zero_disables(): + assert ResultCache(ttl_seconds=10**9).ttl_seconds == LIMITS.cache_ttl_seconds + off, _ = make(ttl_seconds=0) + assert off.put("GET", "u", 200, {}, b"x", False) is False + + +def test_errors_and_truncated_bodies_are_never_cached(): + cache, _ = make() + assert not cache.put("GET", "u", 500, {}, b"x", False) + assert not cache.put("GET", "u", 503, {}, b"x", False) + assert not cache.put("GET", "u", 200, {}, b"x", True) + assert cache.put("GET", "u", 404, {}, b"x", False) + + +def test_an_oversized_entry_is_refused(): + cache, _ = make(entry_max_bytes=10, max_bytes=100) + assert not cache.put("GET", "u", 200, {}, b"x" * 11, False) + assert cache.put("GET", "u", 200, {}, b"x" * 10, False) + + +def test_the_entry_cap_cannot_exceed_the_spec_ceiling(): + assert ResultCache(entry_max_bytes=10**12).entry_max_bytes == LIMITS.cache_entry_max_bytes + + +def test_least_recently_used_goes_first(): + cache, _ = make(max_bytes=30, entry_max_bytes=30) + for name in "abc": + cache.put("GET", name, 200, {}, b"x" * 10, False) + cache.get("GET", "a") + cache.put("GET", "d", 200, {}, b"x" * 10, False) + assert cache.get("GET", "b") is None + assert cache.get("GET", "a") is not None + assert cache.get("GET", "c") is not None + assert cache.get("GET", "d") is not None + + +def test_replacing_an_entry_does_not_leak_bytes(): + cache, _ = make() + cache.put("GET", "u", 200, {}, b"x" * 100, False) + cache.put("GET", "u", 200, {}, b"y" * 40, False) + assert cache.size_bytes == 40 and len(cache) == 1 + + +def test_size_never_exceeds_the_cap_under_a_random_workload(): + cache, clock = make(max_bytes=1000, entry_max_bytes=400, ttl_seconds=50) + rng = random.Random(7) + for step in range(5000): + clock.now += rng.random() + key = f"k{rng.randint(0, 60)}" + if rng.random() < 0.7: + cache.put("GET", key, rng.choice([200, 404, 500]), {}, b"x" * rng.randint(0, 500), False) + else: + cache.get("GET", key) + assert 0 <= cache.size_bytes <= 1000 + assert cache.size_bytes == sum(len(e.body) for e in cache._entries.values()) + + +def test_clear_empties_everything(): + cache, _ = make() + cache.put("GET", "u", 200, {}, b"x", False) + cache.clear() + assert len(cache) == 0 and cache.size_bytes == 0 diff --git a/tests/unit/services/nodes/circuit.py b/tests/unit/services/nodes/circuit.py new file mode 100644 index 00000000..106c2110 --- /dev/null +++ b/tests/unit/services/nodes/circuit.py @@ -0,0 +1,97 @@ +# retoor + +from devplacepy.services.nodes.circuit import CLOSED, HALF_OPEN, OPEN, CircuitBreaker + + +class Clock: + def __init__(self): + self.now = 0.0 + + def __call__(self): + return self.now + + +def make(failures=3, open_seconds=60): + clock = Clock() + return CircuitBreaker(failures, open_seconds, clock), clock + + +def test_unknown_keys_are_closed(): + breaker, _ = make() + assert breaker.state("x") == CLOSED and breaker.allows("x") + + +def test_it_opens_at_the_threshold_and_not_before(): + breaker, _ = make(failures=3) + breaker.failure("x") + breaker.failure("x") + assert breaker.allows("x") + breaker.failure("x") + assert breaker.state("x") == OPEN and not breaker.allows("x") + + +def test_a_success_resets_the_streak(): + breaker, _ = make(failures=3) + breaker.failure("x") + breaker.failure("x") + breaker.success("x") + breaker.failure("x") + breaker.failure("x") + assert breaker.allows("x") + + +def test_it_goes_half_open_after_the_cooldown_and_admits_one_probe(): + breaker, clock = make(failures=1, open_seconds=60) + breaker.failure("x") + clock.now = 59 + assert breaker.state("x") == OPEN + clock.now = 60 + assert breaker.state("x") == HALF_OPEN and breaker.allows("x") + breaker.begin("x") + assert not breaker.allows("x") + + +def test_a_failed_probe_reopens_for_a_full_cooldown(): + breaker, clock = make(failures=1, open_seconds=60) + breaker.failure("x") + clock.now = 70 + breaker.begin("x") + breaker.failure("x") + assert breaker.state("x") == OPEN + clock.now = 129 + assert breaker.state("x") == OPEN + clock.now = 131 + assert breaker.state("x") == HALF_OPEN + + +def test_a_successful_probe_closes_it(): + breaker, clock = make(failures=1) + breaker.failure("x") + clock.now = 100 + breaker.begin("x") + breaker.success("x") + assert breaker.state("x") == CLOSED and breaker.allows("x") + + +def test_keys_are_independent(): + breaker, _ = make(failures=1) + breaker.failure(("node", "a")) + assert not breaker.allows(("node", "a")) + assert breaker.allows(("node", "b")) + assert breaker.allows(("destination", "a")) + + +def test_open_count_and_forget(): + breaker, _ = make(failures=1) + breaker.failure("a") + breaker.failure("b") + assert breaker.open_count() == 2 + breaker.forget("a") + assert breaker.open_count() == 1 + + +def test_a_zero_threshold_still_needs_one_failure(): + breaker = CircuitBreaker(failures=0) + assert breaker.allows("x") + breaker.failure("x") + assert not breaker.allows("x") diff --git a/tests/unit/services/nodes/dispatcher.py b/tests/unit/services/nodes/dispatcher.py new file mode 100644 index 00000000..eaec12b3 --- /dev/null +++ b/tests/unit/services/nodes/dispatcher.py @@ -0,0 +1,415 @@ +# retoor + +import base64 +import json +import random +import time + +import pytest + +from devplacepy.database import get_table +from devplacepy.database import nodes as store +from devplacepy.services.nodes.broker import BodyError, JobBroker, clean_headers, decode_body +from devplacepy.services.nodes.circuit import CircuitBreaker +from devplacepy.services.nodes.dispatcher import JOB_HEADERS, NodeDispatcher +from devplacepy.services.nodes.ledger import UsageLedger +from devplacepy.services.nodes.policy import DestinationPolicy +from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry +from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler +from devplacepy.services.nodes.session import NodeSession +from devplacepy.services.nodes.spec import SPEC +from devplacepy.utils import generate_uid +from tests.conftest import run_async +from tests.unit.services.nodes.scheduler import Clock, node, settings + +LIMITS = SPEC.limits +TARGET_URL = "https://www.reddit.com/r/python.json" + + +class Rig: + def __init__(self): + self.clock = Clock() + self.counters = NodeCounters() + self.registry = NodeRegistry(self.counters) + self.breaker = CircuitBreaker(clock=self.clock) + self.scheduler = NodeScheduler(self.registry, self.breaker, self.clock, rng=random.Random(2)) + self.broker = JobBroker() + self.dispatcher = NodeDispatcher( + self.registry, self.scheduler, self.broker, self.breaker, UsageLedger(), self.counters, + clock=self.clock, + ) + + def connect(self, **overrides): + owner = generate_uid() + device, _ = store.create_device(owner, "d", "native") + session = NodeSession( + node().websocket, {"uid": device["uid"], "user_uid": owner, "kind": "native"} + ) + session.hello = {"modes": ["fetch"]} + session.settings = settings(**overrides) + session.roll_day() + self.dispatcher.attach(session) + self.registry.admit(session) + return session + + def enqueue(self, url=TARGET_URL, caller="c1", kind="user", mode="any"): + target = DestinationPolicy().check(url, "GET", mode) + return self.broker.enqueue(kind, caller, target, mode) + + def tick(self): + return run_async(self.dispatcher.tick()) + + def answer(self, session, uid, **fields): + frame = {"type": "job_result", "job_id": uid, "status": 200, "body_b64": base64.b64encode(b"hello").decode()} + frame.update(fields) + run_async(session._handle(frame)) + + +@pytest.fixture +def rig(local_db): + get_table("node_requests").delete() + yield Rig() + get_table("node_requests").delete() + + +def sent_jobs(session): + return [frame for frame in session.websocket.sent if frame["type"] == "job"] + + +def test_a_queued_request_becomes_a_job_frame(rig): + session = rig.connect() + uid = rig.enqueue() + assert rig.tick() == 1 + job = sent_jobs(session)[0] + assert job == {"type": "job", "job_id": uid, "method": "GET", "url": TARGET_URL, "headers": JOB_HEADERS, "timeout": LIMITS.job_timeout_seconds} + assert store.get_request(uid)["status"] == "assigned" + assert store.get_request(uid)["device_uid"] == session.device_uid + assert uid in session.inflight + + +def test_the_result_completes_the_request_and_is_metered(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + rig.answer(session, uid, headers={"Content-Type": "application/json", "Set-Cookie": "x=1"}) + result = rig.broker.read(uid) + assert result.ok and result.http_status == 200 and result.body == b"hello" + assert result.headers == {"content-type": "application/json"} + assert uid not in session.inflight + today = store.usage_today(session.device_uid) + assert today["requests_ok"] == 1 and today["bytes_down"] == 5 and today["bytes_up"] > len(TARGET_URL) + assert store.get_donor(session.user_uid)["requests_ok"] == 1 + assert session.used_today == today["bytes_up"] + today["bytes_down"] + assert rig.counters.snapshot()["jobs_done"] == 1 + + +def test_no_eligible_node_leaves_the_request_queued(rig): + uid = rig.enqueue() + assert rig.tick() == 0 + assert store.get_request(uid)["status"] == "queued" + rig.connect(allowed_platforms=[]) + assert rig.tick() == 0 + assert store.get_request(uid)["status"] == "queued" + + +def test_one_request_goes_to_exactly_one_node(rig): + sessions = [rig.connect() for _ in range(4)] + rig.enqueue() + assert rig.tick() == 1 + assert sum(len(sent_jobs(item)) for item in sessions) == 1 + + +def test_a_caller_never_gets_its_own_node(rig): + session = rig.connect() + uid = rig.enqueue(caller=session.user_uid) + assert rig.tick() == 0 + assert store.get_request(uid)["status"] == "queued" + guest = rig.enqueue(caller=session.user_uid, kind="guest") + assert rig.tick() == 1 and guest in session.inflight + + +def test_a_policy_failure_at_dispatch_time_fails_the_request(rig): + rig.connect() + uid = store.enqueue_request("user", "c", "https://evil.example/", "GET", "x", "any") + rig.tick() + row = store.get_request(uid) + assert row["status"] == "failed" and row["error"] == "host_not_allowed" + assert rig.counters.snapshot()["jobs_refused"] == 1 + + +def test_a_job_error_fails_the_request_and_counts_against_the_node(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + run_async(session._handle({"type": "job_error", "job_id": uid, "code": "timeout"})) + assert store.get_request(uid)["error"] == "timeout" + assert store.usage_today(session.device_uid)["requests_failed"] == 1 + assert rig.counters.snapshot()["jobs_failed"] == 1 + + +def test_five_node_failures_open_that_nodes_circuit(rig): + session = rig.connect() + for _ in range(LIMITS.breaker_failures): + uid = rig.enqueue() + session.host_started.clear() + rig.scheduler._destination_started.clear() + rig.tick() + run_async(session._handle({"type": "job_error", "job_id": uid, "code": "network"})) + follow = rig.enqueue() + session.host_started.clear() + rig.scheduler._destination_started.clear() + assert rig.tick() == 0 + assert store.get_request(follow)["status"] == "queued" + rig.clock.now += LIMITS.breaker_open_seconds + assert rig.tick() == 1 + + +def test_donor_limit_codes_pause_the_node_without_blaming_it(rig): + session = rig.connect(bandwidth_mb_day=50) + uid = rig.enqueue() + rig.tick() + run_async(session._handle({"type": "job_error", "job_id": uid, "code": "bandwidth_exhausted"})) + assert session.used_today == 50 * MEGABYTE + assert rig.breaker.allows(("node", session.device_uid)) + other = rig.connect() + rig.clock.now += 10 + uid = rig.enqueue() + assert rig.tick() == 1 + run_async(other._handle({"type": "job_error", "job_id": uid, "code": "outside_hours"})) + assert other.paused_until > rig.clock.now + assert rig.breaker.allows(("node", other.device_uid)) + + +def test_a_donor_refusal_of_an_allowed_destination_raises_a_flag(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + run_async(session._handle({"type": "job_error", "job_id": uid, "code": "host_not_allowed"})) + kinds = [flag["kind"] for flag in store.list_flags()] + assert "refusal_mismatch" in kinds + assert rig.breaker.allows(("node", session.device_uid)) + + +@pytest.mark.parametrize( + "fields,code", + [ + ({"status": 99}, "bad_status"), + ({"status": 600}, "bad_status"), + ({"body_b64": "***not base64***"}, "bad_base64"), + ({"body_b64": "abc"}, "bad_base64"), + ], +) +def test_forged_results_are_integrity_failures(rig, fields, code): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + rig.answer(session, uid, **fields) + assert store.get_request(uid)["error"] == code + assert store.usage_today(session.device_uid)["integrity_failures"] == 1 + assert rig.counters.snapshot()["integrity_failures"] == 1 + assert any(flag["kind"] == "integrity" for flag in store.list_flags()) + assert store.get_donor(session.user_uid)["requests_ok"] == 0 + + +def test_an_oversized_body_is_truncated_and_flagged(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + big = base64.b64encode(b"x" * (LIMITS.max_body_bytes + 10)).decode() + rig.answer(session, uid, body_b64=big) + result = rig.broker.read(uid) + assert result.truncated and len(result.body) == LIMITS.max_body_bytes + assert rig.counters.snapshot()["results_truncated"] == 1 + + +def test_a_donor_declared_truncation_is_kept(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + rig.answer(session, uid, truncated=True) + assert rig.broker.read(uid).truncated + + +def test_a_result_for_an_unknown_job_is_a_protocol_error(rig): + session = rig.connect() + from devplacepy.services.nodes.frames import FrameError + + with pytest.raises(FrameError): + rig.answer(session, "never-assigned") + + +def test_a_job_cannot_be_answered_twice(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + rig.answer(session, uid) + from devplacepy.services.nodes.frames import FrameError + + with pytest.raises(FrameError): + rig.answer(session, uid) + assert store.get_donor(session.user_uid)["requests_ok"] == 1 + + +def test_rate_limited_and_server_error_results_open_the_destination_circuit(rig): + for index in range(LIMITS.breaker_failures): + session = rig.connect() + uid = rig.enqueue() + rig.scheduler._destination_started.clear() + rig.tick() + rig.answer(session, uid, status=429 if index % 2 else 503) + assert rig.broker.read(uid).ok + rig.registry.release(session) + assert not rig.breaker.allows(("destination", "reddit.com")) + rig.connect() + rig.enqueue() + rig.scheduler._destination_started.clear() + assert rig.tick() == 0 + + +def test_a_healthy_result_closes_the_destination_circuit(rig): + rig.breaker.failure(("destination", "reddit.com")) + session = rig.connect() + uid = rig.enqueue() + rig.tick() + rig.answer(session, uid, status=200) + assert rig.breaker.state(("destination", "reddit.com")) == "closed" + + +def test_a_job_that_never_answers_times_out(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + rig.clock.now += LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds + 1 + assert run_async(rig.dispatcher.sweep_inflight()) == 1 + assert store.get_request(uid)["error"] == "timeout" + assert session.inflight == {} + + +def test_a_disconnect_fails_every_inflight_job(rig): + session = rig.connect(max_concurrent=2) + uids = [rig.enqueue(), rig.enqueue()] + for _ in uids: + session.host_started.clear() + rig.scheduler._destination_started.clear() + assert rig.tick() == 1 + assert len(session.inflight) == 2 + assert rig.dispatcher.release_session(session) == 2 + for uid in uids: + row = store.get_request(uid) + assert row["status"] == "failed" and row["error"] == "node_disconnected" + assert session.inflight == {} + + +def test_release_session_marks_assigned_jobs_as_node_disconnected(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + assert rig.dispatcher.release_session(session) == 1 + assert store.get_request(uid)["error"] == "node_disconnected" + assert rig.dispatcher.release_session(session) == 0 + + +def test_a_failed_send_releases_the_job(rig): + session = rig.connect() + + async def broken(*args, **kwargs): + raise ConnectionError("gone") + + session.send = broken + uid = rig.enqueue() + assert rig.tick() == 0 + assert store.get_request(uid)["error"] == "send_failed" + assert session.inflight == {} + + +def test_an_idle_tick_does_no_writes_and_no_work(rig): + rig.connect() + started = time.perf_counter() + for _ in range(50): + assert rig.tick() == 0 + assert (time.perf_counter() - started) / 50 < 0.02 + + +def test_the_cap_refusing_the_ledger_pauses_the_node(rig): + session = rig.connect(bandwidth_mb_day=1) + store.save_node_settings(session.user_uid, {"enabled": True, "bandwidth_mb_day": 1}) + session.used_today = 0 + uid = rig.enqueue() + rig.tick() + store.record_job(session.device_uid, session.user_uid, ok=True, bytes_down=MEGABYTE, cap_bytes=MEGABYTE) + rig.answer(session, uid) + assert rig.broker.read(uid).ok + assert session.used_today >= MEGABYTE + + +def test_every_assigned_job_is_answered_at_most_once_under_a_random_workload(rig): + rng = random.Random(11) + sessions = [rig.connect(max_concurrent=2) for _ in range(4)] + seen = set() + for _ in range(150): + rig.enqueue(caller=generate_uid()) + rig.clock.now += 10 + rig.scheduler._destination_started.clear() + rig.tick() + for session in sessions: + for job_id in list(session.inflight): + assert job_id not in seen + action = rng.random() + if action < 0.6: + rig.answer(session, job_id) + seen.add(job_id) + elif action < 0.8: + run_async(session._handle({"type": "job_error", "job_id": job_id, "code": "network"})) + seen.add(job_id) + assert len(session.inflight) <= session.max_concurrent + open_now = {job for item in sessions for job in item.inflight} + assert not (open_now & seen) + + +def test_body_helpers(): + assert decode_body(base64.b64encode(b"abc").decode()) == (b"abc", False) + with pytest.raises(BodyError): + decode_body("!!!") + assert clean_headers({"A": "1", "set-cookie": "s", "Authorization": "x", 5: "x", "b": 5}) == {"a": "1"} + assert clean_headers("nope") == {} + many = {f"h{i}": "v" for i in range(500)} + assert len(clean_headers(many)) == 64 + assert len(next(iter(clean_headers({"k": "v" * 5000}).values()))) == 1024 + assert json.loads(json.dumps(clean_headers({"A": "1"}))) == {"a": "1"} + + +def test_a_saturated_destination_is_scanned_once_per_tick_not_once_per_request(rig, monkeypatch): + rig.connect(allowed_platforms=[]) + for index in range(30): + rig.enqueue(caller=f"c{index}") + scans = [] + original = rig.scheduler.candidates + monkeypatch.setattr(rig.scheduler, "candidates", lambda *a: scans.append(a) or original(*a)) + assert rig.tick() == 0 + assert len(scans) == 1 + + +def test_saturation_does_not_hide_other_destinations(rig): + session = rig.connect() + for index in range(3): + rig.enqueue(caller=f"c{index}") + rig.scheduler._destination_started["reddit.com"] = rig.clock.now + assert rig.tick() == 0 + rig.clock.now += 10 + assert rig.tick() == 1 + assert len(session.inflight) == 1 + + +def test_the_owner_exclusion_does_not_saturate_the_destination(rig): + mine = rig.connect() + other = rig.connect() + first = rig.enqueue(caller=mine.user_uid) + second = rig.enqueue(caller=other.user_uid) + rig.clock.now += 10 + rig.tick() + assert first in other.inflight + rig.clock.now += 10 + rig.scheduler._destination_started.clear() + other.host_started.clear() + rig.tick() + assert second in mine.inflight diff --git a/tests/unit/services/nodes/fabric.py b/tests/unit/services/nodes/fabric.py new file mode 100644 index 00000000..8f61c85e --- /dev/null +++ b/tests/unit/services/nodes/fabric.py @@ -0,0 +1,231 @@ +# retoor + +import json +from datetime import datetime, timezone + +import pytest + +from devplacepy.database import get_table, set_setting +from devplacepy.database import nodes as store +from devplacepy.services.nodes.broker import BrokerResult, JobBroker +from devplacepy.services.nodes.cache import ResultCache +from devplacepy.services.nodes.fabric import ( + FabricQuotaExceeded, + FabricRefused, + FabricUnavailable, + NodeFabric, +) +from devplacepy.services.nodes.spec import SPEC +from devplacepy.utils import generate_uid +from tests.conftest import run_async + +LIMITS = SPEC.limits +URL = "https://www.reddit.com/r/python.json" + + +class ScriptedBroker(JobBroker): + def __init__(self, outcomes): + self.outcomes = list(outcomes) + self.enqueued = 0 + + def enqueue(self, caller_kind, caller_id, target, mode): + self.enqueued += 1 + return f"uid-{self.enqueued}" + + async def wait(self, uid, timeout): + outcome = self.outcomes.pop(0) + if outcome == "ok": + return BrokerResult(uid, "done", 200, {"content-type": "text/plain"}, b"payload", False, "") + return BrokerResult(uid, "failed", 0, {}, b"", False, outcome) + + +def heartbeat(connected): + table = get_table("service_state") + record = { + "name": "nodes", + "status": "running", + "heartbeat": datetime.now(timezone.utc).isoformat(), + "metrics": json.dumps({"stats": [{"label": "connected", "value": connected}]}), + } + existing = table.find_one(name="nodes") + if existing: + table.update({**record, "id": existing["id"]}, ["id"]) + else: + table.insert(record) + + +@pytest.fixture +def online(local_db): + set_setting("nodes_enabled", "1") + set_setting("nodes_daily_quota_user", "1000") + set_setting("nodes_daily_quota_guest", "0") + heartbeat(3) + yield + set_setting("nodes_enabled", "0") + get_table("service_state").delete(name="nodes") + + +def fetch(fabric, **kwargs): + kwargs.setdefault("caller_kind", "user") + kwargs.setdefault("caller_id", generate_uid()) + return run_async(fabric.fetch(URL, **kwargs)) + + +def test_a_node_fetch_returns_the_body_and_caches_it(online): + broker = ScriptedBroker(["ok"]) + fabric = NodeFabric(broker=broker) + first = fetch(fabric, caller_id="c") + assert (first.status, first.body, first.via) == (200, b"payload", "fetch") + second = fetch(fabric, caller_id="c") + assert second.via == "cache" and second.body == b"payload" and second.elapsed_ms == 0 + assert broker.enqueued == 1 + + +def test_use_cache_false_always_asks_a_node(online): + broker = ScriptedBroker(["ok", "ok"]) + fabric = NodeFabric(broker=broker) + fetch(fabric, caller_id="c", use_cache=False) + fetch(fabric, caller_id="c", use_cache=False) + assert broker.enqueued == 2 + + +def test_policy_refusals_surface_with_their_code(online): + fabric = NodeFabric(broker=ScriptedBroker([])) + for url, code in (("https://example.com/", "host_not_allowed"), ("https://reddit.com:22/", "port_not_allowed")): + with pytest.raises(FabricRefused) as info: + run_async(fabric.fetch(url, caller_kind="user", caller_id="c")) + assert info.value.code == code + + +def test_the_kill_switch_and_missing_nodes_are_unavailable(online): + fabric = NodeFabric(broker=ScriptedBroker([])) + heartbeat(0) + with pytest.raises(FabricUnavailable) as info: + fetch(fabric) + assert info.value.code == "no_nodes" + heartbeat(2) + set_setting("nodes_enabled", "0") + with pytest.raises(FabricUnavailable) as info: + fetch(fabric) + assert info.value.code == "kill_switch" + + +def test_a_stale_heartbeat_counts_as_no_nodes(online): + table = get_table("service_state") + row = table.find_one(name="nodes") + table.update({"id": row["id"], "heartbeat": "2000-01-01T00:00:00+00:00"}, ["id"]) + assert NodeFabric(broker=ScriptedBroker([])).available().nodes_online == 0 + + +def test_no_service_row_means_no_nodes(online): + get_table("service_state").delete(name="nodes") + assert NodeFabric().available().nodes_online == 0 + + +def test_quota_is_enforced_before_any_node_is_touched(online): + set_setting("nodes_daily_quota_user", "2") + broker = ScriptedBroker(["ok"] * 5) + fabric = NodeFabric(broker=broker) + caller = generate_uid() + for _ in range(2): + fetch(fabric, caller_id=caller, use_cache=False) + with pytest.raises(FabricQuotaExceeded): + fetch(fabric, caller_id=caller, use_cache=False) + assert broker.enqueued == 2 + + +def test_guests_have_no_quota_by_default(online): + broker = ScriptedBroker(["ok"]) + with pytest.raises(FabricQuotaExceeded): + fetch(NodeFabric(broker=broker), caller_kind="guest", caller_id="g") + assert broker.enqueued == 0 + + +def test_a_cache_hit_costs_no_quota(online): + set_setting("nodes_daily_quota_user", "1") + fabric = NodeFabric(broker=ScriptedBroker(["ok"])) + caller = generate_uid() + fetch(fabric, caller_id=caller) + for _ in range(5): + assert fetch(fabric, caller_id=caller).via == "cache" + + +def test_a_retryable_failure_is_retried_once_on_another_node(online): + broker = ScriptedBroker(["node_disconnected", "ok"]) + result = fetch(NodeFabric(broker=broker)) + assert result.body == b"payload" and broker.enqueued == 2 + + +def test_the_retry_does_not_spend_a_second_quota_unit(online): + set_setting("nodes_daily_quota_user", "1") + caller = generate_uid() + fetch(NodeFabric(broker=ScriptedBroker(["timeout", "ok"])), caller_id=caller) + assert store.caller_requests_today("user", caller) == 1 + + +def test_retries_stop_after_the_attempt_cap(online): + broker = ScriptedBroker(["timeout"] * 5) + with pytest.raises(FabricUnavailable) as info: + fetch(NodeFabric(broker=broker)) + assert info.value.code == "timeout" and broker.enqueued == 2 + + +def test_a_non_retryable_failure_is_not_retried(online): + broker = ScriptedBroker(["bad_status", "ok"]) + with pytest.raises(FabricUnavailable): + fetch(NodeFabric(broker=broker)) + assert broker.enqueued == 1 + + +def test_a_full_queue_is_a_clean_unavailable(online): + class Full(ScriptedBroker): + def enqueue(self, *args): + raise store.QueueFull("full") + + with pytest.raises(FabricUnavailable) as info: + fetch(NodeFabric(broker=Full([]))) + assert info.value.code == "queue_full" + + +def test_the_wait_is_capped_at_the_spec_maximum(online): + seen = [] + + class Spy(ScriptedBroker): + async def wait(self, uid, timeout): + seen.append(timeout) + return await super().wait(uid, timeout) + + fetch(NodeFabric(broker=Spy(["ok"])), timeout=10**6) + assert seen[0] <= LIMITS.max_wait_seconds + + +def test_the_cache_ttl_follows_the_admin_setting_up_to_the_ceiling(online): + cache = ResultCache() + fabric = NodeFabric(broker=ScriptedBroker(["ok"]), cache=cache) + set_setting("nodes_cache_ttl_seconds", "0") + fetch(fabric, caller_id="c") + assert len(cache) == 0 + set_setting("nodes_cache_ttl_seconds", str(10**9)) + fabric._sync_cache_ttl() + assert cache.ttl_seconds == LIMITS.cache_ttl_seconds + set_setting("nodes_cache_ttl_seconds", str(LIMITS.cache_ttl_seconds)) + + +def test_can_serve_needs_policy_switch_and_a_node(online): + fabric = NodeFabric(broker=ScriptedBroker([])) + assert fabric.can_serve(URL) + assert not fabric.can_serve("https://example.com/") + assert not fabric.can_serve(URL, "POST") + heartbeat(0) + assert not fabric.can_serve(URL) + + +def test_the_real_broker_times_out_a_request_nobody_picks_up(online): + get_table("node_requests").delete() + fabric = NodeFabric() + with pytest.raises(FabricUnavailable) as info: + fetch(fabric, timeout=0.3) + assert info.value.code in ("caller_timeout", "failed") + rows = list(get_table("node_requests").find()) + assert rows and all(row["status"] == "failed" for row in rows) + get_table("node_requests").delete() diff --git a/tests/unit/services/nodes/ledger.py b/tests/unit/services/nodes/ledger.py new file mode 100644 index 00000000..2dea3b7d --- /dev/null +++ b/tests/unit/services/nodes/ledger.py @@ -0,0 +1,55 @@ +# retoor + +from devplacepy.database import get_table +from devplacepy.database import nodes as store +from devplacepy.services.nodes import ledger as ledger_module +from devplacepy.services.nodes.ledger import FIRST_BADGE, GIGABYTE, UsageLedger +from devplacepy.utils import generate_uid, register_account + + +def member(local_db): + name = "ledger" + generate_uid()[-10:] + uid, _, _ = register_account(name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True) + return uid + + +def badges(uid): + return {row["badge_name"] for row in get_table("badges").find(user_uid=uid)} + + +def test_first_success_awards_xp_and_the_first_badge(local_db): + uid = member(local_db) + before = get_table("users").find_one(uid=uid).get("xp") or 0 + store.record_job(generate_uid(), uid, ok=True, bytes_up=10, bytes_down=100) + UsageLedger().reward(uid) + assert FIRST_BADGE in badges(uid) + assert (get_table("users").find_one(uid=uid).get("xp") or 0) > before + + +def test_xp_stops_at_the_daily_cap_but_not_the_counters(local_db): + uid = member(local_db) + ledger = UsageLedger() + store.record_job(generate_uid(), uid, ok=True, bytes_up=1, bytes_down=1) + granted = 0 + for _ in range(store.LIMITS.xp_daily_cap + 10): + granted += int(ledger.claim_xp(uid, ledger_module.XP_PER_JOB)) + assert granted == store.LIMITS.xp_daily_cap + + +def test_byte_badges_follow_lifetime_totals(local_db): + uid = member(local_db) + store.record_job( + generate_uid(), uid, ok=True, bytes_up=0, bytes_down=GIGABYTE, cap_bytes=2 * GIGABYTE + ) + UsageLedger().reward(uid) + assert {"Gigabyte Donor"} <= badges(uid) + assert "10 GB Donor" not in badges(uid) + + +def test_a_second_reward_call_does_not_duplicate_badges(local_db): + uid = member(local_db) + store.record_job(generate_uid(), uid, ok=True, bytes_up=1, bytes_down=1) + ledger = UsageLedger() + ledger.reward(uid) + ledger.reward(uid) + assert len(list(get_table("badges").find(user_uid=uid, badge_name=FIRST_BADGE))) == 1 diff --git a/tests/unit/services/nodes/policy.py b/tests/unit/services/nodes/policy.py new file mode 100644 index 00000000..63700cd7 --- /dev/null +++ b/tests/unit/services/nodes/policy.py @@ -0,0 +1,143 @@ +# retoor + +import pytest + +from devplacepy.services.nodes.policy import DestinationPolicy, PolicyError, is_public_address +from devplacepy.services.nodes.spec import SPEC + +POLICY = DestinationPolicy() + + +def code(url, method="GET", mode="any"): + with pytest.raises(PolicyError) as info: + POLICY.check(url, method, mode) + return info.value.code + + +@pytest.mark.parametrize( + "url", + [ + "https://reddit.com/r/python.json", + "https://www.reddit.com/r/python/top.json?t=week", + "http://old.reddit.com/", + "https://REDDIT.COM/", + "https://reddit.com./r/x", + "https://redd.it/abc", + "https://reddit.com:443/x", + "http://reddit.com:80/x", + ], +) +def test_allowlisted_urls_pass(url): + target = POLICY.check(url) + assert target.method == "GET" and target.adapter == "reddit" + assert target.port in (80, 443) + + +@pytest.mark.parametrize( + "url", + [ + "https://evilreddit.com/", + "https://reddit.com.evil.net/", + "https://notreddit.com/", + "https://example.com/", + "https://localhost/", + "https://reddit.evil.com/", + "https://evil.com/reddit.com", + ], +) +def test_other_hosts_are_refused(url): + assert code(url) == "host_not_allowed" + + +@pytest.mark.parametrize( + "url", + [ + "https://127.0.0.1/", + "https://[::1]/", + "http://10.0.0.5/", + "https://169.254.169.254/latest/meta-data", + "https://0x7f.1/", + "https://2130706433/", + ], +) +def test_address_literals_are_refused(url): + assert code(url) in ("host_not_allowed", "bad_url") + + +@pytest.mark.parametrize("port", [22, 25, 8080, 8443, 1, 65535]) +def test_only_web_ports(port): + assert code(f"https://reddit.com:{port}/") == "port_not_allowed" + + +@pytest.mark.parametrize( + "url", + ["ftp://reddit.com/", "file:///etc/passwd", "javascript:alert(1)", "//reddit.com/x", "reddit.com/x", ""], +) +def test_other_schemes_and_shapes_are_bad_urls(url): + assert code(url) == "bad_url" + + +def test_credentials_in_the_url_are_refused(): + assert code("https://user:pw@reddit.com/") == "bad_url" + assert code("https://user@reddit.com/") == "bad_url" + assert code("https://evil.com@reddit.com.evil.com/") == "bad_url" + assert code("https://reddit.com@evil.com/") == "bad_url" + + +def test_overlong_and_non_string_urls_are_refused(): + assert code("https://reddit.com/" + "a" * SPEC.limits.url_max_chars) == "bad_url" + with pytest.raises(PolicyError): + POLICY.check(None) + with pytest.raises(PolicyError): + POLICY.check(["https://reddit.com/"]) + + +@pytest.mark.parametrize("method", ["POST", "PUT", "DELETE", "PATCH", "CONNECT", ""]) +def test_only_declared_methods(method): + assert code("https://reddit.com/", method) == "method_not_allowed" + + +def test_method_is_case_insensitive(): + assert POLICY.check("https://reddit.com/", "get").method == "GET" + + +def test_unknown_mode_is_refused(): + assert code("https://reddit.com/", mode="raw") == "bad_mode" + + +def test_a_redirect_off_the_allowlist_is_a_redirect_refusal(): + with pytest.raises(PolicyError) as info: + POLICY.check_redirect("https://evil.example/") + assert info.value.code == "redirect_not_allowed" + assert POLICY.check_redirect("https://www.reddit.com/new").host == "www.reddit.com" + + +@pytest.mark.parametrize( + "address,expected", + [ + ("8.8.8.8", True), + ("1.1.1.1", True), + ("2606:4700:4700::1111", True), + ("127.0.0.1", False), + ("10.1.2.3", False), + ("172.16.0.1", False), + ("192.168.1.1", False), + ("169.254.1.1", False), + ("100.64.0.1", False), + ("100.127.255.254", False), + ("224.0.0.1", False), + ("0.0.0.0", False), + ("::1", False), + ("fe80::1", False), + ("fc00::1", False), + ("not an ip", False), + ], +) +def test_public_address_classification(address, expected): + assert is_public_address(address) is expected + + +def test_every_destination_in_the_spec_passes_its_own_policy(): + for destination in SPEC.destinations: + target = POLICY.check(f"https://{destination.suffix}/x") + assert target.destination is destination diff --git a/tests/unit/services/nodes/scheduler.py b/tests/unit/services/nodes/scheduler.py new file mode 100644 index 00000000..10acfd38 --- /dev/null +++ b/tests/unit/services/nodes/scheduler.py @@ -0,0 +1,283 @@ +# retoor + +import random +from collections import Counter +from datetime import datetime, timezone + +import pytest + +from devplacepy.services.nodes.circuit import CircuitBreaker +from devplacepy.services.nodes.policy import DestinationPolicy +from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry +from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler, in_active_hours +from devplacepy.services.nodes.session import Inflight, NodeSession +from devplacepy.services.nodes.spec import SPEC +from tests.unit.services.nodes.session import FakeSocket + +LIMITS = SPEC.limits +TARGET = DestinationPolicy().check("https://www.reddit.com/r/python.json") + + +class Clock: + def __init__(self): + self.now = 1000.0 + + def __call__(self): + return self.now + + +def settings(**overrides): + base = { + "enabled": True, + "bandwidth_mb_day": 100, + "max_concurrent": 2, + "active_from": "", + "active_to": "", + "timezone": "UTC", + "allowed_platforms": ["reddit"], + "pause_on_battery": True, + "public_display": False, + } + base.update(overrides) + return base + + +def node(uid="d1", user="u1", modes=("fetch",), **overrides): + session = NodeSession(FakeSocket(), {"uid": uid, "user_uid": user, "kind": "native"}) + session.hello = {"modes": list(modes)} + session.settings = settings(**overrides) + session.roll_day() + return session + + +@pytest.fixture +def world(local_db): + clock = Clock() + registry = NodeRegistry(NodeCounters()) + breaker = CircuitBreaker(clock=clock) + wall = {"now": datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)} + scheduler = NodeScheduler(registry, breaker, clock, lambda: wall["now"], random.Random(1)) + scheduler.wall = wall + scheduler.clock = clock + return scheduler + + +def add(world, **kwargs): + session = node(**kwargs) + world.registry.admit(session) + return session + + +def test_a_healthy_node_is_picked(world): + session = add(world) + assert world.pick(TARGET) is session + + +def test_no_nodes_means_no_pick(world): + assert world.pick(TARGET) is None + + +def test_tunnel_only_jobs_are_not_dispatchable_yet(world): + add(world) + assert world.pick(TARGET, "tunnel") is None + assert world.pick(TARGET, "fetch") is not None + assert world.pick(TARGET, "any") is not None + + +def test_a_node_must_speak_the_transport(world): + add(world, modes=("other",)) + assert world.pick(TARGET) is None + + +def test_a_tunnel_node_serves_tunnel_and_any_but_not_fetch(world): + session = add(world, modes=("tunnel",)) + assert world.pick(TARGET, "tunnel") is session + assert world.pick(TARGET, "any") is session + assert world.pick(TARGET, "fetch") is None + assert world.transport_of(session, TARGET, "any") == "tunnel" + + +def test_any_prefers_nodes_that_can_tunnel(world): + fetcher = add(world, uid="f1", user="u1", modes=("fetch",)) + tunneler = add(world, uid="t1", user="u2", modes=("fetch", "tunnel")) + assert world.candidates(TARGET, "any") == [tunneler] + assert world.transport_of(fetcher, TARGET, "any") == "fetch" + + +def test_platform_opt_out_is_honoured(world): + add(world, allowed_platforms=[]) + assert world.pick(TARGET) is None + + +def test_a_node_never_serves_its_own_owner(world): + add(world, user="alice") + assert world.pick(TARGET, caller_owner="alice") is None + assert world.pick(TARGET, caller_owner="bob") is not None + + +def test_concurrency_is_capped_by_the_lower_of_server_and_donor(world): + session = add(world, max_concurrent=1) + session.inflight["j"] = Inflight("j", "h", "reddit.com", "GET", "u", 0, 0) + assert world.pick(TARGET) is None + session.inflight.clear() + assert session.max_concurrent == 1 + session.settings["max_concurrent"] = 99 + assert session.max_concurrent == LIMITS.max_concurrent_per_node + session.hello["limits"] = {"max_concurrent": 1} + assert session.max_concurrent == 1 + + +def test_the_donor_bandwidth_cap_stops_assignment(world): + session = add(world, bandwidth_mb_day=1) + session.used_today = MEGABYTE - 1 + assert world.pick(TARGET) is session + session.used_today = MEGABYTE + assert world.pick(TARGET) is None + + +def test_a_new_day_resets_the_used_bytes(world): + session = add(world, bandwidth_mb_day=1) + session.used_today = MEGABYTE + session.used_day = "2000-01-01" + assert world.pick(TARGET) is session + assert session.used_today == 0 + + +def test_a_node_waits_between_requests_to_the_same_destination(world): + session = add(world) + world.mark_started(session, TARGET) + world.clock.now += LIMITS.min_seconds_per_node_domain - 0.1 + assert world.eligible(TARGET) == [] + world.clock.now += 0.2 + world.clock.now += 1.0 / TARGET.destination.max_rps + assert world.eligible(TARGET) == [session] + + +def test_the_destination_has_a_global_rate_ceiling(world): + first, second = add(world, uid="a", user="ua"), add(world, uid="b", user="ub") + world.mark_started(first, TARGET) + world.clock.now += 1.0 / TARGET.destination.max_rps - 0.01 + assert world.pick(TARGET) is None + world.clock.now += 0.02 + assert world.pick(TARGET) in (first, second) + + +def test_a_paused_node_is_skipped_until_the_pause_ends(world): + session = add(world) + session.paused_until = world.clock.now + 5 + assert world.pick(TARGET) is None + world.clock.now += 5 + assert world.pick(TARGET) is session + + +def test_an_open_node_circuit_removes_only_that_node(world): + bad, good = add(world, uid="bad", user="u1"), add(world, uid="good", user="u2") + for _ in range(LIMITS.breaker_failures): + world.breaker.failure(("node", "bad")) + assert world.eligible(TARGET) == [good] + world.clock.now += LIMITS.breaker_open_seconds + assert set(world.eligible(TARGET)) == {bad, good} + + +def test_an_open_destination_circuit_stops_everything(world): + add(world) + for _ in range(LIMITS.breaker_failures): + world.breaker.failure(("destination", TARGET.destination.suffix)) + assert world.pick(TARGET) is None + + +def test_least_loaded_then_least_recently_used_wins(world): + busy, idle_old, idle_new = add(world, uid="a", user="1"), add(world, uid="b", user="2"), add(world, uid="c", user="3") + busy.inflight["j"] = Inflight("j", "h", "reddit.com", "GET", "u", 0, 0) + idle_old.last_assigned = 10 + idle_new.last_assigned = 20 + assert world.pick(TARGET) is idle_old + + +def test_selection_is_fair_across_equal_nodes(world): + sessions = [add(world, uid=f"d{i}", user=f"u{i}") for i in range(5)] + picks = Counter() + for _ in range(500): + chosen = world.pick(TARGET) + picks[chosen.device_uid] += 1 + chosen.last_assigned = world.clock.now + world.clock.now += 0.001 + assert set(picks) == {item.device_uid for item in sessions} + assert max(picks.values()) - min(picks.values()) <= 1 + + +@pytest.mark.parametrize( + "start,end,hour,minute,expected", + [ + ("", "", 3, 0, True), + ("09:00", "17:00", 9, 0, True), + ("09:00", "17:00", 16, 59, True), + ("09:00", "17:00", 17, 0, False), + ("09:00", "17:00", 8, 59, False), + ("22:00", "06:00", 23, 0, True), + ("22:00", "06:00", 5, 59, True), + ("22:00", "06:00", 6, 0, False), + ("22:00", "06:00", 12, 0, False), + ("09:00", "09:00", 3, 0, True), + ("bogus", "17:00", 3, 0, True), + ], +) +def test_active_hours_windows(start, end, hour, minute, expected): + moment = datetime(2026, 1, 1, hour, minute, tzinfo=timezone.utc) + assert in_active_hours({"active_from": start, "active_to": end, "timezone": "UTC"}, moment) is expected + + +def test_active_hours_use_the_donors_timezone(): + moment = datetime(2026, 1, 1, 22, 0, tzinfo=timezone.utc) + amsterdam = {"active_from": "09:00", "active_to": "17:00", "timezone": "Europe/Amsterdam"} + assert in_active_hours(amsterdam, moment) is False + morning = datetime(2026, 1, 1, 8, 30, tzinfo=timezone.utc) + assert in_active_hours(amsterdam, morning) is True + broken = {"active_from": "09:00", "active_to": "17:00", "timezone": "Mars/Base"} + assert in_active_hours(broken, datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) is True + + +def test_a_node_outside_its_hours_is_skipped(world): + add(world, active_from="22:00", active_to="23:00") + assert world.pick(TARGET) is None + world.wall["now"] = datetime(2026, 1, 1, 22, 30, tzinfo=timezone.utc) + assert world.pick(TARGET) is not None + + +def test_mark_started_records_both_clocks(world): + session = add(world) + world.mark_started(session, TARGET) + assert session.host_started[TARGET.destination.suffix] == world.clock.now + assert session.last_assigned == world.clock.now + assert world._destination_started[TARGET.destination.suffix] == world.clock.now + + +def test_the_per_destination_maps_stay_bounded_by_the_allowlist(world): + session = add(world) + policy = DestinationPolicy() + for index in range(200): + world.mark_started(session, policy.check(f"https://sub{index}.reddit.com/")) + assert len(session.host_started) == 1 + assert len(world._destination_started) == 1 + + +def test_window_checks_are_cached_per_minute_and_still_correct(world): + from devplacepy.services.nodes.scheduler import _window_open + + _window_open.cache_clear() + window = {"active_from": "09:00", "active_to": "17:00", "timezone": "UTC"} + noon = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + for _ in range(100): + assert in_active_hours(window, noon) is True + assert _window_open.cache_info().misses == 1 + night = datetime(2026, 1, 1, 3, 0, tzinfo=timezone.utc) + assert in_active_hours(window, night) is False + + +def test_choose_applies_the_owner_exclusion_to_a_shared_candidate_list(world): + mine, theirs = add(world, uid="a", user="alice"), add(world, uid="b", user="bob") + found = world.candidates(TARGET) + assert world.choose(found, "alice") is theirs + assert world.choose(found, "bob") is mine + assert world.choose([mine], "alice") is None + assert world.choose([], "") is None diff --git a/tests/unit/services/nodes/service.py b/tests/unit/services/nodes/service.py index 2b035fab..4b5c2c3b 100644 --- a/tests/unit/services/nodes/service.py +++ b/tests/unit/services/nodes/service.py @@ -1,5 +1,6 @@ # retoor +import asyncio import time import pytest @@ -146,3 +147,32 @@ def test_metrics_report_connected_and_capacity(service): def test_the_service_has_the_kill_switch_as_a_config_field(service): keys = [field.key for field in service.all_fields()] assert "nodes_enabled" in keys and "nodes_max_total" in keys + + +def test_changed_settings_are_pushed_to_the_node_once(service): + uid = member() + device, socket = connected(service, uid) + session = service.registry.get(device["uid"]) + session.settings = {"stale": True} + assert reconcile(service) == 0 + pushed = [frame for frame in socket.sent if frame["type"] == "settings_update"] + assert len(pushed) == 1 and pushed[0]["settings"]["enabled"] is True + assert session.settings == pushed[0]["settings"] + reconcile(service) + assert len([frame for frame in socket.sent if frame["type"] == "settings_update"]) == 1 + save_node_settings(uid, {"bandwidth_mb_day": 9}) + reconcile(service) + assert session.settings["bandwidth_mb_day"] == 9 + + +def test_the_dispatch_loop_starts_once_and_stops_with_the_service(service): + async def scenario(): + service.ensure_dispatching() + first = service._dispatch_task + service.ensure_dispatching() + assert service._dispatch_task is first and not first.done() + await service.on_disable() + await asyncio.sleep(0) + assert service._dispatch_task is None and first.cancelled() + + run_async(scenario()) diff --git a/tests/unit/services/nodes/session.py b/tests/unit/services/nodes/session.py index e68b485c..99e1e35f 100644 --- a/tests/unit/services/nodes/session.py +++ b/tests/unit/services/nodes/session.py @@ -93,15 +93,20 @@ def test_a_silent_client_times_out(local_db, monkeypatch): def test_usage_is_advisory_and_summed(): session = session_for(FakeSocket()) - session._handle({"type": "usage", "bytes_up": 5, "bytes_down": 7}) - session._handle({"type": "usage", "bytes_up": -9, "bytes_down": 1}) + run(session._handle({"type": "usage", "bytes_up": 5, "bytes_down": 7})) + run(session._handle({"type": "usage", "bytes_up": -9, "bytes_down": 1})) assert (session.reported_up, session.reported_down) == (5, 8) -@pytest.mark.parametrize("kind", ["job_result", "job_error", "tunnel_data", "tunnel_close", "tunnel_opened"]) +@pytest.mark.parametrize("kind", ["job_result", "job_error"]) def test_frames_without_an_assigned_job_are_protocol_errors(kind): with pytest.raises(FrameError): - session_for(FakeSocket())._handle({"type": kind}) + run(session_for(FakeSocket())._handle({"type": kind, "job_id": "nope"})) + + +@pytest.mark.parametrize("kind", ["tunnel_data", "tunnel_close", "tunnel_opened"]) +def test_late_tunnel_frames_for_a_closed_relay_are_ignored(kind): + run(session_for(FakeSocket())._handle({"type": kind, "conn_id": "gone"})) def test_missed_pings_evict_after_the_floor(monkeypatch): diff --git a/tests/unit/services/nodes/social.py b/tests/unit/services/nodes/social.py new file mode 100644 index 00000000..58bda3c5 --- /dev/null +++ b/tests/unit/services/nodes/social.py @@ -0,0 +1,88 @@ +# retoor + +import json + +from devplacepy.database import set_setting +from devplacepy.services.nodes import social as social_module +from devplacepy.services.nodes.adapters.base import AdapterError +from devplacepy.services.nodes.fabric import FabricUnavailable +from devplacepy.services.nodes.social import SocialSearch +from tests.conftest import run_async + +HN = json.dumps({"hits": [{"objectID": "1", "title": "HN story", "points": 1}]}).encode() +SE = json.dumps({"items": [{"title": "SO question", "link": "https://stackoverflow.com/q/1"}]}).encode() + + +class StubFabric: + def __init__(self, error=None): + self.error = error + self.calls = 0 + + async def fetch(self, url, **kwargs): + self.calls += 1 + if self.error: + raise self.error + raise AssertionError("not expected") + + +def search(monkeypatch, getter, platforms, fabric=None): + monkeypatch.setattr(social_module, "direct_get", getter) + return run_async( + SocialSearch(fabric or StubFabric()).search( + "sqlite", caller_kind="user", caller_id="u1", platforms=platforms + ) + ) + + +def test_hits_are_merged_and_failures_reported(monkeypatch): + async def getter(url): + if "algolia" in url: + return HN + if "stackexchange" in url: + return SE + raise AdapterError("upstream", "HTTP 500") + + outcome = search(monkeypatch, getter, ["hackernews", "stackexchange", "github"]) + assert {item.platform for item in outcome.hits} == {"hackernews", "stackexchange"} + assert outcome.unavailable == {"github": "upstream"} + + +def test_unknown_platform_is_dropped(monkeypatch): + async def getter(url): + return HN + + outcome = search(monkeypatch, getter, ["hackernews", "x", "instagram"]) + assert [item.platform for item in outcome.hits] == ["hackernews"] + assert outcome.unavailable == {} + + +def test_reddit_is_off_unless_routing_through_nodes_is_enabled(monkeypatch): + async def getter(url): + return HN + + fabric = StubFabric() + outcome = search(monkeypatch, getter, ["reddit"], fabric) + assert outcome.unavailable == {"reddit": "nodes_not_used"} + assert fabric.calls == 0 + + +def test_reddit_reports_the_fabric_reason(monkeypatch): + async def getter(url): + return HN + + set_setting("nodes_social_via_nodes", "1") + try: + outcome = search(monkeypatch, getter, ["reddit"], StubFabric(FabricUnavailable("no_nodes"))) + finally: + set_setting("nodes_social_via_nodes", "0") + assert outcome.unavailable == {"reddit": "no_nodes"} + + +def test_blank_query_and_limit_cap(monkeypatch): + async def getter(url): + return HN + + monkeypatch.setattr(social_module, "direct_get", getter) + empty = run_async(SocialSearch(StubFabric()).search(" ", caller_kind="user", caller_id="u")) + assert empty.hits == [] + assert social_module.MAX_LIMIT == 20 diff --git a/tests/unit/services/nodes/tunnel.py b/tests/unit/services/nodes/tunnel.py new file mode 100644 index 00000000..47d6a665 --- /dev/null +++ b/tests/unit/services/nodes/tunnel.py @@ -0,0 +1,112 @@ +# retoor + +import asyncio +import base64 + +from devplacepy.services.nodes import tunnel as tunnel_module +from devplacepy.services.nodes.policy import DestinationPolicy +from devplacepy.services.nodes.tunnel import Relay, TunnelBridge +from tests.conftest import run_async + +TARGET = DestinationPolicy().check("https://www.reddit.com/r/python.json") + + +class Donor: + def __init__(self, bridge, accept=True, garbage=False): + self.bridge = bridge + self.accept = accept + self.garbage = garbage + self.tunnels = {} + self.sent = [] + + async def send(self, frame_type, **fields): + self.sent.append({"type": frame_type, **fields}) + if frame_type == "tunnel_open": + self.bridge.on_frame(self, {"type": "tunnel_opened", "conn_id": fields["conn_id"], "ok": self.accept}) + elif frame_type == "tunnel_data": + raw = base64.b64decode(fields["b64"]) + reply = "***not base64***" if self.garbage else base64.b64encode(raw.upper()).decode() + self.bridge.on_frame(self, {"type": "tunnel_data", "conn_id": fields["conn_id"], "b64": reply}) + + +async def session_through(donor, connect_line, payload=b"hello"): + relay = Relay(donor, TARGET) + server = await asyncio.start_server(relay.handle, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(connect_line.encode() + b"\r\n\r\n") + await writer.drain() + status = await asyncio.wait_for(reader.readline(), 5) + echoed = b"" + if b" 200 " in status: + await reader.readuntil(b"\r\n") + writer.write(payload) + await writer.drain() + try: + echoed = await asyncio.wait_for(reader.read(len(payload)), 5) + except asyncio.TimeoutError: + echoed = b"" + writer.close() + server.close() + await asyncio.sleep(0.05) + return status, echoed, relay + + +def go(donor, line, payload=b"hello"): + return run_async(session_through(donor, line, payload)) + + +def test_connect_to_the_target_relays_both_ways(): + donor = Donor(TunnelBridge()) + status, echoed, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1") + assert b" 200 " in status and echoed == b"HELLO" + assert [item["type"] for item in donor.sent][:2] == ["tunnel_open", "tunnel_data"] + assert donor.sent[0]["host"] == "www.reddit.com" and donor.sent[0]["port"] == 443 + assert relay.up == 5 and relay.down == 5 + + +def test_any_other_host_or_port_is_refused_without_bothering_the_donor(): + for line in ("CONNECT evil.example:443 HTTP/1.1", "CONNECT www.reddit.com:22 HTTP/1.1", "CONNECT 10.0.0.1:443 HTTP/1.1"): + donor = Donor(TunnelBridge()) + status, _, relay = go(donor, line) + assert b" 403 " in status and donor.sent == [] + assert relay.error == "host_not_allowed" + + +def test_only_connect_is_accepted(): + donor = Donor(TunnelBridge()) + status, _, _ = go(donor, "GET http://www.reddit.com/ HTTP/1.1") + assert b" 405 " in status and donor.sent == [] + + +def test_a_donor_that_refuses_is_a_bad_gateway(): + donor = Donor(TunnelBridge(), accept=False) + status, _, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1") + assert b" 502 " in status and relay.error == "tunnel_refused" + + +def test_the_relay_has_a_byte_cap(monkeypatch): + monkeypatch.setattr(tunnel_module, "RELAY_CAP_BYTES", 8) + donor = Donor(TunnelBridge()) + _, echoed, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1", b"0123456789abcdef") + assert relay.error == "body_too_large" and len(echoed) < 16 + + +def test_invalid_base64_from_the_donor_ends_the_tunnel(): + donor = Donor(TunnelBridge(), garbage=True) + _, echoed, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1") + assert relay.error == "bad_base64" and echoed == b"" + + +def test_releasing_a_session_closes_its_relays(): + bridge = TunnelBridge() + donor = Donor(bridge) + relay = Relay(donor, TARGET) + donor.tunnels[relay.conn_id] = relay + bridge.release_session(donor) + assert donor.tunnels == {} and relay.frames.get_nowait() is tunnel_module.CLOSED + + +def test_frames_for_unknown_connections_are_not_consumed(): + bridge = TunnelBridge() + assert bridge.on_frame(Donor(bridge), {"type": "tunnel_data", "conn_id": "nope", "b64": ""}) is False diff --git a/tests/unit/services/sync/__init__.py b/tests/unit/services/sync/__init__.py new file mode 100644 index 00000000..95ee7c3e --- /dev/null +++ b/tests/unit/services/sync/__init__.py @@ -0,0 +1 @@ +# retoor diff --git a/tests/unit/services/sync/alias.py b/tests/unit/services/sync/alias.py new file mode 100644 index 00000000..59339d6d --- /dev/null +++ b/tests/unit/services/sync/alias.py @@ -0,0 +1,273 @@ +# retoor + +import random +from collections import Counter + +import pytest + +from devplacepy.database import get_table, init_db +from devplacepy.services.sync import ai +from devplacepy.services.sync.aliaser import AliasRefused, Aliaser +from devplacepy.services.sync.identity import IdentityResolver, NameGenerator, fallback_name, source_key +from devplacepy.services.sync.options import SyncOptions +from devplacepy.services.sync.rewriter import MentionRewriter +from devplacepy.services.sync.selection import Selector, bucket, jittered +from devplacepy.services.sync.style import Style, plausible +from devplacepy.services.sync.translator import Translator +from tests.conftest import run_async + +ORIGIN = "alias.test" + + +@pytest.fixture(autouse=True) +def clean(monkeypatch): + init_db() + for table in ("sync_identities", "sync_links"): + get_table(table).delete(origin=ORIGIN) + + async def unreachable(system, text, timeout): + return None + + monkeypatch.setattr(ai, "complete", unreachable) + yield + for table in ("sync_identities", "sync_links"): + get_table(table).delete(origin=ORIGIN) + + +def test_rewriter_longest_first_and_single_pass(): + rewriter = MentionRewriter({"alice": "bob", "alice_w": "carol", "bob": "dave"}) + result = rewriter.rewrite("hi @alice_w and alice, also @bob") + assert result.text == "hi @carol and bob, also @dave" + assert len(result.substitutions) == 3 + + +def test_rewriter_protects_code_urls_and_inline_code(): + rewriter = MentionRewriter({"alice": "zed"}) + text = "see `alice` and https://x.test/~alice\n```\nalice = 1\n```\nthen alice" + assert rewriter.rewrite(text).text == text.rsplit("alice", 1)[0] + "zed" + assert rewriter.rewrite("```\nalice unterminated").text == "```\nalice unterminated" + + +def test_rewriter_standalone_floor_and_case_and_boundaries(): + rewriter = MentionRewriter({"ann": "x1", "tef": "y2", "markus": "zeta"}) + assert rewriter.rewrite("ann do, @ann, @tef, tef, Markus, markusx, a-markus").text == "ann do, @x1, @y2, tef, zeta, markusx, a-markus" + + +def test_rewriter_mentioned_skips_code(): + found = MentionRewriter({}).mentioned("hi @Alice `@code` @ab ```\n@fenced\n``` @alice_2") + assert found == ["Alice", "alice_2"] + + +def test_style_roundtrip_and_matching(): + style = Style.of("Alice_99") + assert Style.load(style.dump()) == style + assert style.matches("Marta_12") and not style.matches("marta_12") and not style.matches("Marta12") + assert not plausible("alice_99x", "Alice_99", None) + assert not plausible("robotuser", "zzzz", None) + assert not plausible("a!", "zzzz", None) + + +def test_fallback_name_is_deterministic_and_style_matched(): + for source in ("alice", "Bob_Smith", "dev99", "UPPER", "kebab-case"): + style = Style.of(source) + assert fallback_name(ORIGIN, source, style, 0) == fallback_name(ORIGIN, source, style, 0) + assert any(plausible(fallback_name(ORIGIN, source, style, n), source, style) for n in range(60)) + + +def test_generator_rejects_bad_ai_then_falls_back(monkeypatch): + replies = iter(['{"username": "alice"}', "garbage", '{"username": "x"}']) + + async def scripted(system, text, timeout): + return next(replies) + + monkeypatch.setattr(ai, "complete", scripted) + name, by_ai = run_async(NameGenerator().generate(ORIGIN, "alice", Style.of("alice"), [])) + assert not by_ai and plausible(name, "alice", Style.of("alice")) + + +def test_generator_accepts_good_ai(monkeypatch): + async def good(system, text, timeout): + assert "alice" not in text + return '```json\n{"username": "marta"}\n```' + + monkeypatch.setattr(ai, "complete", good) + assert run_async(NameGenerator().generate(ORIGIN, "alice", Style.of("alice"), [])) == ("marta", True) + + +def test_identity_is_permanent_unique_and_bidirectional(): + first = IdentityResolver(ORIGIN) + names = [run_async(first.ensure(n)) for n in ("alice", "Alice", "bobby", "carol")] + assert names[0] == names[1] + assert len({name.lower() for name in names[:1] + names[2:]}) == 3 + second = IdentityResolver(ORIGIN) + assert run_async(second.ensure("ALICE")) == names[0] + assert second.created == 0 + assert second.mapping()["alice"] == names[0] + rewriter = MentionRewriter({value: key for key, value in second.mapping().items()}) + assert rewriter.rewrite(f"@{names[0]}").text == "@alice" + + +def test_identity_cap_and_invalid_names_fail_closed(): + resolver = IdentityResolver(ORIGIN, cap=1) + assert run_async(resolver.ensure("alice")) + assert run_async(resolver.ensure("bobby")) is None + assert run_async(resolver.ensure("not valid!")) is None + assert source_key("x" * 100) is None + + +def test_dry_run_resolver_persists_nothing(): + resolver = IdentityResolver(ORIGIN, persist=False) + assert run_async(resolver.ensure("alice")) + assert get_table("sync_identities").count(origin=ORIGIN) == 0 + + +def test_samples_are_capped(): + resolver = IdentityResolver(ORIGIN) + run_async(resolver.ensure("alice")) + for n in range(6): + resolver.remember("alice", f"sample number {n}") + assert len(resolver.samples("alice")) == 3 + assert len(IdentityResolver(ORIGIN).samples("alice")) == 3 + + +def records(): + users = [ + {"origin_uid": "1", "username": "alice", "bio": "secret bio", "website": "https://alice.test", "created_at": "2025-01-01T00:00:00Z"}, + {"origin_uid": "2", "username": "bobby", "created_at": "2025-01-02T00:00:00Z"}, + ] + posts = [ + {"origin_uid": "p1", "author_uid": "1", "title": "Hello from alice", "content": "ping @bobby and @ghostly please", "created_at": "2025-02-01T00:00:00Z", "topic": "devlog"}, + {"origin_uid": "p2", "author_uid": "9", "title": "orphan", "content": "no known author here", "created_at": "2025-02-02T00:00:00Z"}, + ] + comments = [ + {"origin_uid": "c1", "author_uid": "2", "post_uid": "p1", "content": "thanks alice", "created_at": "2025-02-03T00:00:00Z"}, + {"origin_uid": "c2", "author_uid": "2", "post_uid": "p1", "parent_uid": "cX", "content": "reply to missing", "created_at": "2025-02-03T00:00:00Z"}, + ] + return {"users": users, "posts": posts, "comments": comments} + + +def leaked(outcome, names=("alice", "bobby")): + blob = " ".join( + str(item.get(name) or "") + for kind in ("users", "posts", "comments") + for item in outcome.records[kind] + for name in ("username", "title", "content", "bio", "website") + ).lower() + return [name for name in names if name in blob] + + +def test_aliaser_rewrites_everything_and_fails_closed(): + outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records())) + assert leaked(outcome) == [] + assert len(outcome.records["posts"]) == 1 + assert outcome.skipped["no_author"] == 1 + assert [c["origin_uid"] for c in outcome.records["comments"]] == ["c1"] + assert outcome.skipped["parent_skipped"] == 1 + users = outcome.records["users"] + assert len(users) == 3 + assert all(set(user) == {"origin_uid", "username", "created_at", "deleted"} for user in users) + assert {user["origin_uid"] for user in users} == {"alice", "bobby", "ghostly"} + assert outcome.records["posts"][0]["author_uid"] == "alice" + assert outcome.substitutions >= 4 + + +def test_aliaser_is_stable_across_runs(): + first = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records())) + second = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records())) + assert first.records == second.records + assert second.identities_created == 0 + + +def test_aliaser_refuses_origin_with_original_links(): + get_table("sync_links").insert({"uid": "l1", "origin": ORIGIN, "kind": "users", "origin_uid": "1", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""}) + with pytest.raises(AliasRefused): + run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records())) + + +def test_aliaser_cap_skips_item_as_transient(monkeypatch): + resolver = IdentityResolver(ORIGIN, cap=1) + outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True), resolver=resolver).apply(records())) + assert outcome.transient() >= 1 + assert leaked(outcome) == [] + + +class FakeTranslator(Translator): + def __init__(self, reply): + super().__init__("nl") + self.reply = reply + + async def translate(self, text, samples=None): + return self.reply(text) if self.reply else None + + +def test_translation_failure_skips_item_and_holds(): + translator = FakeTranslator(None) + outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True, translate_to="nl"), language="en", translator=translator).apply(records())) + assert outcome.records["posts"] == [] and outcome.skipped["translation_failed"] == 1 + assert outcome.transient() >= 1 + + +def test_translation_output_is_rewritten_again(): + translator = FakeTranslator(lambda text: text + " (met @alice)") + outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True, translate_to="nl"), language="en", translator=translator).apply(records())) + assert leaked(outcome) == [] + assert outcome.translated >= 1 + + +def test_translation_skipped_when_language_matches(): + assert not Translator("en").needed("en-US") + assert Translator("en").needed("") and Translator("nl").needed("en") + + +def test_options_validation_and_roundtrip(): + options = SyncOptions.build(alias=True, translate_to="NL", sample_percent=40, min_comments=2, topics="devlog, rant", jitter_percent=10) + assert SyncOptions.from_row({"sync_options": options.to_dict()}) == options + assert not SyncOptions().active() + for bad in ({"sample_percent": 0}, {"jitter_percent": 99}, {"topics": "nonsense"}, {"translate_to": "x y"}, {"min_comments": -1}): + with pytest.raises(ValueError): + SyncOptions.build(**bad) + assert options.merged(alias=False).alias is False and options.merged(alias=None).alias is True + + +def test_selector_deterministic_sampling_and_filters(): + posts = [{"origin_uid": f"p{n}", "author_uid": "1", "topic": "devlog"} for n in range(400)] + comments = [{"origin_uid": "c", "author_uid": "1", "post_uid": "p0"}] + data = {"users": [], "posts": posts, "comments": comments} + options = SyncOptions(sample_percent=50) + kept, skipped = Selector(ORIGIN, options).select(data) + again, _ = Selector(ORIGIN, options).select(data) + assert kept["posts"] == again["posts"] + assert 150 < len(kept["posts"]) < 250 + assert skipped["sampled_out"] == 400 - len(kept["posts"]) + assert bucket(ORIGIN, "p0") == bucket(ORIGIN, "p0") + kept, skipped = Selector(ORIGIN, SyncOptions(topics=("rant",))).select(data) + assert kept["posts"] == [] and kept["comments"] == [] and skipped["topic_not_allowed"] == 400 + kept, skipped = Selector(ORIGIN, SyncOptions(min_comments=1)).select(data) + assert [post["origin_uid"] for post in kept["posts"]] == ["p0"] + untouched, none = Selector(ORIGIN, SyncOptions()).select(data) + assert untouched is data and not none + + +def test_selector_never_drops_already_synced_posts(): + get_table("sync_links").insert({"uid": "l2", "origin": ORIGIN, "kind": "posts", "origin_uid": "p0", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""}) + data = {"users": [], "posts": [{"origin_uid": "p0", "author_uid": "1", "topic": "devlog"}], "comments": []} + kept, _ = Selector(ORIGIN, SyncOptions(topics=("rant",))).select(data) + assert len(kept["posts"]) == 1 + + +def test_jitter_is_bounded(): + rng = random.Random(1) + values = [jittered(1000, 20, rng) for _ in range(500)] + assert min(values) >= 800 and max(values) <= 1200 and len(set(values)) > 50 + assert jittered(1000, 0) == 1000 + assert Counter() == Counter() + + +def test_mentions_are_case_insensitive_end_to_end(): + rewriter = MentionRewriter({"Alice": "zed"}) + assert rewriter.rewrite("@ALICE @alice @AlIcE and ALICE").text == "@zed @zed @zed and zed" + data = records() + data["posts"][0]["content"] = "ping @BOBBY and @Bobby and @bObBy" + outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(data)) + assert leaked(outcome) == [] + assert len(outcome.records["users"]) == 2 + 0 or {u["origin_uid"] for u in outcome.records["users"]} >= {"alice", "bobby"} diff --git a/tests/unit/services/sync/normalize.py b/tests/unit/services/sync/normalize.py new file mode 100644 index 00000000..215d32fd --- /dev/null +++ b/tests/unit/services/sync/normalize.py @@ -0,0 +1,97 @@ +# retoor + +from datetime import datetime, timedelta, timezone + +import pytest + +from devplacepy.services.sync.normalize import ( + clamp_instant, + clean_text, + fingerprint, + instant_iso, + normalize_origin, + normalize_origin_uid, + normalize_username, + parse_instant, + username_candidates, +) +from devplacepy.services.sync.spec import LIMITS + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("Other.Example.ORG", "other.example.org"), + (" a1 ", "a1"), + ("devplace-net", "devplace-net"), + ], +) +def test_origin_is_normalized(raw, expected): + assert normalize_origin(raw) == expected + + +@pytest.mark.parametrize("raw", ["", "a", "-bad", "bad-", "a..b", "has space", "x" * 64, "ünï.org"]) +def test_origin_is_rejected(raw): + with pytest.raises(ValueError): + normalize_origin(raw) + + +@pytest.mark.parametrize("raw", ["", " ", "a b", "x" * 65, "a/b"]) +def test_origin_uid_is_rejected(raw): + with pytest.raises(ValueError): + normalize_origin_uid(raw) + + +def test_origin_uid_accepts_integers_and_uuids(): + assert normalize_origin_uid(42) == "42" + assert normalize_origin_uid("01a139e9-a2f0-7fa1-8635-1a98fbf95e3d") + + +def test_clean_text_strips_control_characters_and_newlines(): + assert clean_text("a\x00b\r\nc\rd\x07", multiline=True) == "ab\nc\nd" + assert clean_text("a \n b\t c", multiline=False) == "a b c" + + +def test_username_always_valid_across_inputs(): + samples = ["", "a", "ab", "Ünï cøde!", "x" * 80, "--__--", "john.doe@mail", "日本語", "ok_name-1"] + for sample in samples: + name = normalize_username(sample, "uid-1") + assert LIMITS.min_username_length <= len(name) <= LIMITS.max_username_length + assert all(ch.isalnum() or ch in "_-" for ch in name) and name.isascii() + + +def test_username_is_stable_for_same_input(): + assert normalize_username("a", "u1") == normalize_username("a", "u1") + assert normalize_username("a", "u1") != normalize_username("a", "u2") + + +def test_username_candidates_are_distinct_and_bounded(): + seen = [] + for candidate in username_candidates("x" * 32, "other.example.org"): + seen.append(candidate) + assert len(candidate) <= LIMITS.max_username_length + if len(seen) == 50: + break + assert len(set(seen)) == 50 + assert seen[1].endswith("-other") + + +def test_instants_are_utc_iso_and_clamped(): + assert instant_iso("2025-01-01T01:00:00+01:00") == "2025-01-01T00:00:00+00:00" + assert instant_iso("2025-01-01").startswith("2025-01-01T00:00:00") + assert parse_instant(1700000000).year == 2023 + now = datetime.now(timezone.utc) + future = now + timedelta(days=30) + assert clamp_instant(future, now) == now + assert instant_iso("1970-01-01").startswith("2000-01-01") + + +@pytest.mark.parametrize("raw", ["", None, "nope", True, "2025-13-45"]) +def test_instant_rejects_garbage(raw): + with pytest.raises(ValueError): + parse_instant(raw) + + +def test_fingerprint_ignores_key_order_and_detects_change(): + assert fingerprint({"a": 1, "b": 2}) == fingerprint({"b": 2, "a": 1}) + assert fingerprint({"a": 1}) != fingerprint({"a": 2}) diff --git a/tests/unit/services/sync/records.py b/tests/unit/services/sync/records.py new file mode 100644 index 00000000..d88e0139 --- /dev/null +++ b/tests/unit/services/sync/records.py @@ -0,0 +1,85 @@ +# retoor + +import pytest +from pydantic import ValidationError + +from devplacepy.services.sync.records import ( + CommentRecord, + PostRecord, + SyncBatchForm, + UserRecord, +) +from devplacepy.services.sync.spec import LIMITS + +BASE = {"origin_uid": "r1", "created_at": "2025-01-01T00:00:00Z"} + + +def test_updated_at_defaults_to_created_and_never_precedes_it(): + record = PostRecord.model_validate( + {**BASE, "author_uid": "u", "content": "ten chars!!"} + ) + assert record.updated_at == record.created_at + earlier = PostRecord.model_validate( + {**BASE, "author_uid": "u", "content": "ten chars!!", "updated_at": "2024-01-01"} + ) + assert earlier.updated_at == earlier.created_at + + +def test_unknown_topic_becomes_random_and_known_is_kept(): + base = {**BASE, "author_uid": "u", "content": "ten chars!!"} + assert PostRecord.model_validate({**base, "topic": "zzz"}).topic == "random" + assert PostRecord.model_validate({**base, "topic": "DevLog"}).topic == "devlog" + + +@pytest.mark.parametrize( + "payload", + [ + {"author_uid": "u", "content": "short"}, + {"author_uid": "u", "content": "x" * (LIMITS.max_post_length + 1)}, + {"author_uid": "", "content": "ten chars!!"}, + {"content": "ten chars!!"}, + {"author_uid": "u", "content": "ten chars!!", "title": "t" * (LIMITS.max_title_length + 1)}, + ], +) +def test_invalid_posts_are_rejected(payload): + with pytest.raises(ValidationError): + PostRecord.model_validate({**BASE, **payload}) + + +def test_comment_parent_is_optional_and_post_is_required(): + base = {**BASE, "author_uid": "u", "content": "fine"} + assert CommentRecord.model_validate({**base, "post_uid": "p"}).parent_uid is None + assert CommentRecord.model_validate({**base, "post_uid": "p", "parent_uid": ""}).parent_uid is None + with pytest.raises(ValidationError): + CommentRecord.model_validate(base) + + +def test_user_urls_are_fixed_or_dropped(): + record = UserRecord.model_validate( + {**BASE, "username": "bob", "website": "example.com", "git_link": "not a url"} + ) + assert record.website == "https://example.com" + assert record.git_link == "" + + +def test_user_bio_over_limit_is_rejected(): + with pytest.raises(ValidationError): + UserRecord.model_validate({**BASE, "username": "bob", "bio": "x" * (LIMITS.max_bio_length + 1)}) + + +def test_user_counters_and_credentials_are_not_fields(): + known = UserRecord.known_fields() + for name in ("email", "password_hash", "role", "xp", "level", "stars", "api_key"): + assert name not in known + assert "stars" not in PostRecord.known_fields() + + +def test_envelope_normalizes_origin_and_bounds_batch(): + form = SyncBatchForm(origin="Other.ORG", records=[{}]) + assert form.origin == "other.org" + with pytest.raises(ValidationError): + SyncBatchForm(origin="other.org", records=[]) + with pytest.raises(ValidationError): + SyncBatchForm(origin="other.org", records=[{}] * (LIMITS.max_batch_records + 1)) + with pytest.raises(ValidationError): + SyncBatchForm(origin="bad origin", records=[{}])