From 02ab747cd0853a64568bac1c07669a5009fda1d7 Mon Sep 17 00:00:00 2001 From: retoor Date: Sat, 17 Oct 2026 02:23:43 +0200 Subject: [PATCH] Code Farm: per-plot auto-replant pause and crop-lock check; community sync and node work Continuous Delivery could never be turned off once a crop was planted. Add a one-shot per-plot "stop after harvest" flag (POST /game/plot/stop, Devii tool game_plot_stop, game_plots.stop_after_harvest), cleared by the next plant, autoreplant claim or Refactor. Autoreplant now also respects crop locks through the shared _crop_lock predicate. Plan and measurements in happyplanting.md. Also includes the pending community sync, node network and related work in the tree; the full suite passed (4999) with all of it. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 8 +- devplacepy/attachments.py | 4 +- devplacepy/cli/community.py | 6 +- devplacepy/cli/nodes.py | 32 ++ devplacepy/config.py | 3 + devplacepy/database/core.py | 3 + devplacepy/database/nodes.py | 185 +++++- devplacepy/database/schema.py | 97 ++++ devplacepy/database/sync.py | 229 ++++++++ devplacepy/docs_api/groups/community.py | 37 ++ devplacepy/docs_api/groups/game.py | 13 + devplacepy/docs_api/groups/nodes.py | 16 + devplacepy/docs_api/groups/sync.py | 191 +++++- devplacepy/models.py | 20 + devplacepy/routers/CLAUDE.md | 2 +- devplacepy/routers/admin/communities.py | 123 +++- devplacepy/routers/admin/nodes.py | 95 ++- devplacepy/routers/game/index.py | 22 + devplacepy/routers/nodes/ticket.py | 7 +- devplacepy/routers/nodes/ws.py | 17 +- devplacepy/routers/sync/__init__.py | 25 +- devplacepy/routers/sync/attachments.py | 21 + devplacepy/routers/sync/deletions.py | 65 +++ devplacepy/routers/sync/export.py | 52 ++ devplacepy/routers/sync/follows.py | 21 + devplacepy/routers/sync/poll_votes.py | 21 + devplacepy/routers/sync/polls.py | 21 + devplacepy/routers/sync/projects.py | 21 + devplacepy/routers/sync/reactions.py | 21 + devplacepy/routers/sync/votes.py | 21 + devplacepy/schemas/__init__.py | 3 + devplacepy/schemas/admin.py | 34 ++ devplacepy/schemas/game.py | 2 + devplacepy/schemas/sync.py | 31 + devplacepy/services/community/CLAUDE.md | 24 +- devplacepy/services/community/checkpoint.py | 51 ++ devplacepy/services/community/discoverer.py | 68 ++- devplacepy/services/community/extract.py | 229 ++++++++ devplacepy/services/community/feeds.py | 16 +- devplacepy/services/community/fetcher.py | 22 +- devplacepy/services/community/files.py | 4 + devplacepy/services/community/generator.py | 30 +- devplacepy/services/community/learner.py | 67 ++- devplacepy/services/community/ledger.py | 36 +- devplacepy/services/community/ledger_store.py | 29 +- devplacepy/services/community/pusher.py | 26 +- devplacepy/services/community/runner.py | 108 +++- devplacepy/services/community/runtime.py | 269 ++++++++- devplacepy/services/community/schemacheck.py | 254 ++++++++ devplacepy/services/community/service.py | 3 + devplacepy/services/community/spec.py | 9 +- devplacepy/services/community/store.py | 23 +- .../devii/actions/catalog/community.py | 29 + .../services/devii/actions/catalog/game.py | 17 + .../services/devii/actions/catalog/nodes.py | 18 + .../services/devii/actions/catalog/sync.py | 83 ++- .../services/devii/actions/dispatcher.py | 15 + devplacepy/services/game/CLAUDE.md | 11 +- devplacepy/services/game/store/__init__.py | 1 + devplacepy/services/game/store/actions.py | 40 +- devplacepy/services/game/store/serialize.py | 5 + devplacepy/services/jobs/deepsearch/crawl.py | 60 +- devplacepy/services/jobs/deepsearch/worker.py | 2 + devplacepy/services/nodes/CLAUDE.md | 29 +- devplacepy/services/nodes/addressclass.py | 66 +++ devplacepy/services/nodes/broker.py | 66 ++- devplacepy/services/nodes/cache.py | 70 ++- devplacepy/services/nodes/circuit.py | 48 +- devplacepy/services/nodes/dispatcher.py | 105 +++- devplacepy/services/nodes/fabric.py | 5 +- devplacepy/services/nodes/geo.py | 293 ++++++++++ devplacepy/services/nodes/registry.py | 41 +- devplacepy/services/nodes/scheduler.py | 31 +- devplacepy/services/nodes/service.py | 49 +- devplacepy/services/nodes/session.py | 13 +- devplacepy/services/nodes/shape.py | 70 +++ devplacepy/services/nodes/spec.py | 9 +- devplacepy/services/nodes/wake.py | 70 +++ devplacepy/services/skills/CLAUDE.md | 2 +- devplacepy/services/sync/CLAUDE.md | 33 +- devplacepy/services/sync/batch.py | 8 +- devplacepy/services/sync/deletions.py | 73 +++ devplacepy/services/sync/exporter.py | 287 +++++++++ devplacepy/services/sync/handles.py | 109 ++++ devplacepy/services/sync/healing.py | 68 +++ devplacepy/services/sync/options.py | 6 +- devplacepy/services/sync/pending.py | 51 ++ devplacepy/services/sync/processors.py | 543 +++++++++++++++++- devplacepy/services/sync/records.py | 284 ++++++++- devplacepy/services/sync/selection.py | 68 ++- devplacepy/services/sync/spec.py | 116 +++- devplacepy/static/css/admin.css | 6 +- devplacepy/static/css/communities.css | 20 + devplacepy/static/js/AdminCommunity.js | 39 ++ devplacepy/static/js/AdminNodes.js | 14 + devplacepy/static/js/Application.js | 2 + devplacepy/static/js/GameFarm.js | 9 + devplacepy/templates/_game_grid.html | 6 + devplacepy/templates/_game_plot_stop.html | 5 + devplacepy/templates/admin_communities.html | 3 + devplacepy/templates/admin_community.html | 20 +- .../templates/admin_community_ledger.html | 12 +- .../templates/admin_community_schema.html | 33 ++ devplacepy/templates/admin_nodes.html | 79 ++- devplacepy/templates/docs/code-farm.html | 8 +- devplacepy/templates/docs/privacy.html | 4 +- events.md | 17 +- gaplord.md | 405 +++++++++++++ happyplanting.md | 124 ++++ locustfile.py | 8 + nicknames.md | 101 ++++ nodes/client/README.md | 4 +- nodes/client/battery.go | 53 ++ nodes/client/battery_darwin.go | 19 + nodes/client/battery_linux.go | 53 ++ nodes/client/battery_linux_test.go | 60 ++ nodes/client/battery_other.go | 9 + nodes/client/battery_windows.go | 28 + nodes/client/limits.go | 13 +- nodes/client/limits_test.go | 91 +++ nodes/client/main.go | 1 + nodes/client/protocol.go | 11 +- nodes/client/session.go | 52 +- nodes/client/session_test.go | 117 +++- nodes/extension/README.md | 18 +- nodes/extension/build.sh | 4 +- nodes/extension/icons/icon-128.png | Bin 0 -> 6715 bytes nodes/extension/icons/icon-16.png | Bin 0 -> 720 bytes nodes/extension/icons/icon-32.png | Bin 0 -> 1548 bytes nodes/extension/icons/icon-48.png | Bin 0 -> 2202 bytes nodes/extension/manifest.json | 15 +- nodes/extension/src/Limits.js | 38 +- nodes/extension/src/State.js | 3 +- nodes/extension/src/WsClient.js | 58 +- nodes/extension/src/spec.js | 9 + nodes/extension/test/limits.test.mjs | 67 +++ nodes/extension/test/package.test.mjs | 29 + nodes/extension/test/popup.test.mjs | 20 + nodes/extension/test/wsclient.test.mjs | 109 ++++ nodes/extension/tools/make_icons.py | 43 ++ nodes/extension/ui/popup.css | 3 +- nodes/extension/ui/popup.js | 33 +- pyproject.toml | 3 +- tests/api/admin/communities/index.py | 21 + tests/api/admin/communities/schema.py | 97 ++++ tests/api/admin/nodes/index.py | 25 + tests/api/admin/nodes/override.py | 53 ++ tests/api/game/mutations.py | 59 ++ tests/api/nodes/jobs.py | 4 +- tests/api/nodes/ticket.py | 1 + tests/api/nodes/ws.py | 42 ++ tests/api/sync/attachments.py | 80 +++ tests/api/sync/comments.py | 3 +- tests/api/sync/conftest.py | 93 +++ tests/api/sync/deletions.py | 109 ++++ tests/api/sync/export.py | 203 +++++++ tests/api/sync/follows.py | 31 + tests/api/sync/pending.py | 120 ++++ tests/api/sync/poll_votes.py | 41 ++ tests/api/sync/polls.py | 75 +++ tests/api/sync/posts.py | 6 +- tests/api/sync/projects.py | 64 +++ tests/api/sync/reactions.py | 33 ++ tests/api/sync/votes.py | 82 +++ tests/e2e/admin/communities/__init__.py | 0 tests/e2e/admin/communities/live.py | 81 +++ tests/e2e/game/index.py | 28 + tests/e2e/nodes/__init__.py | 0 tests/e2e/nodes/extension.py | 168 ++++++ tests/e2e/nodes/node_app.py | 38 ++ tests/unit/database/nodes.py | 62 ++ tests/unit/services/community/alias.py | 5 +- tests/unit/services/community/checkpoint.py | 152 +++++ tests/unit/services/community/dedupe.py | 92 +++ tests/unit/services/community/discoverer.py | 41 ++ tests/unit/services/community/engagement.py | 199 +++++++ tests/unit/services/community/fakes.py | 25 +- tests/unit/services/community/ledger.py | 4 +- tests/unit/services/community/login.py | 238 ++++++++ tests/unit/services/community/pagination.py | 176 ++++++ tests/unit/services/community/schemacheck.py | 89 +++ tests/unit/services/community/structured.py | 100 ++++ tests/unit/services/community/submitter.py | 60 ++ tests/unit/services/devii/actions/catalog.py | 1 + tests/unit/services/game/store.py | 108 ++++ tests/unit/services/jobs/deepsearch/crawl.py | 86 +++ tests/unit/services/jobs/deepsearch/worker.py | 4 +- tests/unit/services/nodes/addressclass.py | 56 ++ tests/unit/services/nodes/cache.py | 70 +++ tests/unit/services/nodes/circuit.py | 54 ++ tests/unit/services/nodes/dispatcher.py | 218 ++++++- tests/unit/services/nodes/fabric.py | 2 + tests/unit/services/nodes/geo.py | 126 ++++ tests/unit/services/nodes/registry.py | 55 ++ tests/unit/services/nodes/scheduler.py | 47 +- tests/unit/services/nodes/service.py | 75 +++ tests/unit/services/nodes/shape.py | 70 +++ tests/unit/services/nodes/wake.py | 90 +++ tests/unit/services/skills/fetcher.py | 132 +++++ tests/unit/services/sync/handles.py | 134 +++++ tests/unit/services/sync/healing.py | 99 ++++ tests/unit/services/sync/records.py | 64 +++ tests/unit/services/sync/selection.py | 125 ++++ 203 files changed, 11624 insertions(+), 301 deletions(-) create mode 100644 devplacepy/routers/sync/attachments.py create mode 100644 devplacepy/routers/sync/deletions.py create mode 100644 devplacepy/routers/sync/export.py create mode 100644 devplacepy/routers/sync/follows.py create mode 100644 devplacepy/routers/sync/poll_votes.py create mode 100644 devplacepy/routers/sync/polls.py create mode 100644 devplacepy/routers/sync/projects.py create mode 100644 devplacepy/routers/sync/reactions.py create mode 100644 devplacepy/routers/sync/votes.py create mode 100644 devplacepy/services/community/checkpoint.py create mode 100644 devplacepy/services/community/schemacheck.py create mode 100644 devplacepy/services/nodes/addressclass.py create mode 100644 devplacepy/services/nodes/geo.py create mode 100644 devplacepy/services/nodes/shape.py create mode 100644 devplacepy/services/nodes/wake.py create mode 100644 devplacepy/services/sync/deletions.py create mode 100644 devplacepy/services/sync/exporter.py create mode 100644 devplacepy/services/sync/handles.py create mode 100644 devplacepy/services/sync/healing.py create mode 100644 devplacepy/services/sync/pending.py create mode 100644 devplacepy/static/js/AdminCommunity.js create mode 100644 devplacepy/templates/_game_plot_stop.html create mode 100644 devplacepy/templates/admin_community_schema.html create mode 100644 gaplord.md create mode 100644 happyplanting.md create mode 100644 nicknames.md create mode 100644 nodes/client/battery.go create mode 100644 nodes/client/battery_darwin.go create mode 100644 nodes/client/battery_linux.go create mode 100644 nodes/client/battery_linux_test.go create mode 100644 nodes/client/battery_other.go create mode 100644 nodes/client/battery_windows.go create mode 100644 nodes/extension/icons/icon-128.png create mode 100644 nodes/extension/icons/icon-16.png create mode 100644 nodes/extension/icons/icon-32.png create mode 100644 nodes/extension/icons/icon-48.png create mode 100644 nodes/extension/test/package.test.mjs create mode 100644 nodes/extension/test/popup.test.mjs create mode 100644 nodes/extension/tools/make_icons.py create mode 100644 tests/api/admin/communities/schema.py create mode 100644 tests/api/admin/nodes/override.py create mode 100644 tests/api/sync/attachments.py create mode 100644 tests/api/sync/deletions.py create mode 100644 tests/api/sync/export.py create mode 100644 tests/api/sync/follows.py create mode 100644 tests/api/sync/pending.py create mode 100644 tests/api/sync/poll_votes.py create mode 100644 tests/api/sync/polls.py create mode 100644 tests/api/sync/projects.py create mode 100644 tests/api/sync/reactions.py create mode 100644 tests/api/sync/votes.py create mode 100644 tests/e2e/admin/communities/__init__.py create mode 100644 tests/e2e/admin/communities/live.py create mode 100644 tests/e2e/nodes/__init__.py create mode 100644 tests/e2e/nodes/extension.py create mode 100644 tests/e2e/nodes/node_app.py create mode 100644 tests/unit/services/community/checkpoint.py create mode 100644 tests/unit/services/community/dedupe.py create mode 100644 tests/unit/services/community/engagement.py create mode 100644 tests/unit/services/community/login.py create mode 100644 tests/unit/services/community/pagination.py create mode 100644 tests/unit/services/community/schemacheck.py create mode 100644 tests/unit/services/community/structured.py create mode 100644 tests/unit/services/community/submitter.py create mode 100644 tests/unit/services/nodes/addressclass.py create mode 100644 tests/unit/services/nodes/geo.py create mode 100644 tests/unit/services/nodes/registry.py create mode 100644 tests/unit/services/nodes/shape.py create mode 100644 tests/unit/services/nodes/wake.py create mode 100644 tests/unit/services/sync/handles.py create mode 100644 tests/unit/services/sync/healing.py create mode 100644 tests/unit/services/sync/selection.py diff --git a/README.md b/README.md index 5c40050d..4a3fd870 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ devplacepy/ | `/topics` | Crawlable per-topic category pages (public): `/topics` hub links every topic with a live post count, `/topics/{topic}` lists that topic's posts with its own canonical URL, title, and breadcrumbs | | `/skills` | Per-user Agent Skills: list, detail, install from a URL, enable/disable, delete. Private to the owner, `noindex` | | `/nodes` | Community node network: `GET /nodes/spec.json`, `GET/POST /nodes/devices` (+ `/rotate`, `/delete`), `POST /nodes/ticket`, `WS /nodes/ws`. Off by default; also `GET /nodes/me`, `POST /nodes/settings`, `GET /nodes/social/search`, `POST /nodes/internal/fetch`, `GET /nodes/stats`, `GET /nodes/leaderboard`, `GET /nodes/client-version`, `GET /nodes/download/{artifact}` (see `plan.md`) | -| `/sync` | Community sync: `GET /sync/spec.json`, `POST /sync/{users,posts,comments}`. Administrator only, off by default (`sync_enabled`), idempotent batch upserts keyed on the origin community's ids | +| `/sync` | Community sync: `GET /sync/spec.json`, `POST /sync/{users,projects,posts,comments,polls,votes,reactions,follows,poll-votes,attachments}`, `GET /sync/export/{kind}`, `GET /sync/deletions`, `POST /sync/deletions/ack`. Administrator only, off by default (`sync_enabled`), idempotent batch upserts keyed on the origin community's ids | | `/news` | Developer news listing, detail page with comments | | `/posts` | Post detail, creation | | `/gists` | Code gist listing, detail, creation, and editing; left panel offers language filtering and free-text `search` (title, description, and author username), public read. A plain Markdown gist has a **View rendered / View raw** toggle beside its Copy button, switching between the raw source and the same rendered view a `Markdown Rendered` gist always shows | @@ -220,7 +220,7 @@ The **Code Farm** (`/game`) is a cooperative idle game in the spirit of Farmvill - **Leaderboards.** Several boards, selectable from the game page: **Overall score** (a composite weighing refactor/prestige count, XP, lifetime harvests, a capped coin contribution, CI tier, plots, perks, and streak - the cap keeps it a measure of what you built rather than what you hoard), **Prestige**, **Harvests this week**, **Raid efficiency** (average coins per successful raid), **Fastest to Kernel** (time since your last refactor), **Fair play** (rewards recent activity over hoarding), and (when running) the current **Era** board. - **Eras (admin-managed seasons).** Administrators can start an Era at `/admin/game`: every farm's *visible* Era coins/harvests counters reset to zero, but real coin balances, prestige, Stars, Legacy, and Mastery are never touched. Ending an Era ranks farms by Era score (which gives prestige only partial weight, so veterans keep an edge without it being insurmountable), awards Stars to the top 10, and permanently records the results. -The farm refreshes live over the pub/sub bus (a watered build appears on the owner's screen at once) and every plot countdown ticks client-side. Every endpoint also answers JSON, and Devii can play the game on the member's behalf via the `game_*` tools (`game_state`, `game_plant`, `game_harvest`, `game_buy_plot`, `game_upgrade_ci`, `game_fertilize`, `game_daily`, `game_claim_quest`, `game_water`, `game_steal`, `game_view_farm`, `game_leaderboard`, `game_upgrade_perk`, `game_upgrade_legacy`, `game_prestige`, `game_claim_grant`, `game_upgrade_mastery`, `game_buy_infrastructure`, `game_upgrade_defense`, `game_buy_cosmetic`, `game_equip_cosmetic`, `game_downgrade_defense`). See the API reference group **Code Farm** and the full player guide at `/docs/code-farm.html`. +The farm refreshes live over the pub/sub bus (a watered build appears on the owner's screen at once) and every plot countdown ticks client-side. Every endpoint also answers JSON, and Devii can play the game on the member's behalf via the `game_*` tools (`game_state`, `game_plant`, `game_harvest`, `game_plot_stop`, `game_buy_plot`, `game_upgrade_ci`, `game_fertilize`, `game_daily`, `game_claim_quest`, `game_water`, `game_steal`, `game_view_farm`, `game_leaderboard`, `game_upgrade_perk`, `game_upgrade_legacy`, `game_prestige`, `game_claim_grant`, `game_upgrade_mastery`, `game_buy_infrastructure`, `game_upgrade_defense`, `game_buy_cosmetic`, `game_equip_cosmetic`, `game_downgrade_defense`). See the API reference group **Code Farm** and the full player guide at `/docs/code-farm.html`. ## Engagement @@ -255,11 +255,11 @@ Members can install [Agent Skills](https://agentskills.io) from a URL at `/skill ## Community sync -Users, posts and comments can be copied in from another community. The sending side posts batches of up to 50 records to `POST /sync/users`, `/sync/posts` and `/sync/comments` (administrator API key, `sync_enabled` set to `1`). Each record carries the id it has in the origin community (`origin_uid`); the server keeps a `sync_links` row from `(origin, kind, origin_uid)` to the local row, so a repeated record is an update or a no-op and never a duplicate. Records are validated and normalized with Pydantic (`devplacepy/services/sync/records.py`) and every record gets its own result, so one bad record never fails the batch. Votes, stars, XP and levels cannot be verified across communities and are not imported; they are listed under `ignored` in the result. Synced content sends no notifications, awards no XP, runs no AI correction and passes the content filter. `sync_origins` optionally restricts which origins may send. Limits, statuses and the record schemas are published at `GET /sync/spec.json`. The opt-in Community sync service (`CommunityService`) is the scheduled sender: it learns an external community, runs its generated collectors and feeds this API. A run refused by the kill switch, the allowlist or the aliasing switch is recorded as `skipped` and is never counted as a failure. Every run also writes a per-record sync log (`/admin/communities/{slug}/ledger?run=`): a verdict, exact counts per outcome (created, updated, unchanged, stale, skipped, warning, rejected, failed, error), and a filterable, paginated list with the reason each record was not synced in plain words. It is index-bound and capped, so it stays fast however large a run is. There is no export endpoint. Usage is written up in `prompt.md`. +Users, posts and comments can be copied in from another community. The sending side posts batches of up to 50 records to `POST /sync/users`, `/sync/posts` and `/sync/comments` (administrator API key, `sync_enabled` set to `1`). Each record carries the id it has in the origin community (`origin_uid`); the server keeps a `sync_links` row from `(origin, kind, origin_uid)` to the local row, so a repeated record is an update or a no-op and never a duplicate. Records are validated and normalized with Pydantic (`devplacepy/services/sync/records.py`) and every record gets its own result, so one bad record never fails the batch. Stars, XP and levels cannot be verified across communities and are not imported; they are listed under `ignored` in the result. A record whose post, parent, poll or project has not arrived yet is parked in a bounded pending queue and released when it does. Synced content sends no notifications, awards no XP, runs no AI correction and passes the content filter. `sync_origins` optionally restricts which origins may send. Limits, statuses and the record schemas are published at `GET /sync/spec.json`. The opt-in Community sync service (`CommunityService`) is the scheduled sender: it learns an external community, runs its generated collectors and feeds this API. A run refused by the kill switch, the allowlist or the aliasing switch is recorded as `skipped` and is never counted as a failure. Every run also writes a per-record sync log (`/admin/communities/{slug}/ledger?run=`): a verdict, exact counts per outcome (created, updated, unchanged, stale, skipped, warning, rejected, failed, error), and a filterable, paginated list with the reason each record was not synced in plain words. It is index-bound and capped, so it stays fast however large a run is. The origin can read what we hold through `GET /sync/export/{kind}` (public data only, rows that arrived by sync are never echoed back) and which of its records we removed through `GET /sync/deletions`. Usage is written up in `prompt.md`. ### Community discovery -Administrators can add an external community by name or address at `/admin/communities` (or `devplace community discover`). The system searches (rsearch), detects RSS, Atom, JSON and HTML feeds, learns the structure once, records the community's description, rules and size for the admin detail page, generates standalone collector code under `data/communities//`, and then runs it on a schedule through the sync API above. Members never see these communities. Login-protected communities take a stored account; JavaScript pages are rendered through stealthii; the internal AI model helps with HTML structure it cannot read otherwise. Administrators can manage everything through the admin pages, `devplace community ...` and Devii. Dry run (`--dry-run`, the admin button, and the default of the `Community sync` service) validates and logs everything without writing. Design and limits: `add_community.md` and `devplacepy/services/community/CLAUDE.md`. Per community, administrators can optionally (all off by default) sample which posts are accepted, filter by topic and minimum comments, jitter the schedule, translate content, and store authors under permanent pseudonyms with every mention rewritten (`community_sync_additional.md`); this is administrator-only and invisible to members. +Administrators can add an external community by name or address at `/admin/communities` (or `devplace community discover`). The system searches (rsearch), detects RSS, Atom, JSON and HTML feeds, learns the structure once, records the community's description, rules and size for the admin detail page, generates standalone collector code under `data/communities//`, and then runs it on a schedule through the sync API above. Members never see these communities. Login-protected communities take a stored account (form, JSON-API and TOTP logins; a captcha is detected and reported, never solved); a run keeps checkpoints so a killed run stays visible; administrators can edit and restore a community's learned schema at `/admin/communities/{slug}/schema`; new accounts created by a community run get a realistic handle derived from the author (the person behind a `by`/`via` is extracted from feed strings, then a seeded, weighted pattern adds a year, number or light humor; existing accounts are never renamed; `devplacepy/services/sync/CLAUDE.md`, "Realistic handles"); JavaScript pages are rendered through stealthii; the internal AI model helps with HTML structure it cannot read otherwise. Administrators can manage everything through the admin pages, `devplace community ...` and Devii. Dry run (`--dry-run`, the admin button, and the default of the `Community sync` service) validates and logs everything without writing. Design and limits: `add_community.md` and `devplacepy/services/community/CLAUDE.md`. Per community, administrators can optionally (all off by default) sample which posts are accepted, filter by topic and minimum comments, jitter the schedule, translate content, and store authors under permanent pseudonyms with every mention rewritten (`community_sync_additional.md`); this is administrator-only and invisible to members. ## Community nodes (built, off by default) diff --git a/devplacepy/attachments.py b/devplacepy/attachments.py index a07a0b7c..3e62e183 100644 --- a/devplacepy/attachments.py +++ b/devplacepy/attachments.py @@ -274,14 +274,14 @@ def delete_inline_image(filename): logger.warning(f"Failed to delete inline image {filename}: {e}") -def store_attachment(file_bytes, original_filename, user_uid): +def store_attachment(file_bytes, original_filename, user_uid, uid=None): if len(file_bytes) > _get_max_upload_bytes(): return None ext = Path(original_filename).suffix.lower() if not is_extension_allowed(ext): return None - uid = generate_uid() + uid = uid or generate_uid() stored_name = f"{uid}{ext}" directory = _directory_for(uid) file_dir = ATTACHMENTS_DIR / directory diff --git a/devplacepy/cli/community.py b/devplacepy/cli/community.py index b1726e70..747dc366 100644 --- a/devplacepy/cli/community.py +++ b/devplacepy/cli/community.py @@ -18,6 +18,7 @@ def _credentials(args) -> dict: "username": args.login_user or "", "email": args.login_email or "", "password": args.login_password or os.environ.get("COMMUNITY_PASSWORD", ""), + "totp": args.login_totp or os.environ.get("COMMUNITY_TOTP_SECRET", ""), } @@ -102,14 +103,14 @@ def cmd_community_login(args): print(f"Unknown community '{args.slug}'") sys.exit(1) if args.clear: - store.update(row["uid"], login_username="", login_email="", login_password="") + store.update(row["uid"], login_username="", login_email="", login_password="", login_totp_secret="") print(f"Removed the stored account of {row['slug']}") else: secrets = _credentials(args) if not secrets["password"] or not (secrets["username"] or secrets["email"]): print("Give --login-user or --login-email, and --login-password (or COMMUNITY_PASSWORD)") sys.exit(1) - store.update(row["uid"], login_username=secrets["username"], login_email=secrets["email"], login_password=secrets["password"]) + store.update(row["uid"], login_username=secrets["username"], login_email=secrets["email"], login_password=secrets["password"], login_totp_secret=secrets["totp"]) print(f"Stored the account for {row['slug']}; run 'community discover \"{row['query']}\" --relearn' to learn behind the login") _audit_cli("cli.community.login", f"CLI changed the stored account of community {row['slug']}", target_label=row["slug"]) @@ -133,6 +134,7 @@ def add_login_arguments(parser): parser.add_argument("--login-user", help="username of an account on the community") parser.add_argument("--login-email", help="email of an account on the community") parser.add_argument("--login-password", help="password (or set COMMUNITY_PASSWORD)") + parser.add_argument("--login-totp", help="authenticator secret of the account, for a login with two-factor codes (or set COMMUNITY_TOTP_SECRET)") def register_community(subparsers): diff --git a/devplacepy/cli/nodes.py b/devplacepy/cli/nodes.py index 35248613..fa3040e4 100644 --- a/devplacepy/cli/nodes.py +++ b/devplacepy/cli/nodes.py @@ -110,6 +110,33 @@ def cmd_nodes_clear(args): _audit_cli("cli.nodes.clear","CLI cleared the node job queue and event log", metadata=counts) +def cmd_nodes_geo_sync(args): + import asyncio + + from devplacepy.services.nodes.geo import GeoError, geo_sync + + try: + summary = asyncio.run(geo_sync(dry_run=args.dry_run, month=args.month)) + except GeoError as exc: + print(f"geo-sync refused: {exc}") + sys.exit(1) + verb = "installed" if summary["installed"] else "verified, not installed (dry run)" + print(f"{verb}: month {summary['month']}, {summary['asn_rows']} ASN ranges, {summary['country_rows']} country ranges") + if summary["installed"]: + _audit_cli("cli.nodes.geo_sync", f"CLI installed the node address database for {summary['month']}", metadata={"month": summary["month"]}) + + +def cmd_nodes_geo_status(args): + from devplacepy.services.nodes.geo import shared_geo + + status = shared_geo().status() + if not status["available"]: + print(f"not installed at {status['path']}; every donor is classed unknown") + return + print(f"month {status.get('month')}, {status.get('asn_rows')} ASN ranges, {status.get('country_rows')} country ranges, loaded {status.get('loaded_at')}") + print(status["attribution"]) + + def register_nodes(subparsers): nodes = subparsers.add_parser("nodes", help="Community donor node network") sub = nodes.add_subparsers(title="action", dest="action") @@ -130,3 +157,8 @@ def register_nodes(subparsers): clear = sub.add_parser("clear", help="Empty the job queue and the event log (never usage or donor totals)") clear.add_argument("--force", action="store_true", help="actually delete; without it only counts are shown") clear.set_defaults(func=cmd_nodes_clear) + geo_sync = sub.add_parser("geo-sync", help="Download the DB-IP address database used to class donor addresses") + geo_sync.add_argument("--dry-run", action="store_true", help="download and verify, do not install") + geo_sync.add_argument("--month", default="", help="YYYY-MM, default the newest published") + geo_sync.set_defaults(func=cmd_nodes_geo_sync) + sub.add_parser("geo-status", help="Show the installed address database").set_defaults(func=cmd_nodes_geo_status) diff --git a/devplacepy/config.py b/devplacepy/config.py index 37a75674..97348021 100644 --- a/devplacepy/config.py +++ b/devplacepy/config.py @@ -26,6 +26,8 @@ FORK_STAGING_DIR = DATA_DIR / "fork_staging" BACKUPS_DIR = DATA_DIR / "backups" BACKUP_STAGING_DIR = DATA_DIR / "backup_staging" NODE_CLIENTS_DIR = DATA_DIR / "node_clients" +NODE_GEO_DIR = DATA_DIR / "node_geo" +NODE_GEO_DB = NODE_GEO_DIR / "geo.db" RCLONE_BIN = environ.get("DEVPLACE_RCLONE_BIN", "rclone") RCLONE_CONFIG_FILE = environ.get( @@ -155,6 +157,7 @@ DATA_PATHS: dict[str, Path] = { "backups": BACKUPS_DIR, "backup_staging": BACKUP_STAGING_DIR, "node_clients": NODE_CLIENTS_DIR, + "node_geo": NODE_GEO_DIR, "seo_reports": SEO_REPORTS_DIR, "planning_reports": PLANNING_REPORTS_DIR, "dbapi": DBAPI_DIR, diff --git a/devplacepy/database/core.py b/devplacepy/database/core.py index bbbaca95..5abbd856 100644 --- a/devplacepy/database/core.py +++ b/devplacepy/database/core.py @@ -2,6 +2,7 @@ import dataset import logging +import warnings from pathlib import Path from sqlalchemy import or_ from sqlalchemy.pool import NullPool @@ -20,6 +21,8 @@ from devplacepy.config import ( logger = logging.getLogger(__name__) +warnings.filterwarnings("ignore", message="Skipped unsupported reflection of expression-based index") + ensure_data_dirs() if DATABASE_URL.startswith("sqlite:///"): _db_file = DATABASE_URL[len("sqlite:///") :] diff --git a/devplacepy/database/nodes.py b/devplacepy/database/nodes.py index 00c5f199..c2ae1780 100644 --- a/devplacepy/database/nodes.py +++ b/devplacepy/database/nodes.py @@ -24,6 +24,7 @@ LABEL_MAX = 60 EVENT_DETAIL_MAX = 500 MEGABYTE = 1024 * 1024 BATCH_SIZE = 500 +COUNTER_ROW_LIMIT = 500 HOURS_PATTERN = re.compile(r"^([01]\d|2[0-3]):[0-5]\d$") PLATFORMS = tuple(sorted({item.adapter for item in SPEC.destinations})) FLAG_OPEN = "open" @@ -714,14 +715,18 @@ def enqueue_request( method: str, adapter: str, mode: str, + *, + shadow_of: str = "", + exclude_device: str = "", ) -> str: uid = _uid() created = _execute( "INSERT INTO node_requests (uid, caller_kind, caller_id, url, method, adapter, " "mode, status, device_uid, result_status, result_bytes, result_body, " - "result_headers, truncated, error, created_at, assigned_at, finished_at) " + "result_headers, truncated, error, device_asn_class, shadow_of, exclude_device, " + "created_at, assigned_at, finished_at) " "SELECT :uid, :kind, :caller, :url, :method, :adapter, :mode, 'queued', '', 0, 0, " - "'', '', 0, '', :now, '', '' " + "'', '', 0, '', '', :shadow_of, :exclude_device, :now, '', '' " "WHERE (SELECT COUNT(*) FROM node_requests WHERE status IN ('queued','assigned')) " "< :max_depth", { @@ -732,6 +737,8 @@ def enqueue_request( "method": method, "adapter": adapter, "mode": mode, + "shadow_of": shadow_of, + "exclude_device": exclude_device, "now": _now_iso(), "max_depth": LIMITS.max_queue_depth, }, @@ -766,12 +773,18 @@ def count_open_requests() -> int: )["c"] -def claim_request(request_uid: str, device_uid: str) -> bool: +def claim_request(request_uid: str, device_uid: str, asn_class: str = "") -> bool: return ( _execute( "UPDATE node_requests SET status = 'assigned', device_uid = :device, " - "assigned_at = :now WHERE uid = :uid AND status = 'queued'", - {"device": device_uid, "now": _now_iso(), "uid": request_uid}, + "device_asn_class = :asn_class, assigned_at = :now " + "WHERE uid = :uid AND status = 'queued'", + { + "device": device_uid, + "asn_class": asn_class, + "now": _now_iso(), + "uid": request_uid, + }, ) == 1 ) @@ -989,3 +1002,165 @@ def recent_devices(limit: int = ADMIN_LIST_LIMIT) -> list[dict]: n=limit, ) ] + + +def set_class_override(device_uid: str, value: str) -> bool: + return ( + _execute( + "UPDATE node_devices SET class_override = :value, updated_at = :now " + "WHERE uid = :uid AND deleted_at IS NULL", + {"value": value, "now": _now_iso(), "uid": device_uid}, + ) + == 1 + ) + + +def touch_devices(device_uids, moment: datetime | None = None) -> int: + stamp = (moment or datetime.now(timezone.utc)).isoformat() + touched = 0 + for chunk in _batches(device_uids): + placeholders, params = _in_clause(chunk) + touched += _execute( + f"UPDATE node_devices SET last_seen_at = :now WHERE uid IN ({placeholders})", + {"now": stamp, **params}, + ) + return touched + + +def flush_counters(deltas: dict[str, int]) -> int: + rows = {name: int(value) for name, value in deltas.items() if int(value) > 0} + if not rows: + return 0 + now = _now_iso() + with db: + for name, value in rows.items(): + db.executable.execute( + text( + "INSERT INTO node_counters (name, value, updated_at) VALUES (:name, :value, :now) " + "ON CONFLICT(name) DO UPDATE SET value = COALESCE(value, 0) + excluded.value, " + "updated_at = excluded.updated_at" + ), + {"name": name, "value": value, "now": now}, + ) + return len(rows) + + +def lifetime_counters(prefix: str = "") -> dict[str, int]: + rows = db.query( + "SELECT name, COALESCE(value, 0) AS value FROM node_counters " + "WHERE name LIKE :prefix ESCAPE '\\' ORDER BY name LIMIT :n", + prefix=prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%", + n=COUNTER_ROW_LIMIT, + ) + return {row["name"]: int(row["value"]) for row in rows} + + +def save_circuits(rows: list[dict]) -> int: + now = _now_iso() + with db: + db.executable.execute(text("DELETE FROM node_circuits")) + for row in rows[: LIMITS.circuit_snapshot_max]: + db.executable.execute( + text( + "INSERT INTO node_circuits (key, failures, state, opened_at, updated_at) " + "VALUES (:key, :failures, :state, :opened_at, :now)" + ), + { + "key": row["key"], + "failures": int(row["failures"]), + "state": row["state"], + "opened_at": float(row["opened_at"]), + "now": now, + }, + ) + return min(len(rows), LIMITS.circuit_snapshot_max) + + +def load_circuits() -> list[dict]: + return [ + dict(row) + for row in db.query( + "SELECT key, failures, state, opened_at FROM node_circuits LIMIT :n", + n=LIMITS.circuit_snapshot_max, + ) + ] + + +def cache_get(key: str, moment: datetime | None = None) -> dict | None: + now = (moment or datetime.now(timezone.utc)).isoformat() + rows = list( + db.query( + "SELECT status, headers, body, stored_at FROM node_cache " + "WHERE key = :key AND expires_at > :now LIMIT 1", + key=key, + now=now, + ) + ) + return dict(rows[0]) if rows else None + + +def cache_put( + key: str, + method: str, + url: str, + status: int, + headers: str, + body: str, + size: int, + ttl_seconds: int, + moment: datetime | None = None, +) -> bool: + if ttl_seconds <= 0: + return False + now = moment or datetime.now(timezone.utc) + return ( + _execute( + "INSERT INTO node_cache (key, method, url, status, headers, body, size, stored_at, expires_at) " + "VALUES (:key, :method, :url, :status, :headers, :body, :size, :stored, :expires) " + "ON CONFLICT(key) DO UPDATE SET status = excluded.status, headers = excluded.headers, " + "body = excluded.body, size = excluded.size, stored_at = excluded.stored_at, " + "expires_at = excluded.expires_at", + { + "key": key, + "method": method, + "url": url[: LIMITS.url_max_chars], + "status": int(status), + "headers": headers, + "body": body, + "size": int(size), + "stored": now.isoformat(), + "expires": (now + timedelta(seconds=ttl_seconds)).isoformat(), + }, + ) + == 1 + ) + + +def cache_prune(max_bytes: int, moment: datetime | None = None) -> dict[str, int]: + now = (moment or datetime.now(timezone.utc)).isoformat() + expired = _execute("DELETE FROM node_cache WHERE expires_at <= :now", {"now": now}) + total = next(iter(db.query("SELECT COALESCE(SUM(size), 0) AS total FROM node_cache")))["total"] + evicted = 0 + while total > max_bytes: + rows = list( + db.query("SELECT key, size FROM node_cache ORDER BY stored_at, id LIMIT :n", n=BATCH_SIZE) + ) + if not rows: + break + for row in rows: + if total <= max_bytes: + break + evicted += _execute("DELETE FROM node_cache WHERE key = :key", {"key": row["key"]}) + total -= int(row["size"] or 0) + return {"expired": expired, "evicted": evicted, "bytes": max(0, int(total))} + + +def cache_clear() -> int: + return _execute("DELETE FROM node_cache", {}) + + +def cache_stats() -> dict[str, int]: + row = next( + iter(db.query("SELECT COUNT(*) AS entries, COALESCE(SUM(size), 0) AS bytes FROM node_cache")) + ) + return {"entries": int(row["entries"]), "bytes": int(row["bytes"])} diff --git a/devplacepy/database/schema.py b/devplacepy/database/schema.py index a7e37908..8329c526 100644 --- a/devplacepy/database/schema.py +++ b/devplacepy/database/schema.py @@ -1506,6 +1506,7 @@ def init_db(): ("ready_at", ""), ("watered_by", "[]"), ("raided_fraction", 0.0), + ("stop_after_harvest", 0), ("created_at", ""), ("updated_at", ""), ): @@ -1863,6 +1864,10 @@ def init_db(): for table in SOFT_DELETE_TABLES: ensure_soft_delete_columns(table) + from .sync import ensure_export_indexes + + ensure_export_indexes() + _index( db, "comments", @@ -2222,6 +2227,7 @@ NODES_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = { ("country", ""), ("asn", ""), ("asn_class", ""), + ("class_override", ""), ("ip_hash", ""), ("last_seen_at", ""), ("created_at", ""), @@ -2292,10 +2298,36 @@ NODES_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = { ("result_headers", ""), ("truncated", 0), ("error", ""), + ("device_asn_class", ""), + ("shadow_of", ""), + ("exclude_device", ""), ("created_at", ""), ("assigned_at", ""), ("finished_at", ""), ), + "node_counters": ( + ("name", ""), + ("value", 0), + ("updated_at", ""), + ), + "node_circuits": ( + ("key", ""), + ("failures", 0), + ("state", ""), + ("opened_at", 0.0), + ("updated_at", ""), + ), + "node_cache": ( + ("key", ""), + ("method", ""), + ("url", ""), + ("status", 0), + ("headers", ""), + ("body", ""), + ("size", 0), + ("stored_at", ""), + ("expires_at", ""), + ), "node_caller_daily": ( ("uid", ""), ("caller_kind", ""), @@ -2360,6 +2392,11 @@ def _ensure_nodes_tables() -> None: where="status IN ('queued','assigned')", ) _index(db, "node_requests", "idx_node_requests_created", ["created_at"]) + _index(db, "node_requests", "idx_node_requests_shadow", ["shadow_of"], where="shadow_of != ''") + _index(db, "node_counters", "idx_node_counters_name", ["name"], unique=True) + _index(db, "node_circuits", "idx_node_circuits_key", ["key"], unique=True) + _index(db, "node_cache", "idx_node_cache_key", ["key"], unique=True) + _index(db, "node_cache", "idx_node_cache_expires", ["expires_at"]) _index( db, "node_caller_daily", @@ -2390,6 +2427,28 @@ SYNC_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = { ("origin_updated_at", ""), ("first_synced_at", ""), ("synced_at", ""), + ("ack_at", ""), + ), + "sync_pending": ( + ("uid", ""), + ("origin", ""), + ("kind", ""), + ("origin_uid", ""), + ("missing_kind", ""), + ("missing_uid", ""), + ("record", ""), + ("attempts", 0), + ("created_at", ""), + ("expires_at", ""), + ), + "sync_candidates": ( + ("uid", ""), + ("origin", ""), + ("post_uid", ""), + ("record", ""), + ("comment_count", 0), + ("created_at", ""), + ("expires_at", ""), ), "sync_identities": ( ("uid", ""), @@ -2420,6 +2479,11 @@ def _ensure_sync_tables() -> None: unique=True, ) _index(db, "sync_links", "idx_sync_links_local", ["kind", "local_uid"]) + _index(db, "sync_pending", "idx_sync_pending_record", ["origin", "kind", "origin_uid"], unique=True) + _index(db, "sync_pending", "idx_sync_pending_missing", ["origin", "missing_kind", "missing_uid"]) + _index(db, "sync_pending", "idx_sync_pending_expires", ["expires_at"]) + _index(db, "sync_candidates", "idx_sync_candidates_post", ["origin", "post_uid"], unique=True) + _index(db, "sync_candidates", "idx_sync_candidates_expires", ["expires_at"]) _index( db, "sync_identities", @@ -2461,6 +2525,8 @@ COMMUNITY_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = { ("login_username", ""), ("login_email", ""), ("login_password", ""), + ("login_totp_secret", ""), + ("schema_previous", ""), ("sync_options", ""), ("claimed_at", ""), ("deleted_at", ""), @@ -2516,6 +2582,37 @@ def _ensure_community_tables() -> None: _index(db, "community_run_items", "idx_community_items_run_outcome", ["run_uid", "outcome", "kind", "seq"]) _index(db, "community_run_items", "idx_community_items_community_rank", ["community_uid", "rank", "created_at DESC", "seq"]) _index(db, "community_run_items", "idx_community_items_community_outcome", ["community_uid", "outcome", "created_at DESC", "seq"]) + _index(db, "community_run_items", "idx_community_items_run_seq", ["run_uid", "seq"], unique=True) + _ensure_ledger_search() + + +LEDGER_SEARCH_TABLE = "community_run_items_fts" +LEDGER_SEARCH_COLUMNS = ("label", "origin_uid", "detail") + + +def _ensure_ledger_search() -> None: + try: + exists = bool(list(db.query("SELECT name FROM sqlite_master WHERE name = :name", name=LEDGER_SEARCH_TABLE))) + columns = ", ".join(LEDGER_SEARCH_COLUMNS) + new_columns = ", ".join(f"new.{column}" for column in LEDGER_SEARCH_COLUMNS) + old_columns = ", ".join(f"old.{column}" for column in LEDGER_SEARCH_COLUMNS) + with db: + db.query( + f"CREATE VIRTUAL TABLE IF NOT EXISTS {LEDGER_SEARCH_TABLE} USING fts5({columns}, " + "content='community_run_items', content_rowid='id', tokenize='trigram')" + ) + db.query( + "CREATE TRIGGER IF NOT EXISTS community_run_items_fts_insert AFTER INSERT ON community_run_items BEGIN " + f"INSERT INTO {LEDGER_SEARCH_TABLE}(rowid, {columns}) VALUES (new.id, {new_columns}); END" + ) + db.query( + "CREATE TRIGGER IF NOT EXISTS community_run_items_fts_delete AFTER DELETE ON community_run_items BEGIN " + f"INSERT INTO {LEDGER_SEARCH_TABLE}({LEDGER_SEARCH_TABLE}, rowid, {columns}) VALUES ('delete', old.id, {old_columns}); END" + ) + if not exists: + db.query(f"INSERT INTO {LEDGER_SEARCH_TABLE}({LEDGER_SEARCH_TABLE}) VALUES ('rebuild')") + except Exception as exc: + logger.warning("Ledger full-text search is unavailable, the text filter will scan: %s", exc) def _ensure_moderation_tables() -> None: diff --git a/devplacepy/database/sync.py b/devplacepy/database/sync.py index d12ecf6d..bcf0986f 100644 --- a/devplacepy/database/sync.py +++ b/devplacepy/database/sync.py @@ -2,6 +2,8 @@ from __future__ import annotations +from datetime import datetime, timedelta, timezone + from sqlalchemy import text from devplacepy.utils.text import generate_uid @@ -129,3 +131,230 @@ def claim_identity( def save_style_samples(identity_uid: str, samples_json: str) -> None: get_table("sync_identities").update({"uid": identity_uid, "style_samples": samples_json}, ["uid"]) + + +PENDING_TABLE = "sync_pending" + + +def park_pending( + origin: str, + kind: str, + origin_uid: str, + missing_kind: str, + missing_uid: str, + record_json: str, + *, + attempts: int, + ttl_days: int, + cap: int, +) -> bool: + now = datetime.now(timezone.utc) + with db: + result = db.executable.execute( + text( + "INSERT INTO sync_pending (uid, origin, kind, origin_uid, missing_kind, missing_uid, " + "record, attempts, created_at, expires_at) " + "SELECT :uid, :origin, :kind, :origin_uid, :missing_kind, :missing_uid, :record, " + ":attempts, :now, :expires " + "WHERE (SELECT COUNT(*) FROM sync_pending WHERE origin = :origin) < :cap " + "ON CONFLICT(origin, kind, origin_uid) DO UPDATE SET " + "missing_kind = excluded.missing_kind, missing_uid = excluded.missing_uid, " + "record = excluded.record, attempts = excluded.attempts, expires_at = excluded.expires_at" + ), + { + "uid": generate_uid(), + "origin": origin, + "kind": kind, + "origin_uid": origin_uid, + "missing_kind": missing_kind, + "missing_uid": missing_uid, + "record": record_json, + "attempts": int(attempts), + "now": now.isoformat(), + "expires": (now + timedelta(days=ttl_days)).isoformat(), + "cap": int(cap), + }, + ) + return result.rowcount >= 1 + + +def pending_exists(origin: str, kind: str, origin_uid: str) -> bool: + return get_table(PENDING_TABLE).find_one(origin=origin, kind=kind, origin_uid=origin_uid) is not None + + +def ready_pending(limit: int, origin: str = "") -> list[dict]: + clause = "AND p.origin = :origin " if origin else "" + rows = db.query( + "SELECT p.* FROM sync_pending p JOIN sync_links l ON l.origin = p.origin " + "AND l.kind = p.missing_kind AND l.origin_uid = p.missing_uid " + f"WHERE 1 = 1 {clause}ORDER BY p.created_at, p.id LIMIT :n", + n=int(limit), + **({"origin": origin} if origin else {}), + ) + return [dict(row) for row in rows] + + +def delete_pending(uid: str) -> int: + with db: + return db.executable.execute(text("DELETE FROM sync_pending WHERE uid = :uid"), {"uid": uid}).rowcount + + +def expire_pending(max_attempts: int) -> int: + with db: + return db.executable.execute( + text("DELETE FROM sync_pending WHERE expires_at <= :now OR attempts >= :attempts"), + {"now": _now_iso(), "attempts": int(max_attempts)}, + ).rowcount + + +def pending_counts() -> dict[str, int]: + rows = db.query("SELECT origin, COUNT(*) AS n FROM sync_pending GROUP BY origin ORDER BY n DESC LIMIT 100") + return {row["origin"]: int(row["n"]) for row in rows} + + +STAR_TABLES = {"post": "posts", "project": "projects"} + + +def recompute_stars(target_type: str, target_uid: str) -> int | None: + table = STAR_TABLES.get(target_type) + if table is None: + return None + with db: + db.executable.execute( + text( + f"UPDATE {table} SET stars = (SELECT COALESCE(SUM(value), 0) FROM votes " + "WHERE target_uid = :uid AND target_type = :type AND deleted_at IS NULL) " + "WHERE uid = :uid" + ), + {"uid": target_uid, "type": target_type}, + ) + row = get_table(table).find_one(uid=target_uid) + return int(row["stars"] or 0) if row else None + + +EXPORT_CHANGED: dict[str, str] = { + "users": "created_at", + "projects": "COALESCE(deleted_at, updated_at, created_at)", + "posts": "COALESCE(deleted_at, updated_at, created_at)", + "comments": "COALESCE(deleted_at, updated_at, created_at)", + "polls": "COALESCE(deleted_at, created_at)", + "follows": "COALESCE(deleted_at, created_at)", + "attachments": "COALESCE(deleted_at, created_at)", +} + + +def ensure_export_indexes() -> None: + from .core import _index + + for table, expression in EXPORT_CHANGED.items(): + if table in db.tables: + _index(db, table, f"idx_{table}_changed", [expression, "uid"]) + + +def deletion_rows(origin: str, kind: str, table: str, after_id: int, limit: int) -> list[dict]: + if table not in db.tables: + return [] + soft = db[table].has_column("deleted_at") + columns = "t.deleted_at AS deleted_at, t.deleted_by AS deleted_by" if soft else "NULL AS deleted_at, NULL AS deleted_by" + deleted = "OR (t.deleted_at IS NOT NULL AND COALESCE(t.deleted_by, '') != :actor)" if soft else "" + rows = db.query( + f"SELECT l.id AS link_id, l.kind AS kind, l.origin_uid AS origin_uid, t.uid AS local_uid, {columns} " + f"FROM sync_links l LEFT JOIN {table} t ON t.uid = l.local_uid " + "WHERE l.origin = :origin AND l.kind = :kind AND l.id > :after " + "AND COALESCE(l.ack_at, '') = '' " + f"AND (t.uid IS NULL {deleted}) " + "ORDER BY l.id LIMIT :n", + origin=origin, + kind=kind, + after=int(after_id), + n=int(limit), + **({"actor": "sync"} if soft else {}), + ) + return [dict(row) for row in rows] + + +def acknowledge_links(origin: str, items: list[tuple[str, str]]) -> int: + stamp = _now_iso() + done = 0 + with db: + for kind, origin_uid in items: + done += db.executable.execute( + text( + "UPDATE sync_links SET ack_at = :now WHERE origin = :origin AND kind = :kind " + "AND origin_uid = :origin_uid AND COALESCE(ack_at, '') = ''" + ), + {"now": stamp, "origin": origin, "kind": kind, "origin_uid": origin_uid}, + ).rowcount + return done + + +def rename_identity(identity_uid: str, our_name: str) -> bool: + try: + with db: + result = db.executable.execute( + text("UPDATE sync_identities SET our_name = :name, our_key = :key WHERE uid = :uid"), + {"name": our_name, "key": our_name.lower(), "uid": identity_uid}, + ) + return result.rowcount == 1 + except Exception: + return False + + +def linked_rows_containing(origin: str, kind: str, table: str, columns: tuple[str, ...], needle: str, limit: int) -> list[dict]: + pattern = "%" + needle.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" + matches = " OR ".join(f"t.{column} LIKE :pattern ESCAPE '\\'" for column in columns) + selected = ", ".join(f"t.{column} AS {column}" for column in columns) + rows = db.query( + f"SELECT t.uid AS uid, {selected} FROM sync_links l JOIN {table} t ON t.uid = l.local_uid " + f"WHERE l.origin = :origin AND l.kind = :kind AND ({matches}) ORDER BY l.id LIMIT :limit", + origin=origin, + kind=kind, + pattern=pattern, + limit=int(limit), + ) + return [dict(row) for row in rows] + + +def load_candidates(origin: str) -> list[dict]: + expire_candidates() + return [dict(row) for row in get_table("sync_candidates").find(origin=origin, order_by=["id"])] + + +def save_candidate(origin: str, post_uid: str, record_json: str, comment_count: int, *, ttl_days: int, cap: int) -> bool: + now = datetime.now(timezone.utc) + with db: + result = db.executable.execute( + text( + "INSERT INTO sync_candidates (uid, origin, post_uid, record, comment_count, created_at, expires_at) " + "SELECT :uid, :origin, :post_uid, :record, :count, :now, :expires " + "WHERE (SELECT COUNT(*) FROM sync_candidates WHERE origin = :origin) < :cap " + "ON CONFLICT(origin, post_uid) DO UPDATE SET record = excluded.record, " + "comment_count = excluded.comment_count" + ), + { + "uid": generate_uid(), + "origin": origin, + "post_uid": post_uid, + "record": record_json, + "count": int(comment_count), + "now": now.isoformat(), + "expires": (now + timedelta(days=ttl_days)).isoformat(), + "cap": int(cap), + }, + ) + return result.rowcount >= 1 + + +def delete_candidate(origin: str, post_uid: str) -> int: + with db: + return db.executable.execute( + text("DELETE FROM sync_candidates WHERE origin = :origin AND post_uid = :post_uid"), + {"origin": origin, "post_uid": post_uid}, + ).rowcount + + +def expire_candidates() -> int: + with db: + return db.executable.execute( + text("DELETE FROM sync_candidates WHERE expires_at <= :now"), {"now": _now_iso()} + ).rowcount diff --git a/devplacepy/docs_api/groups/community.py b/devplacepy/docs_api/groups/community.py index 1f54a26c..46a7e9d0 100644 --- a/devplacepy/docs_api/groups/community.py +++ b/devplacepy/docs_api/groups/community.py @@ -7,6 +7,7 @@ ACCOUNT = [ field("login_username", "form", "string", False, "bot", "Username of an account on the community, for a login-protected community."), field("login_email", "form", "string", False, "bot@example.com", "Email of that account, when the community logs in by email."), field("login_password", "form", "string", False, "", "Password of that account. Stored for the collectors and never returned by any endpoint."), + field("login_totp_secret", "form", "string", False, "", "Authenticator (TOTP) secret of that account when it uses two-factor codes. Stored for the collectors and never returned by any endpoint."), ] ITEM_SAMPLE = { @@ -93,6 +94,41 @@ Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html) "busy": False, }, ), + endpoint( + id="community-schema", + method="GET", + path="/admin/communities/{slug}/schema", + title="Learned schema", + summary="The complete learned schema of a community as JSON, editable by hand.", + auth="admin", + params=[SLUG], + sample_response={"community": ITEM_SAMPLE, "schema_text": "{...}", "has_previous": False, "errors": [], "notice": ""}, + ), + endpoint( + id="community-schema-save", + method="POST", + path="/admin/communities/{slug}/schema", + title="Replace the learned schema", + summary="Validate a schema, check it against a small live sample, and only then store it. The previous schema is kept.", + auth="admin", + encoding="form", + destructive=True, + params=[SLUG, field("schema", "form", "string", True, "{...}", "The complete schema as a JSON object string.")], + sample_response={"ok": True, "redirect": "/admin/communities/SLUG/schema", "data": {"posts": 5, "comments": 12}}, + notes=["`400` lists every structural problem; `422` means the live sample collected no post. Nothing is stored in either case. Audited as `admin.community.schema`."], + ), + endpoint( + id="community-schema-restore", + method="POST", + path="/admin/communities/{slug}/schema/restore", + title="Restore the previous schema", + summary="Swap back to the schema that was replaced by the last save.", + auth="admin", + encoding="form", + params=[SLUG], + sample_response={"ok": True, "redirect": "/admin/communities/SLUG/schema"}, + notes=["`409` when there is no previous schema. Audited as `admin.community.schema_restore`."], + ), endpoint( id="community-ledger", method="GET", @@ -177,6 +213,7 @@ Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html) field("min_comments", "form", "integer", False, "0", "Minimum comments collected in the run for a new post to be accepted."), field("topics", "form", "string", False, "devlog, rant", "Comma separated topic allow-list; empty accepts all."), field("jitter_percent", "form", "integer", False, "0", "Randomise the next run time by up to this percentage of the interval, 0 to 50."), + field("collect_attachments", "form", "boolean", False, "false", "Download the images and PDFs linked from posts and comments (at most 4 per post, 40 and 25 MB per run) and store them as attachments."), ], sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"}, ), diff --git a/devplacepy/docs_api/groups/game.py b/devplacepy/docs_api/groups/game.py index 366fa188..960b4ff4 100644 --- a/devplacepy/docs_api/groups/game.py +++ b/devplacepy/docs_api/groups/game.py @@ -163,6 +163,19 @@ client are on the [Code Farm guide](/docs/code-farm.html). params=[field("slot", "form", "integer", True, "0", "Plot slot index, 0-based.")], sample_response={"ok": True, "farm": {"coins": 86}}, ), + endpoint( + id="game-plot-stop", + method="POST", + path="/game/plot/stop", + title="Pause auto-replant on a plot", + summary="Needs the Continuous Delivery Mastery upgrade. With stop true, the next harvest of this plot does not replant, so you can choose a different crop. The pause clears itself when the plot is planted again. With stop false it cancels a pending pause. Rejected on an empty plot.", + auth="user", + params=[ + field("slot", "form", "integer", True, "0", "Plot slot index, 0-based."), + field("stop", "form", "boolean", True, "true", "`true` to skip the replant after the next harvest, `false` to cancel."), + ], + sample_response={"ok": True, "farm": {"plots": [{"slot": 0, "can_stop_replant": True, "stop_after_harvest": True}]}}, + ), endpoint( id="game-buy-plot", method="POST", diff --git a/devplacepy/docs_api/groups/nodes.py b/devplacepy/docs_api/groups/nodes.py index 87edac30..4ec3ae32 100644 --- a/devplacepy/docs_api/groups/nodes.py +++ b/devplacepy/docs_api/groups/nodes.py @@ -386,6 +386,22 @@ page. Every endpoint follows the shared [Conventions & Errors](/docs/conventions sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "DEVICE_UID", "minutes": 60}}, notes=["Refused (audited as `denied`) when the device owner is a more senior administrator. Audited as `node.suspend`."], ), + endpoint( + id="admin-nodes-class", + method="POST", + path="/admin/nodes/devices/{device_uid}/class", + title="Override a device's address class", + summary="Let one donor device serve jobs although its address is not residential or mobile.", + auth="admin", + encoding="form", + destructive=True, + params=[ + field("device_uid", "path", "string", True, "DEVICE_UID", "Device uid."), + field("value", "form", "string", True, "allow", "`allow` to override, `clear` to remove the override."), + ], + sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "DEVICE_UID", "class_override": "allow"}}, + notes=["Refused (audited as `denied`) when the device owner is a more senior administrator. Audited as `node.device.override`."], + ), endpoint( id="admin-nodes-unsuspend", method="POST", diff --git a/devplacepy/docs_api/groups/sync.py b/devplacepy/docs_api/groups/sync.py index c498f82e..5cb21eaa 100644 --- a/devplacepy/docs_api/groups/sync.py +++ b/devplacepy/docs_api/groups/sync.py @@ -13,11 +13,12 @@ RESULT_SAMPLE = { "kind": "posts", "dry_run": False, "total": 2, - "counts": {"created": 1, "updated": 0, "unchanged": 0, "stale": 0, "rejected": 1, "failed": 0}, + "counts": {"created": 1, "updated": 0, "unchanged": 0, "stale": 0, "rejected": 0, "failed": 0, "deferred": 1}, "results": [ - {"index": 0, "origin_uid": "p-1", "status": "created", "reason": "", "local_uid": "LOCAL_UID", "errors": [], "ignored": ["stars"]}, - {"index": 1, "origin_uid": "p-2", "status": "rejected", "reason": "unknown_author", "local_uid": "", "errors": ["author u-9 is not synced yet, send users first"], "ignored": []}, + {"index": 0, "origin_uid": "p-1", "status": "created", "reason": "", "local_uid": "LOCAL_UID", "errors": [], "ignored": ["stars"], "needs_kind": "", "needs_uid": ""}, + {"index": 1, "origin_uid": "p-2", "status": "deferred", "reason": "unknown_author", "local_uid": "", "errors": ["author u-9 is not synced yet, send users first"], "ignored": [], "needs_kind": "users", "needs_uid": "u-9"}, ], + "released": {"created": 3}, } GROUP = { @@ -27,7 +28,7 @@ GROUP = { "intro": """ # Community sync -Copy users, posts and comments from another community into this one. The API is idempotent +Copy users, projects, posts, comments, polls, votes, reactions, follows, poll ballots and attachments from another community into this one, and pull this community's own public data out. The API is idempotent and safe to repeat: every record carries the id it has in the **origin** community (`origin_uid`), the server remembers which local row that id maps to, and a repeated record is an update or a no-op, never a duplicate. @@ -35,16 +36,26 @@ an update or a no-op, never a duplicate. - **Validated and normalized here.** The sender posts raw data; the server fixes what can be fixed (usernames, whitespace, control characters, unknown topics, timestamps) and rejects what cannot, record by record. One bad record never fails the batch. -- **Counters are not imported.** Votes, stars, XP and levels cannot be verified across - communities, so they are ignored and listed under `ignored` in the result. +- **Counters are never sent as numbers.** Stars, XP and levels cannot be verified across + communities, so they are ignored and listed under `ignored` in the result. A post or project + gets stars only from synced votes, and a vote needs a synced voter and a synced target. - **No side effects.** Synced content sends no notifications, awards no XP and runs no AI correction. Content that matches the content filter is rejected. -- **Order matters.** Send users first, then posts, then comments (parents before replies). +- **Order is forgiven, not required.** Send users, projects, posts, comments, polls, then the + engagement kinds. A record whose author, post, parent, target, poll or project is not synced + yet is answered `deferred` with `needs_kind` and `needs_uid`, parked for seven days (at most + 5000 per origin) and applied by itself when that record arrives; the response then lists + `released` counts. - **Off by default.** An administrator enables it at `/admin/settings` (`sync_enabled`) and may restrict senders with `sync_origins` (comma separated; empty allows any origin). - **JSON or form.** The body is JSON. A form-encoded body is accepted too, with `records` as a - JSON array string; Devii uses that form through the `sync_users`, `sync_posts` and - `sync_comments` tools. + JSON array string; Devii uses that form through the `sync_` tools. +- **Pull feed.** `GET /sync/export/{kind}` pages out this community's own users, projects, + posts, comments, polls, follows and attachments. Votes, reactions and poll ballots are never + exported because who voted is not public, and rows that were themselves synced in are never + echoed back. +- **Deletions are reported.** `GET /sync/deletions` lists synced records deleted or purged here + and `POST /sync/deletions/ack` stops them being reported. The accepted fields and every limit are published at `/sync/spec.json`. Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html). @@ -60,8 +71,8 @@ the shared [Conventions & Errors](/docs/conventions.html). interactive=True, sample_response={ "limits": {"max_batch_records": 50}, - "kinds": ["users", "posts", "comments"], - "statuses": ["created", "updated", "unchanged", "stale", "rejected", "failed"], + "kinds": ["users", "projects", "posts", "comments", "polls", "votes", "reactions", "follows", "poll_votes", "attachments"], + "statuses": ["created", "updated", "unchanged", "stale", "rejected", "failed", "deferred"], "reasons": ["invalid", "unknown_author"], "records": {"users": {"title": "UserRecord", "type": "object"}}, }, @@ -115,5 +126,163 @@ the shared [Conventions & Errors](/docs/conventions.html). "A parent that belongs to a different post is rejected with `parent_mismatch`.", ], ), + endpoint( + id="sync-projects", + method="POST", + path="/sync/projects", + title="Upsert projects", + summary="Create or update public, writable projects. The author must already be synced.", + auth="admin", + encoding="json", + interactive=False, + params=ENVELOPE_PARAMS, + sample_response=RESULT_SAMPLE, + notes=[ + "Record fields: `origin_uid`, `author_uid`, `title`, `description`, `created_at`, optional `project_type` (game, game_asset, software, mobile_app, website), `status`, `platforms`, `website_url`, `repo_url`, `updated_at`, `deleted`.", + "Posts link to a project with `project_uid`.", + ], + ), + endpoint( + id="sync-polls", + method="POST", + path="/sync/polls", + title="Upsert polls", + summary="Create or update the poll of a synced post (one poll per post, 2 to 6 options).", + auth="admin", + encoding="json", + interactive=False, + params=ENVELOPE_PARAMS, + sample_response=RESULT_SAMPLE, + notes=[ + "Record fields: `origin_uid`, `author_uid`, `post_uid`, `question`, `options` (a list of `{origin_uid, label}`), `created_at`, optional `deleted`.", + "Editing the options adds, renames and removes them; ballots for a removed option stay.", + ], + ), + endpoint( + id="sync-votes", + method="POST", + path="/sync/votes", + title="Upsert votes", + summary="Create, change or clear a vote of a synced user on a synced post, comment or project.", + auth="admin", + encoding="json", + interactive=False, + params=ENVELOPE_PARAMS, + sample_response=RESULT_SAMPLE, + notes=[ + "Record fields: `origin_uid`, `voter_uid`, `target_kind` (post, comment, project), `target_uid`, `value` (1 or -1), `created_at`, optional `deleted` to clear.", + "A second origin id for the same voter and target revives the same row. Stars of posts and projects are recomputed from the live synced votes.", + ], + ), + endpoint( + id="sync-reactions", + method="POST", + path="/sync/reactions", + title="Upsert reactions", + summary="Add or remove an emoji reaction of a synced user.", + auth="admin", + encoding="json", + interactive=False, + params=ENVELOPE_PARAMS, + sample_response=RESULT_SAMPLE, + notes=["Record fields: `origin_uid`, `author_uid`, `target_kind`, `target_uid`, `emoji` (one emoji), `created_at`, optional `deleted`."], + ), + endpoint( + id="sync-follows", + method="POST", + path="/sync/follows", + title="Upsert follows", + summary="Create or remove a follow between two synced users. No notification, XP or badge is produced.", + auth="admin", + encoding="json", + interactive=False, + params=ENVELOPE_PARAMS, + sample_response=RESULT_SAMPLE, + notes=["Record fields: `origin_uid`, `follower_uid`, `following_uid`, `created_at`, optional `deleted`. Following yourself is rejected with `self_follow`."], + ), + endpoint( + id="sync-poll-votes", + method="POST", + path="/sync/poll_votes", + title="Upsert poll ballots", + summary="Record a synced user's ballot in a synced poll (one per voter; changing the option updates it).", + auth="admin", + encoding="json", + interactive=False, + params=ENVELOPE_PARAMS, + sample_response=RESULT_SAMPLE, + notes=["Record fields: `origin_uid`, `voter_uid`, `poll_uid`, `option_uid`, `created_at`, optional `deleted`. An option that does not exist in that poll is rejected with `unknown_option`."], + ), + endpoint( + id="sync-attachments", + method="POST", + path="/sync/attachments", + title="Upsert attachments", + summary="Store a file for a synced post, comment or project. The sender downloads the file and sends its bytes.", + auth="admin", + encoding="json", + interactive=False, + params=ENVELOPE_PARAMS, + sample_response=RESULT_SAMPLE, + notes=[ + "Record fields: `origin_uid`, `author_uid`, `target_kind`, `target_uid`, `filename`, `content_b64` (strict base64, at most 5 MB), `created_at`, optional `deleted`.", + "The normal upload rules apply (allowed types, size). An attachment cannot change: a known `origin_uid` with different bytes is rejected with `immutable`; only a tombstone or a restore is accepted.", + ], + ), + endpoint( + id="sync-export", + method="GET", + path="/sync/export/{kind}", + title="Export own records", + summary="Page out this community's users, projects, posts, comments, polls, follows or attachments in sync record form.", + auth="admin", + encoding="json", + interactive=False, + params=[ + field("kind", "path", "string", True, "posts", "users, projects, posts, comments, polls, follows or attachments."), + field("since", "query", "string", False, "", "Only records changed after this ISO 8601 instant. Users have no change time, so for them it filters by sign-up time."), + field("cursor", "query", "string", False, "", "The `next_cursor` of the previous page."), + field("limit", "query", "integer", False, "50", "Records per page, 1 to 50."), + ], + sample_response={"kind": "posts", "count": 1, "has_more": False, "next_cursor": "", "records": [{"origin_uid": "POST_UID", "author_uid": "USER_UID", "content": "...", "deleted": False}]}, + notes=[ + "Only public data: no email, password hash, role or API key; private projects are left out; attachments carry a `url` instead of bytes.", + "Votes, reactions and poll ballots are never exported. Rows that arrived by sync are never exported, so two communities cannot echo each other's data.", + "Soft-deleted rows appear as records with `deleted: true`. Audited as `sync.export`.", + ], + ), + endpoint( + id="sync-deletions", + method="GET", + path="/sync/deletions", + title="List local deletions", + summary="Synced records of one origin that were deleted or purged on this side and not yet acknowledged.", + auth="admin", + encoding="json", + interactive=False, + params=[ + field("origin", "query", "string", True, "other.example.org", "The sending community's origin id."), + field("cursor", "query", "string", False, "", "The `next_cursor` of the previous page."), + field("limit", "query", "integer", False, "50", "Entries per page, 1 to 50."), + ], + sample_response={"origin": "other.example.org", "count": 1, "has_more": False, "next_cursor": "", "items": [{"kind": "posts", "origin_uid": "p-1", "by": "member", "deleted_at": "2026-10-16T10:00:00+00:00"}]}, + notes=["`by` is `purged` (the row is gone), `system` or `member`. A deletion made by the sync itself is never listed."], + ), + endpoint( + id="sync-deletions-ack", + method="POST", + path="/sync/deletions/ack", + title="Acknowledge local deletions", + summary="Tell this community that the origin has seen the deletions, so they are no longer listed or reported.", + auth="admin", + encoding="json", + interactive=False, + params=[ + field("origin", "body", "string", True, "other.example.org", "The sending community's origin id."), + field("items", "body", "array", True, "[]", "1-50 objects `{kind, origin_uid}`."), + ], + sample_response={"origin": "other.example.org", "acknowledged": 1}, + notes=["After acknowledging, a re-sent record answers `unchanged` with reason `acknowledged_deleted`. Audited as `sync.deletions.ack`."], + ), ], } diff --git a/devplacepy/models.py b/devplacepy/models.py index 777e263f..a0740b8c 100644 --- a/devplacepy/models.py +++ b/devplacepy/models.py @@ -333,6 +333,7 @@ class CommunityAddForm(BaseModel): login_username: str = Field(default="", max_length=200) login_email: str = Field(default="", max_length=254) login_password: str = Field(default="", max_length=500) + login_totp_secret: str = Field(default="", max_length=200) @field_validator("query") @classmethod @@ -340,12 +341,19 @@ class CommunityAddForm(BaseModel): return value.strip() +class CommunitySchemaForm(BaseModel): + schema_text: str = Field(min_length=2, max_length=300_000, alias="schema") + + model_config = {"populate_by_name": True} + + class CommunityUpdateForm(BaseModel): interval_seconds: int = Field(ge=900, le=2592000) enabled: bool = False login_username: str = Field(default="", max_length=200) login_email: str = Field(default="", max_length=254) login_password: str = Field(default="", max_length=500) + login_totp_secret: str = Field(default="", max_length=200) clear_login: bool = False alias: Optional[bool] = None translate_to: Optional[str] = Field(default=None, max_length=12) @@ -353,6 +361,7 @@ class CommunityUpdateForm(BaseModel): min_comments: Optional[int] = Field(default=None, ge=0, le=1000) topics: Optional[str] = Field(default=None, max_length=500) jitter_percent: Optional[int] = Field(default=None, ge=0, le=50) + collect_attachments: Optional[bool] = None class CommunityRunForm(BaseModel): @@ -793,6 +802,11 @@ class GameSlotForm(BaseModel): slot: int = Field(ge=0, le=64) +class GamePlotStopForm(BaseModel): + slot: int = Field(ge=0, le=64) + stop: bool + + class GamePerkForm(BaseModel): perk: str = Field(min_length=1, max_length=40) @@ -1206,6 +1220,12 @@ class AdminNodeSettingsForm(BaseModel): nodes_cache_ttl_seconds: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX) nodes_social_via_nodes: Optional[Literal["0", "1"]] = None nodes_daily_social_user: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX) + nodes_allow_unknown_class: Optional[Literal["0", "1"]] = None + nodes_redundancy_percent: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX) + + +class AdminNodeClassForm(BaseModel): + value: Literal["allow", "clear"] class AdminNodeSuspendForm(BaseModel): diff --git a/devplacepy/routers/CLAUDE.md b/devplacepy/routers/CLAUDE.md index 41df9718..7b3a22f3 100644 --- a/devplacepy/routers/CLAUDE.md +++ b/devplacepy/routers/CLAUDE.md @@ -49,7 +49,7 @@ Prefixes are wired in `main.py`: | `/dbapi` | dbapi/ package - **primary-administrator-only, strictly READ-ONLY** generic database API over `dataset` (never inserts/updates/deletes/restores; the write surface was removed because it bypassed every per-route admin safeguard). `tables.py` (`GET /tables`, `GET /{table}/schema`), `crud.py` (read only: `GET /{table}` + `GET /{table}/{key}/{value}`, `?include_deleted`), `query.py` (`POST /query` hard SELECT-only validated read-only run; `POST /query/async` + `GET /query/{uid}` + `WS /query/{uid}/ws` via `DbApiJobService` kind `dbquery`), `nl.py` (`POST /nl` natural-language->SQL via the AI gateway, returns validated SELECT, `execute=true` runs it read-only). **Auth = the PRIMARY administrator only** (the earliest-created Admin, `utils.is_primary_admin` / `database.get_primary_admin_uid` - the same identity that gates backup downloads) by session/api_key (`services/dbapi/policy.py`); every other administrator is refused like a member, and there is **no internal-key path** - the gateway `internal_gateway_key()` is NOT accepted (no service-to-service access). SQL validated by `services/dbapi/validate.py` (sqlglot classify + suspicious-flag + read-only `EXPLAIN` dry-run). Devii tools `db_*` are read-only (list/get/query/nl; no write tools) and flagged `requires_primary_admin=True`, so they are added to the LLM tool list ONLY for the primary administrator (every other session never sees them and Devii is unaware they exist). `services/pubsub/policy.py` resolves its own admin/internal actor and does NOT reuse `dbapi.policy.caller_for`, so the primary-admin restriction does not leak onto the pub/sub bus. See `devplacepy/services/dbapi/CLAUDE.md` | | `/quizzes` | quizzes/ package - **Quizzes**: `index.py` (hub, create, import, detail, export, edit, delete, publish, per-quiz leaderboard, cross-quiz scoreboard), `questions.py` (builder page + question CRUD + reorder), `attempts.py` (start/resume, play, answer, finish, results). The hub `GET /quizzes` is the approved three-column `.feed-layout` reused verbatim from `/feed`: filters + search left (`?filter=all|todo|done|mine|drafts`, `?search=`, `?page=`), the quiz list centre with the **New quiz** and **Create quiz with Devii** actions, and the cross-quiz **scoreboard** rail right (score per user, best attempt per quiz, 15s display cache). Publishing is terminal - every write on a published quiz is a 400 and there is no unpublish route. See `devplacepy/services/quiz/CLAUDE.md` | | `/battles` | battles.py - **Opinion Wars**, week-long two-faction battles attached to posts: `GET ""` (listing, quizzes-style filters active/ended/mine + search), `GET /{uid}` (JSON-only full state, runs lazy resolution), `GET /{uid}/events?after=` (durable event trail replay), `POST /{uid}/join` (join or switch faction), `POST /{uid}/fight` (spend 25 Code Farm coins, deal level-weighted damage, 24h cooldown). The battle card renders on the parent post (no battle detail HTML page). See `devplacepy/services/opinionwar/CLAUDE.md` | -| `/game` | game/ package - the **Code Farm** idle game (`index.py` + `farm.py`): `GET /game` (page), `GET /game/state`, `GET /game/leaderboard?board=`, `POST /game/{plant,harvest,buy-plot,upgrade,fertilize,daily,grant,perk,prestige,legacy,mastery,quests/claim}`, `POST /game/{defense/upgrade,defense/downgrade,infrastructure/buy,cosmetics/buy,cosmetics/equip}`, plus social `GET /game/farm/{username}` and `POST /game/farm/{username}/{water,steal}`. See `devplacepy/services/game/CLAUDE.md` | +| `/game` | game/ package - the **Code Farm** idle game (`index.py` + `farm.py`): `GET /game` (page), `GET /game/state`, `GET /game/leaderboard?board=`, `POST /game/{plant,harvest,plot/stop,buy-plot,upgrade,fertilize,daily,grant,perk,prestige,legacy,mastery,quests/claim}`, `POST /game/{defense/upgrade,defense/downgrade,infrastructure/buy,cosmetics/buy,cosmetics/equip}`, plus social `GET /game/farm/{username}` and `POST /game/farm/{username}/{water,steal}`. See `devplacepy/services/game/CLAUDE.md` | | (none) | push.py - push + PWA: `GET /push.json` (VAPID public key + the providers accepting registrations; `apns` includes `environment` when active), `POST /push.json` (register with any active provider; a body without `provider` is a `webpush` body; an APNs body is `token` plus optional `client_id` for device-stable upsert; created/revived rows are probed and the JSON may include `delivered`/`error`; an unknown, disabled or unconfigured provider is a 400), `DELETE /push.json` (unregister exactly the one registration named by `endpoint`/`token`/`client_id`, same identity priority as registration; idempotent, always 200 `{unregistered}`; wired into `PushManager.js`'s logout-link interceptor so a webpush subscription is dropped before the browser navigates to `/auth/logout`), `GET /service-worker.js`, `GET /manifest.json`. Provider protocol and delivery live in `devplacepy/push/` - see `devplacepy/push/CLAUDE.md` | | (none) | docs.py (docs/ package) - the documentation site (prose pages + API reference). See `routers/docs/CLAUDE.md` for the deep detail on this tree (`DOCS_PAGES`, audience tiers, prose rendering pipeline) | | `/skills` | skills.py - per-user Agent Skills: `GET ""` (list), `GET /{name}` (detail), `POST /install` (fetch a `SKILL.md` from a URL, update in place), `POST /{name}/enabled`, `POST /{name}/delete` (owner, or an administrator naming `owner`). Private to the owner, `noindex`. See `devplacepy/services/skills/CLAUDE.md` | diff --git a/devplacepy/routers/admin/communities.py b/devplacepy/routers/admin/communities.py index 42f17e03..274d7ac6 100644 --- a/devplacepy/routers/admin/communities.py +++ b/devplacepy/routers/admin/communities.py @@ -1,6 +1,7 @@ # retoor import asyncio +import json import logging from datetime import datetime from typing import Annotated @@ -12,8 +13,8 @@ from fastapi.responses import HTMLResponse from devplacepy.database import get_setting, get_table, get_users_by_uids, resolve_object_url from devplacepy.database.sync import count_links, identity_map from devplacepy.responses import action_result, json_error, respond -from devplacepy.schemas import AdminCommunitiesOut, AdminCommunityLedgerOut, AdminCommunityOut -from devplacepy.models import CommunityAddForm, CommunityRunForm, CommunityUpdateForm +from devplacepy.schemas import AdminCommunitiesOut, AdminCommunityLedgerOut, AdminCommunityOut, AdminCommunitySchemaOut +from devplacepy.models import CommunityAddForm, CommunityRunForm, CommunitySchemaForm, CommunityUpdateForm from devplacepy.seo import base_seo_context, site_url, website_schema from devplacepy.services.audit import record as audit from devplacepy.services.community.discoverer import slug_for_query @@ -31,7 +32,8 @@ from devplacepy.services.community.ledger import ( from devplacepy.services.community.ledger_store import ledger_store from devplacepy.services.community.runner import TRIGGER_ADMIN, CommunityRunner from devplacepy.services.community.service import DRY_RUN_KEY -from devplacepy.services.community.spec import STATUS_LEARNING +from devplacepy.services.community.schemacheck import validate_schema +from devplacepy.services.community.spec import STATUS_LEARNING, STATUS_READY from devplacepy.services.community.store import iso, now, store from devplacepy.services.sync.gate import sync_enabled from devplacepy.services.sync.options import SyncOptions @@ -44,10 +46,11 @@ background_jobs: set[asyncio.Task] = set() LIST_URL = "/admin/communities" -def spawn(coroutine) -> None: +async def spawn(coroutine) -> None: task = asyncio.create_task(coroutine) background_jobs.add(task) task.add_done_callback(background_jobs.discard) + await asyncio.sleep(0) def service_flags() -> dict: @@ -58,8 +61,9 @@ def service_flags() -> dict: def public(row: dict) -> dict: - view = {key: value for key, value in row.items() if key != "login_password"} + view = {key: value for key, value in row.items() if key not in ("login_password", "login_totp_secret")} view["has_login"] = bool(row.get("login_password")) + view["has_totp"] = bool(row.get("login_totp_secret")) return view @@ -71,7 +75,7 @@ def require_community(slug: str) -> dict: def credentials_of(data) -> dict: - return {"username": data.login_username.strip(), "email": data.login_email.strip(), "password": data.login_password} + return {"username": data.login_username.strip(), "email": data.login_email.strip(), "password": data.login_password, "totp": data.login_totp_secret.strip()} def seo(request: Request, title: str, path: str, crumb: str): @@ -207,8 +211,9 @@ async def admin_community_add(request: Request, data: Annotated[CommunityAddForm login_username=secrets["username"], login_email=secrets["email"], login_password=secrets["password"], + login_totp_secret=secrets["totp"], ) - spawn(CommunityRunner().discover(data.query, admin["uid"], relearn_slug=slug)) + await spawn(CommunityRunner().discover(data.query, admin["uid"], relearn_slug=slug)) audit.record( request, "admin.community.add", @@ -226,6 +231,7 @@ async def admin_community_add(request: Request, data: Annotated[CommunityAddForm async def admin_community(request: Request, slug: str): admin = require_admin(request) row = require_community(slug) + store.reap_interrupted() source = files.read(row["slug"]) return respond( request, @@ -289,6 +295,7 @@ async def admin_community_ledger( "run": run_row, "run_errors": (run_row or {}).get("errors") or [], "run_log": (run_row or {}).get("log") or "", + "busy": (run_row["status"] == "running") if run_row else store.busy(row["uid"]), "verdict": {"level": run_row["verdict"], "title": LEVEL_TITLES.get(run_row["verdict"], ""), "headline": run_row["headline"]} if run_row else {}, "tiles": [ {"key": o.key, "label": o.label, "tone": o.tone, "help": o.help, "count": int(exact.get(o.key, 0)), "listed": int(listed.get(o.key, 0))} @@ -311,7 +318,7 @@ async def admin_community_run(request: Request, slug: str, data: Annotated[Commu row = require_community(slug) if store.busy(row["uid"]): return json_error(409, "A run is already in progress for this community") - spawn(CommunityRunner().run(row, dry_run=data.dry_run, trigger=TRIGGER_ADMIN)) + await spawn(CommunityRunner().run(row, dry_run=data.dry_run, trigger=TRIGGER_ADMIN)) audit.record( request, "admin.community.run", @@ -331,7 +338,7 @@ async def admin_community_relearn(request: Request, slug: str): row = require_community(slug) if store.busy(row["uid"]): return json_error(409, "A run is already in progress for this community") - spawn(CommunityRunner().discover(row["query"] or row["url"], admin["uid"], relearn_slug=slug)) + await spawn(CommunityRunner().discover(row["query"] or row["url"], admin["uid"], relearn_slug=slug)) audit.record( request, "admin.community.relearn", @@ -350,9 +357,9 @@ async def admin_community_update(request: Request, slug: str, data: Annotated[Co row = require_community(slug) changes = {"interval_seconds": data.interval_seconds, "enabled": int(data.enabled)} if data.clear_login: - changes.update(login_username="", login_email="", login_password="") + changes.update(login_username="", login_email="", login_password="", login_totp_secret="") else: - for column, value in (("login_username", data.login_username.strip()), ("login_email", data.login_email.strip()), ("login_password", data.login_password)): + for column, value in (("login_username", data.login_username.strip()), ("login_email", data.login_email.strip()), ("login_password", data.login_password), ("login_totp_secret", data.login_totp_secret.strip())): if value: changes[column] = value submitted = {name: getattr(data, name) for name in SyncOptions.__dataclass_fields__ if getattr(data, name) is not None} @@ -372,12 +379,104 @@ async def admin_community_update(request: Request, slug: str, data: Annotated[Co target_type="community", target_uid=row["uid"], target_label=slug, - metadata={"interval_seconds": data.interval_seconds, "enabled": data.enabled, "sync_options": changes.get("sync_options"), "login_changed": data.clear_login or bool(data.login_password or data.login_username or data.login_email)}, + metadata={"interval_seconds": data.interval_seconds, "enabled": data.enabled, "sync_options": changes.get("sync_options"), "login_changed": data.clear_login or bool(data.login_password or data.login_username or data.login_email or data.login_totp_secret)}, summary=f"{admin['username']} updated community {slug}", ) return action_result(request, f"{LIST_URL}/{slug}") +def schema_text(schema) -> str: + return json.dumps(schema, indent=2, ensure_ascii=False) + + +def schema_page(request: Request, admin: dict, row: dict, text: str, errors: list[str], notice: str, status_code: int): + context = { + **seo(request, f"{row['name'] or row['slug']} schema", f"{LIST_URL}/{row['slug']}/schema", "Communities"), + "request": request, + "user": admin, + "community": public(row), + "schema_text": text, + "has_previous": bool(row.get("schema_previous")), + "errors": errors, + "notice": notice, + "admin_section": "communities", + } + response = respond(request, "admin_community_schema.html", context, model=AdminCommunitySchemaOut) + response.status_code = status_code + return response + + +@router.get("/communities/{slug}/schema", response_class=HTMLResponse) +async def admin_community_schema(request: Request, slug: str): + admin = require_admin(request) + row = require_community(slug) + return schema_page(request, admin, row, schema_text(row.get("schema") or {}), [], "", 200) + + +@router.post("/communities/{slug}/schema") +async def admin_community_schema_save(request: Request, slug: str, data: Annotated[CommunitySchemaForm, Form()]): + admin = require_admin(request) + row = require_community(slug) + if store.busy(row["uid"]): + return json_error(409, "A run is already in progress for this community") + try: + schema = json.loads(data.schema_text) + except ValueError as exc: + return schema_page(request, admin, row, data.schema_text, [f"The text is not valid JSON: {exc}"], "", 400) + errors = validate_schema(schema) + if errors: + return schema_page(request, admin, row, data.schema_text, errors, "", 400) + verdict = await CommunityRunner().verify_schema(row, schema) + if not verdict.ok: + failed = ["The schema was checked against the live source and was not saved."] + verdict.problems[:10] + return schema_page(request, admin, row, data.schema_text, failed, "", 422) + store.update( + row["uid"], + schema=schema, + schema_previous=row.get("schema") or {}, + status=STATUS_READY, + last_error="", + failure_count=0, + enabled=1, + ) + files.write(row["slug"], schema) + audit.record( + request, + "admin.community.schema", + user=admin, + target_type="community", + target_uid=row["uid"], + target_label=slug, + metadata={"feeds": len(schema["feeds"]), "sample_posts": verdict.posts, "sample_comments": verdict.comments}, + summary=f"{admin['username']} replaced the learned schema of community {slug}", + ) + return action_result(request, f"{LIST_URL}/{slug}/schema", data={"posts": verdict.posts, "comments": verdict.comments}) + + +@router.post("/communities/{slug}/schema/restore") +async def admin_community_schema_restore(request: Request, slug: str): + admin = require_admin(request) + row = require_community(slug) + previous = row.get("schema_previous") or {} + if not previous: + return json_error(409, "There is no previous schema to restore") + errors = validate_schema(previous) + if errors: + return schema_page(request, admin, row, schema_text(previous), errors, "", 400) + store.update(row["uid"], schema=previous, schema_previous=row.get("schema") or {}) + files.write(row["slug"], previous) + audit.record( + request, + "admin.community.schema_restore", + user=admin, + target_type="community", + target_uid=row["uid"], + target_label=slug, + summary=f"{admin['username']} restored the previous schema of community {slug}", + ) + return action_result(request, f"{LIST_URL}/{slug}/schema") + + @router.post("/communities/{slug}/delete") async def admin_community_delete(request: Request, slug: str): admin = require_admin(request) diff --git a/devplacepy/routers/admin/nodes.py b/devplacepy/routers/admin/nodes.py index 75a1c3cb..d861b855 100644 --- a/devplacepy/routers/admin/nodes.py +++ b/devplacepy/routers/admin/nodes.py @@ -17,24 +17,35 @@ from devplacepy.database.nodes import ( add_event, count_open_requests, device_breakdown, + cache_stats, effective_status, get_device, + lifetime_counters, list_flags, + load_circuits, recent_devices, request_status_counts, resolve_flag, + set_class_override, suspend_device, unsuspend_device, usage_day_totals, ) from devplacepy.dependencies import json_or_form -from devplacepy.models import AdminNodeKillswitchForm, AdminNodeSettingsForm, AdminNodeSuspendForm +from devplacepy.models import ( + AdminNodeClassForm, + AdminNodeKillswitchForm, + AdminNodeSettingsForm, + AdminNodeSuspendForm, +) from devplacepy.responses import action_result, json_error, respond from devplacepy.routers.admin._shared import deny_senior, is_senior_admin from devplacepy.routers.nodes.social import SOCIAL_DAILY_DEFAULT from devplacepy.schemas import ( + AdminNodeCircuitOut, AdminNodeDeviceOut, AdminNodeFlagOut, + AdminNodeOutcomeOut, AdminNodePoolOut, AdminNodeQueueOut, AdminNodeSettingOut, @@ -43,6 +54,9 @@ from devplacepy.schemas import ( from devplacepy.seo import base_seo_context, site_url, website_schema from devplacepy.services.audit import record as audit from devplacepy.services.base import BaseService +from devplacepy.services.nodes.circuit import CircuitBreaker +from devplacepy.services.nodes.geo import shared_geo +from devplacepy.services.nodes.registry import OUTCOME_PREFIX from devplacepy.utils import not_found, require_admin logger = logging.getLogger(__name__) @@ -81,6 +95,8 @@ SETTING_FIELDS: tuple[NodeSettingField, ...] = ( NodeSettingField("nodes_cache_ttl_seconds", "Result cache lifetime (seconds)", LIMITS.cache_ttl_seconds, LIMITS.cache_ttl_seconds), NodeSettingField("nodes_daily_social_user", "Daily social searches per member", SOCIAL_DAILY_DEFAULT, QUOTA_SOCIAL_CEILING), NodeSettingField("nodes_social_via_nodes", "Route social search through nodes", 0, None, "flag"), + NodeSettingField("nodes_allow_unknown_class", "Allow donors whose address class is unknown", 0, None, "flag"), + NodeSettingField("nodes_redundancy_percent", "Redundancy sample of fetch jobs (percent)", LIMITS.redundancy_percent, LIMITS.redundancy_max_percent), ) SETTING_FIELDS_BY_KEY = {field.key: field for field in SETTING_FIELDS} @@ -151,6 +167,8 @@ def device_views(rows: list[dict], owners: dict) -> list[AdminNodeDeviceOut]: suspend_reason=row.get("suspend_reason") or "", country=row.get("country") or "", asn_class=row.get("asn_class") or "unknown", + asn=row.get("asn") or "", + class_override=row.get("class_override") or "", last_seen_at=row.get("last_seen_at") or None, created_at=row.get("created_at"), owner_uid=row.get("user_uid") or "", @@ -177,7 +195,44 @@ def flag_views(rows: list[dict], owners: dict, devices: dict) -> list[AdminNodeF ] +def circuit_views() -> list[AdminNodeCircuitOut]: + now = datetime.now(timezone.utc).timestamp() + breaker = CircuitBreaker() + views = [] + for row in load_circuits(): + age = max(0.0, now - float(row.get("opened_at") or 0.0)) + failures = int(row.get("failures") or 0) + if failures < breaker.failures: + state, left = "closed", 0 + elif age < breaker.open_seconds: + state, left = "open", int(breaker.open_seconds - age) + else: + state, left = "half_open", 0 + views.append(AdminNodeCircuitOut(key=row["key"], state=state, failures=failures, seconds_left=left)) + return views + + +def outcome_views(counters: dict[str, int]) -> list[AdminNodeOutcomeOut]: + grouped: dict[tuple[str, str], dict[str, int]] = {} + for name, value in counters.items(): + _, adapter, asn_class, kind = name.split(":") + grouped.setdefault((adapter, asn_class), {})[kind] = value + return [ + AdminNodeOutcomeOut( + adapter=adapter, + asn_class=asn_class, + ok=kinds.get("ok", 0), + blocked=kinds.get("blocked", 0), + failed=kinds.get("failed", 0), + total=sum(kinds.values()), + ) + for (adapter, asn_class), kinds in sorted(grouped.items()) + ] + + def overview() -> dict: + lifetime = lifetime_counters() + outcomes = {name: value for name, value in lifetime.items() if name.startswith(OUTCOME_PREFIX)} devices = recent_devices(ADMIN_LIST_LIMIT) flags = list_flags()[:ADMIN_LIST_LIMIT] device_by_uid = {row["uid"]: row for row in devices} @@ -199,6 +254,11 @@ def overview() -> dict: "devices": device_views(devices, owners), "settings": settings_view(), "limits": LIMITS.describe(), + "circuits": circuit_views(), + "outcomes": outcome_views(outcomes), + "lifetime": {name: value for name, value in lifetime.items() if name not in outcomes}, + "geo": shared_geo().status(), + "cache": cache_stats(), "admin_section": "nodes", } @@ -328,6 +388,39 @@ async def admin_nodes_suspend( return action_result(request, LIST_URL, data={"uid": device_uid, "minutes": data.minutes}) +@router.post("/nodes/devices/{device_uid}/class") +async def admin_nodes_class( + request: Request, + device_uid: str, + data: Annotated[AdminNodeClassForm, Depends(json_or_form(AdminNodeClassForm))], +): + admin = require_admin(request) + device = require_device(device_uid) + owner = owner_of(device) + if is_senior_admin(admin, owner): + return deny_senior(request, admin, owner["uid"], owner, "node.device.override", LIST_URL) + value = "allow" if data.value == "allow" else "" + before = device.get("class_override") or "" + set_class_override(device_uid, value) + audit.record( + request, + "node.device.override", + user=admin, + target_type="node_device", + target_uid=device_uid, + target_label=device.get("label") or device_uid, + old_value=before, + new_value=value, + metadata={"owner_uid": device.get("user_uid"), "asn_class": device.get("asn_class") or "unknown"}, + summary=( + f"admin {admin['username']} {'allowed' if value else 'cleared the override for'} " + f"node device {device.get('label') or device_uid} regardless of its address class" + ), + links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))], + ) + return action_result(request, LIST_URL, data={"uid": device_uid, "class_override": value}) + + @router.post("/nodes/devices/{device_uid}/unsuspend") async def admin_nodes_unsuspend(request: Request, device_uid: str): admin = require_admin(request) diff --git a/devplacepy/routers/game/index.py b/devplacepy/routers/game/index.py index 9c37383a..0fe82a81 100644 --- a/devplacepy/routers/game/index.py +++ b/devplacepy/routers/game/index.py @@ -12,6 +12,7 @@ from devplacepy.models import ( GameMasteryForm, GamePerkForm, GamePlantForm, + GamePlotStopForm, GameQuestForm, GameSlotForm, ) @@ -98,6 +99,27 @@ async def game_plant(request: Request, data: Annotated[GamePlantForm, Form()]): ) +@router.post("/plot/stop") +async def game_plot_stop(request: Request, data: Annotated[GamePlotStopForm, Form()]): + user = require_user(request) + + def recorded(result): + audit.record( + request, + "game.plot.stop", + user=user, + metadata=result, + summary=( + f"{user['username']} {'paused' if result['stop'] else 'resumed'} " + f"auto-replant on plot {result['slot']}" + ), + ) + + return await _respond_action( + request, user, lambda: store.set_plot_stop(user, data.slot, data.stop), recorded + ) + + @router.post("/harvest") async def game_harvest(request: Request, data: Annotated[GameSlotForm, Form()]): user = require_user(request) diff --git a/devplacepy/routers/nodes/ticket.py b/devplacepy/routers/nodes/ticket.py index 40d13c19..7a088a6f 100644 --- a/devplacepy/routers/nodes/ticket.py +++ b/devplacepy/routers/nodes/ticket.py @@ -36,7 +36,12 @@ async def node_ticket(request: Request): if device is None: return json_error(401, "Unknown device secret") if effective_status(device) != STATUS_ACTIVE: - return json_error(403, "This device is suspended", reason="suspended") + return json_error( + 403, + "This device is suspended", + reason="suspended", + until=(device.get("suspended_until") or "").strip(), + ) verdict = GATE.allows(device["user_uid"]) if not verdict.allowed: return json_error(403, verdict.message, reason=verdict.reason) diff --git a/devplacepy/routers/nodes/ws.py b/devplacepy/routers/nodes/ws.py index bf172fd4..68ca5830 100644 --- a/devplacepy/routers/nodes/ws.py +++ b/devplacepy/routers/nodes/ws.py @@ -4,6 +4,7 @@ import logging from fastapi import APIRouter, WebSocket, WebSocketDisconnect +from devplacepy.database import refresh_snapshot from devplacepy.database.nodes import ( STATUS_ACTIVE, add_event, @@ -16,6 +17,7 @@ from devplacepy.routers.nodes._shared import GATE, NODES_SCOPE from devplacepy.services.manager import service_manager from devplacepy.services.messaging.tickets import redeem_ticket_row from devplacepy.services.nodes.frames import FrameError +from devplacepy.services.nodes.geo import shared_geo from devplacepy.services.nodes.session import NodeSession from devplacepy.services.nodes.spec import ( CLOSE_CAPACITY, @@ -39,6 +41,7 @@ def _address(websocket: WebSocket) -> str: @router.websocket("/ws") async def node_ws(websocket: WebSocket, ticket: str = ""): await websocket.accept() + refresh_snapshot() service = service_manager.get_service("nodes") if service is None or not service.is_enabled(): await websocket.close(code=TRY_AGAIN_LATER) @@ -78,12 +81,24 @@ async def node_ws(websocket: WebSocket, ticket: str = ""): ended = "disconnect" try: hello = await session.handshake() + address = _address(websocket) + geo = shared_geo() + if not geo.available(): + counters.inc("geo_unavailable") + if not geo.warned: + geo.warned = True + logger.warning("node geo database is missing at %s, every donor is class unknown", geo.path) + found = geo.lookup(address) + session.asn_class = found.asn_class record_device_connection( device["uid"], platform=hello["platform"], client_version=hello["client_version"], protocol_version=hello["protocol"], - ip_hash=hash_ip(_address(websocket)), + country=found.country, + asn=str(found.asn) if found.asn else "", + asn_class=found.asn_class, + ip_hash=hash_ip(address), ) add_event("connect", device_uid=device["uid"], user_uid=device["user_uid"]) ended = await session.run() diff --git a/devplacepy/routers/sync/__init__.py b/devplacepy/routers/sync/__init__.py index 5e99488b..baa4d91e 100644 --- a/devplacepy/routers/sync/__init__.py +++ b/devplacepy/routers/sync/__init__.py @@ -2,10 +2,33 @@ from fastapi import APIRouter -from . import comments, posts, spec, users +from . import ( + attachments, + comments, + deletions, + export, + follows, + poll_votes, + polls, + posts, + projects, + reactions, + spec, + users, + votes, +) router = APIRouter() router.include_router(spec.router) router.include_router(users.router) +router.include_router(projects.router) router.include_router(posts.router) router.include_router(comments.router) +router.include_router(polls.router) +router.include_router(votes.router) +router.include_router(reactions.router) +router.include_router(follows.router) +router.include_router(poll_votes.router) +router.include_router(attachments.router) +router.include_router(export.router) +router.include_router(deletions.router) diff --git a/devplacepy/routers/sync/attachments.py b/devplacepy/routers/sync/attachments.py new file mode 100644 index 00000000..5ef78041 --- /dev/null +++ b/devplacepy/routers/sync/attachments.py @@ -0,0 +1,21 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request + +from devplacepy.dependencies import json_or_form +from devplacepy.routers.sync._shared import require_sync_caller, run_batch +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KIND_ATTACHMENTS + +router = APIRouter() + + +@router.post("/attachments") +async def sync_attachments( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))], +): + return run_batch(request, user, KIND_ATTACHMENTS, data) diff --git a/devplacepy/routers/sync/deletions.py b/devplacepy/routers/sync/deletions.py new file mode 100644 index 00000000..d1196a6a --- /dev/null +++ b/devplacepy/routers/sync/deletions.py @@ -0,0 +1,65 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse + +from devplacepy.dependencies import json_or_form +from devplacepy.responses import json_error +from devplacepy.routers.sync._shared import NO_STORE, require_sync_caller +from devplacepy.schemas.sync import SyncAckOut, SyncDeletionsOut +from devplacepy.services.audit import record as audit +from devplacepy.services.sync.deletions import DeletionCursorError, acknowledge, deletions_page +from devplacepy.services.sync.gate import sync_enabled +from devplacepy.services.sync.normalize import normalize_origin +from devplacepy.services.sync.records import SyncAckForm +from devplacepy.services.sync.spec import LIMITS + +router = APIRouter() + + +@router.get("/deletions") +async def sync_deletions( + request: Request, + origin: str, + user: dict = Depends(require_sync_caller), + cursor: str = "", + limit: int = LIMITS.deletions_page_size, +): + if not sync_enabled(): + return json_error(503, "Community sync is disabled") + try: + name = normalize_origin(origin) + page = deletions_page(name, cursor, limit) + except (DeletionCursorError, ValueError) as exc: + return json_error(400, str(exc)) + body = SyncDeletionsOut( + origin=name, + count=len(page.items), + has_more=page.has_more, + next_cursor=page.next_cursor, + items=page.items, + ) + return JSONResponse(body.model_dump(mode="json"), headers=NO_STORE) + + +@router.post("/deletions/ack") +async def sync_deletions_ack( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncAckForm, Depends(json_or_form(SyncAckForm))], +): + if not sync_enabled(): + return json_error(503, "Community sync is disabled") + done = acknowledge(data.origin, data.items) + audit.record( + request, + "sync.deletions.ack", + user=user, + target_type="sync", + target_label=data.origin, + metadata={"origin": data.origin, "items": len(data.items), "acknowledged": done}, + summary=f"{user['username']} acknowledged {done} local deletions for {data.origin}", + ) + return JSONResponse(SyncAckOut(origin=data.origin, acknowledged=done).model_dump(mode="json"), headers=NO_STORE) diff --git a/devplacepy/routers/sync/export.py b/devplacepy/routers/sync/export.py new file mode 100644 index 00000000..511bf2da --- /dev/null +++ b/devplacepy/routers/sync/export.py @@ -0,0 +1,52 @@ +# retoor + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse + +from devplacepy.responses import json_error +from devplacepy.routers.sync._shared import NO_STORE, require_sync_caller +from devplacepy.schemas.sync import SyncExportOut +from devplacepy.services.audit import record as audit +from devplacepy.services.sync.exporter import ExportError, export_page +from devplacepy.services.sync.gate import sync_enabled +from devplacepy.services.sync.normalize import instant_iso +from devplacepy.services.sync.spec import EXPORT_KINDS, LIMITS + +router = APIRouter() + + +@router.get("/export/{kind}") +async def sync_export( + request: Request, + kind: str, + user: dict = Depends(require_sync_caller), + since: str = "", + cursor: str = "", + limit: int = LIMITS.export_page_size, +): + if not sync_enabled(): + return json_error(503, "Community sync is disabled") + if kind not in EXPORT_KINDS: + return json_error(404, f"{kind} cannot be exported", reason="not_exportable", exportable=list(EXPORT_KINDS)) + try: + floor = instant_iso(since) if since else "" + page = export_page(kind, floor, cursor, limit) + except (ExportError, ValueError) as exc: + return json_error(400, str(exc)) + audit.record( + request, + "sync.export", + user=user, + target_type="sync", + target_label=kind, + metadata={"kind": kind, "count": len(page.records), "since": floor, "has_more": page.has_more}, + summary=f"{user['username']} exported {len(page.records)} {kind}", + ) + body = SyncExportOut( + kind=kind, + count=len(page.records), + has_more=page.has_more, + next_cursor=page.next_cursor, + records=page.records, + ) + return JSONResponse(body.model_dump(mode="json"), headers=NO_STORE) diff --git a/devplacepy/routers/sync/follows.py b/devplacepy/routers/sync/follows.py new file mode 100644 index 00000000..6df8dd1f --- /dev/null +++ b/devplacepy/routers/sync/follows.py @@ -0,0 +1,21 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request + +from devplacepy.dependencies import json_or_form +from devplacepy.routers.sync._shared import require_sync_caller, run_batch +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KIND_FOLLOWS + +router = APIRouter() + + +@router.post("/follows") +async def sync_follows( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))], +): + return run_batch(request, user, KIND_FOLLOWS, data) diff --git a/devplacepy/routers/sync/poll_votes.py b/devplacepy/routers/sync/poll_votes.py new file mode 100644 index 00000000..34312287 --- /dev/null +++ b/devplacepy/routers/sync/poll_votes.py @@ -0,0 +1,21 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request + +from devplacepy.dependencies import json_or_form +from devplacepy.routers.sync._shared import require_sync_caller, run_batch +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KIND_POLL_VOTES + +router = APIRouter() + + +@router.post("/poll_votes") +async def sync_poll_votes( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))], +): + return run_batch(request, user, KIND_POLL_VOTES, data) diff --git a/devplacepy/routers/sync/polls.py b/devplacepy/routers/sync/polls.py new file mode 100644 index 00000000..5bad40e6 --- /dev/null +++ b/devplacepy/routers/sync/polls.py @@ -0,0 +1,21 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request + +from devplacepy.dependencies import json_or_form +from devplacepy.routers.sync._shared import require_sync_caller, run_batch +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KIND_POLLS + +router = APIRouter() + + +@router.post("/polls") +async def sync_polls( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))], +): + return run_batch(request, user, KIND_POLLS, data) diff --git a/devplacepy/routers/sync/projects.py b/devplacepy/routers/sync/projects.py new file mode 100644 index 00000000..a8b51008 --- /dev/null +++ b/devplacepy/routers/sync/projects.py @@ -0,0 +1,21 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request + +from devplacepy.dependencies import json_or_form +from devplacepy.routers.sync._shared import require_sync_caller, run_batch +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KIND_PROJECTS + +router = APIRouter() + + +@router.post("/projects") +async def sync_projects( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))], +): + return run_batch(request, user, KIND_PROJECTS, data) diff --git a/devplacepy/routers/sync/reactions.py b/devplacepy/routers/sync/reactions.py new file mode 100644 index 00000000..db782b15 --- /dev/null +++ b/devplacepy/routers/sync/reactions.py @@ -0,0 +1,21 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request + +from devplacepy.dependencies import json_or_form +from devplacepy.routers.sync._shared import require_sync_caller, run_batch +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KIND_REACTIONS + +router = APIRouter() + + +@router.post("/reactions") +async def sync_reactions( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))], +): + return run_batch(request, user, KIND_REACTIONS, data) diff --git a/devplacepy/routers/sync/votes.py b/devplacepy/routers/sync/votes.py new file mode 100644 index 00000000..514e9456 --- /dev/null +++ b/devplacepy/routers/sync/votes.py @@ -0,0 +1,21 @@ +# retoor + +from typing import Annotated + +from fastapi import APIRouter, Depends, Request + +from devplacepy.dependencies import json_or_form +from devplacepy.routers.sync._shared import require_sync_caller, run_batch +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import KIND_VOTES + +router = APIRouter() + + +@router.post("/votes") +async def sync_votes( + request: Request, + user: Annotated[dict, Depends(require_sync_caller)], + data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))], +): + return run_batch(request, user, KIND_VOTES, data) diff --git a/devplacepy/schemas/__init__.py b/devplacepy/schemas/__init__.py index 6043bbb0..0b66c06e 100644 --- a/devplacepy/schemas/__init__.py +++ b/devplacepy/schemas/__init__.py @@ -116,12 +116,15 @@ from devplacepy.schemas.admin import ( AdminCommunitiesOut, AdminCommunityItemOut, AdminCommunityLedgerOut, + AdminCommunitySchemaOut, AdminCommunityOut, AdminDeviiTaskItemOut, AdminDeviiTasksOut, AdminGameOut, AdminMediaItemOut, + AdminNodeCircuitOut, AdminNodeDeviceOut, + AdminNodeOutcomeOut, AdminNodeFlagOut, AdminNodePoolOut, AdminNodeQueueOut, diff --git a/devplacepy/schemas/admin.py b/devplacepy/schemas/admin.py index 9e80bc51..2c8f968f 100644 --- a/devplacepy/schemas/admin.py +++ b/devplacepy/schemas/admin.py @@ -134,6 +134,7 @@ class AdminCommunityItemOut(_Out): login_username: str = "" login_email: str = "" has_login: bool = False + has_totp: bool = False class AdminCommunitiesOut(_Out): @@ -202,12 +203,22 @@ class AdminLedgerTileOut(_Out): listed: int = 0 +class AdminCommunitySchemaOut(_Out): + community: AdminCommunityItemOut + schema_text: str = "" + has_previous: bool = False + errors: list[str] = [] + notice: str = "" + admin_section: Optional[str] = None + + class AdminCommunityLedgerOut(_Out): community: AdminCommunityItemOut scope: str = "community" run: Optional[AdminRunSummaryOut] = None run_errors: list[str] = [] run_log: str = "" + busy: bool = False verdict: dict = {} tiles: list[AdminLedgerTileOut] = [] items: list[AdminLedgerItemOut] = [] @@ -228,6 +239,8 @@ class AdminNodeDeviceOut(_Out): suspend_reason: str = "" country: str = "" asn_class: str = "unknown" + asn: str = "" + class_override: str = "" last_seen_at: Optional[str] = None created_at: Optional[str] = None owner_uid: str = "" @@ -277,6 +290,22 @@ class AdminNodeQueueOut(_Out): expired: int = 0 +class AdminNodeCircuitOut(_Out): + key: str = "" + state: str = "closed" + failures: int = 0 + seconds_left: int = 0 + + +class AdminNodeOutcomeOut(_Out): + adapter: str = "" + asn_class: str = "unknown" + ok: int = 0 + blocked: int = 0 + failed: int = 0 + total: int = 0 + + class AdminNodesOut(_Out): enabled: bool = False pool: AdminNodePoolOut = AdminNodePoolOut() @@ -290,4 +319,9 @@ class AdminNodesOut(_Out): devices: list[AdminNodeDeviceOut] = [] settings: list[AdminNodeSettingOut] = [] limits: dict[str, int] = {} + circuits: list[AdminNodeCircuitOut] = [] + outcomes: list[AdminNodeOutcomeOut] = [] + lifetime: dict[str, int] = {} + geo: dict[str, Any] = {} + cache: dict[str, int] = {} admin_section: Optional[str] = None diff --git a/devplacepy/schemas/game.py b/devplacepy/schemas/game.py index 30f51f4a..eea66ba7 100644 --- a/devplacepy/schemas/game.py +++ b/devplacepy/schemas/game.py @@ -44,6 +44,8 @@ class GamePlotOut(_Out): fertilize_cost: int = 0 raided_fraction: float = 0.0 raided_pct: int = 0 + can_stop_replant: bool = False + stop_after_harvest: bool = False class GamePerkOut(_Out): diff --git a/devplacepy/schemas/sync.py b/devplacepy/schemas/sync.py index 6b37b796..dbeb7712 100644 --- a/devplacepy/schemas/sync.py +++ b/devplacepy/schemas/sync.py @@ -13,6 +13,8 @@ class SyncRecordResultOut(_Out): local_uid: str = "" errors: list[str] = [] ignored: list[str] = [] + needs_kind: str = "" + needs_uid: str = "" class SyncBatchOut(_Out): @@ -22,3 +24,32 @@ class SyncBatchOut(_Out): total: int = 0 counts: dict[str, int] = {} results: list[SyncRecordResultOut] = [] + released: dict[str, int] = {} + + +class SyncExportOut(_Out): + kind: str + count: int = 0 + has_more: bool = False + next_cursor: str = "" + records: list[dict] = [] + + +class SyncDeletionOut(_Out): + kind: str + origin_uid: str + by: str = "member" + deleted_at: str = "" + + +class SyncDeletionsOut(_Out): + origin: str + count: int = 0 + has_more: bool = False + next_cursor: str = "" + items: list[SyncDeletionOut] = [] + + +class SyncAckOut(_Out): + origin: str + acknowledged: int = 0 diff --git a/devplacepy/services/community/CLAUDE.md b/devplacepy/services/community/CLAUDE.md index 7cc91894..5095c858 100644 --- a/devplacepy/services/community/CLAUDE.md +++ b/devplacepy/services/community/CLAUDE.md @@ -19,6 +19,7 @@ This file documents community discovery (`devplacepy/services/community/`, `rout | Devii tools (`community_*`), docs group, privacy row | `services/devii/actions/catalog/community.py`, `docs_api/groups/community.py` | built, all administrator-only | | AI structure assistant | `ai.py` | built, optional, every proposal validated against the sample | | Login and credentials | `extract.find_login_form`, `runtime.Collector.login`, `discoverer.LoginLearner` | built | +| Checkpoints, schema validation | `checkpoint.py`, `schemacheck.py` | built, tested | ## Flow @@ -31,7 +32,7 @@ Feed types: `rss`, `atom`, `json_api`, `html_list`. Learned field maps use the * - **stealthii is a project dependency** (`pyproject.toml`, git URL; the Dockerfile installs `git`). `fetcher.Fetcher` shares one stealth Chromium per event loop (`SharedBrowser`), applies the SSRF guard (`guard_public_url`) before every request and every `page.goto`, and waits at least `delay_seconds` between requests to one host. Without the browser it falls back to the guarded plain client and logs that JavaScript rendering is unavailable. `CommunityService.on_disable` closes the browser. - **JavaScript pages.** If a page yields nothing statically, detection renders it (`Fetcher.render_html`, `page.content()`); the feed is then marked `"render": true` and the collector renders it on every run. The same fallback applies to post pages when comments only exist after rendering. - **Comments and post bodies from HTML pages.** For any feed (RSS, Atom, JSON, HTML list) whose posts expose an address, `PageProbe` opens up to three post pages, finds the repeated comment element (class hint `comment|reply|response|answer`, nesting from DOM containment, id from `::attr(id)` else a stable hash), and learns a `mode: "page"` comments spec. It also learns a `post.detail.content` selector when the post page carries more text than the listing. JSON feeds still prefer `inline`/`url` comment modes. -- **Login.** A community can store a username, an email and a password (columns `login_*`, never returned by any endpoint, admin form, Devii tool or CLI). Discovery finds the login form (page, login links, `/login`-style paths, rendered page), logs in with the session `community:`, then learns behind the wall. `schema.login` holds the form (action, field names, hidden defaults re-read at login for CSRF tokens, or selectors for a JavaScript page); `login_required` makes a missing account a clear `LoginRequired`. A rejected password raises `LoginFailed` and the community goes to `login_required`. Not supported: JSON-API logins, two-factor, captchas. +- **Login.** A community can store a username, an email and a password (columns `login_*`, never returned by any endpoint, admin form, Devii tool or CLI). Discovery finds the login form (page, login links, `/login`-style paths, rendered page), logs in with the session `community:`, then learns behind the wall. `schema.login` holds the form (action, field names, hidden defaults re-read at login for CSRF tokens, or selectors for a JavaScript page); `login_required` makes a missing account a clear `LoginRequired`. A rejected password raises `LoginFailed` and the community goes to `login_required`. Two-factor (TOTP) and JSON-API logins are supported; captchas are not (see Additions). - **AI.** When HTML heuristics are weak (no author, no title, content only raw text, no date) or find no item container, comment structure or post body, `AiAssistant` asks the internal gateway model (site setting `community_ai_model`, default `molodetz`; `community_ai=0` turns it off) for selectors. A proposal is accepted per field only if it matches at least 60% of the sampled elements. If the gateway is unreachable the learner continues with heuristics and logs it. - **The discussion page is not the article.** A feed can link to an external article while the discussion lives elsewhere (lobste.rs: RSS `` is the article, `` the discussion). The learner reads a `comments_url` field (a JSON/RSS key such as `comments`/`discussion`, or an HTML anchor whose text says "comments"/"replies"/"discuss") and learns the comment structure from those pages, while the full-text `detail` is still learned from the post `url`. `Collector.discussion_link` falls back to the post link. Measured on lobste.rs: 0 comments before, 325 comments for 100 posts after. - **Identifiers must be distinct.** For HTML lists `HtmlInferrer.first(distinct=True)` rejects an id/url selector that yields the same value for most items (the first `a[href]` of a lobste.rs story is the constant `/login` vote link, which collapsed every story into one identity and gave the title "1"). Anchor classes are tried by text length, and the title reuses the chosen anchor. If no distinct selector exists the feed is rejected, never learned with a constant id. @@ -49,7 +50,7 @@ Every run writes one row per record that needs an explanation, so an administrat - **Links.** Each row links to the synced record here (batched, at most three queries a page) and to the source (`Collector.sources`, the post link; a comment links to its post's source). A removed record has no local link. - **A refusal is not a failure.** See "Refusals are not failures"; a refused run is `skipped` with an `attention` verdict, not an error. -Gaps: the page does not live-refresh while a run is in progress (reload it); the run log and the ledger are written when the run finishes, so a run that is killed mid-way leaves neither; a text search across the whole community scans up to 40000 rows. +The ledger page live-refreshes while a run is busy (`AdminCommunity.js`, `data-busy`/`data-live` sections). A run writes a checkpoint every `checkpoint_seconds` (15, `checkpoint.py` `RunCheckpoint`): tally, log and ledger rows so far, so a killed run keeps its work. `CommunityStore.reap_interrupted` (called by the service and the admin page) marks a run that stopped heartbeating as `interrupted`. Unique index `(run_uid, seq)` keeps the checkpoint append idempotent. Community-wide text search of 3+ characters uses the FTS5 trigram table `community_run_items_fts` (insert/delete triggers, rebuilt when first created); run scope keeps LIKE. Measured on 40000 rows: no-match 34 ms to 0.4 ms, rare term 34.5 ms to 0.8 ms, a common term (10000+ hits) 9.4 ms to 21 ms (slower, accepted). ## Output: one engine, concrete file @@ -72,14 +73,23 @@ Gaps: the page does not live-refresh while a run is in progress (reload it); the Real run against `https://lobste.rs` (dry run, scratch database): RSS, JSON and HTML feeds detected, description/favicon/language/rules text learned, 25 posts and 24 authors collected from one page. Fixture tests cover RSS, Atom, JSON with inline comments and page pagination, and HTML lists. With stealthii and Chromium the same run learned lobste.rs from its `/rss` feed plus post-page comments (`li.comments_subtree`). Not measured: a real login-protected site, cursor pagination against a live site, the AI path against a live gateway (the local gateway rejected the scratch key, so heuristics ran). Fixtures cover JavaScript rendering, form login, a rejected password, RSS and HTML comments from post pages, feed deduplication and the AI validation. +## Additions since the first build + +- **Comment pagination** (`extract.find_next_page`, hints `next_link`, `page_param`, `json_next`, `none`, learned into `comments.pagination`). The runtime stops on a page with no new comments and honors `max_comment_pages`. +- **Logins.** TOTP second factor (`totp()` checked against the RFC 6238 vectors; secret in `communities.login_totp_secret`, never returned, `has_totp` only), JSON-API logins (`login.mode = json`, token header), OTP form detection. A captcha is detected (`detect_captcha`) and reported as `CaptchaRequired`; it is never solved. +- **Structured data.** Feed type `json_ld` (schema.org `Article`/`DiscussionForumPosting` blocks), detected by `FeedDetector.ld_candidate`. +- **Schema editor.** `/admin/communities/{slug}/schema` (GET and POST, plus `/schema/restore`) validates with `schemacheck.validate_schema` (ceilings, floors, selector syntax), verifies on a sample (`sample_limits`) before saving, keeps `schema_previous`, and regenerates `collectors.py`. Audit keys `admin.community.schema` and `admin.community.schema_restore`; Devii `community_schema_get` and `community_schema_set` (confirm gated). +- **Engagement.** The collector now emits votes, reactions and attachments where the source exposes them (`options.attachments`, `SyncOptions.collect_attachments`). While aliasing is on, votes and reactions are dropped (no identity for the voter) and attachments take the target's author. The generator is `VERSION = 2`; `files.load` regenerates an outdated collectors file and keeps its embedded schema. +- **Usernames.** `SyncPusher` runs batches with `humanize=True`, so a new account gets a realistic handle derived from the author (see "Realistic handles" in `devplacepy/services/sync/CLAUDE.md`); existing accounts keep their names. The collector also splits a feed author of the form ` by|via ` (Lobsters sends this for every story) with `extract.split_submitter`, so the person alone is the username and the identity key: one person sharing from many sites is one account with one link, and the stored handle always comes from the database after the first sight. Generator `VERSION = 3` regenerates existing collector files; because the key changed, `by`/`via` authors collected before this change do not match their old links and appear as new accounts on the next run (the old accounts are left as they are). +- **Cross-run dedupe.** `state["links"]`, capped at `max_remembered_links` (20000), skips a link already seen in an earlier run. + ## Gaps -- HTML post-page comments follow `rel=next` links only (`a[rel=next]` or `link[rel=next]`); a site that paginates comments any other way yields the first `max_comment_pages` it can reach, or the first page only. -- No two-factor, captcha or JSON-API login. -- HTML learning is heuristic plus AI; a site whose markup defeats both needs a hand-edited `SCHEMA`. -- Votes, reactions and attachments are not collected (the sync API does not take them). +- A site with no JSON-LD, feed, API or repeating structure needs a hand-written schema through the editor; ready-made Discourse and WordPress specs were not built. +- Captchas are detected, never solved; the run is reported and stops. - Cross-feed duplicate detection compares the sampled post addresses at discovery time only. -- Passwords are stored in plain text in the database, as chosen for now; they are masked everywhere they could be displayed. +- Not measured against a live login-protected site, live TOTP, or live cursor pagination; fixtures cover them. +- Passwords are stored in plain text in the database, as chosen for now (owner decision, excluded from the gap work); they are masked everywhere they could be displayed. ## Selection, aliasing, translation diff --git a/devplacepy/services/community/checkpoint.py b/devplacepy/services/community/checkpoint.py new file mode 100644 index 00000000..2fda09ac --- /dev/null +++ b/devplacepy/services/community/checkpoint.py @@ -0,0 +1,51 @@ +# retoor + +from __future__ import annotations + +import asyncio +import logging + +from devplacepy.services.community.ledger import LEVEL_RUNNING, tally_of +from devplacepy.services.community.logbook import LogBook +from devplacepy.services.community.spec import LIMITS + +logger = logging.getLogger("community.run") + + +class RunCheckpoint: + def __init__(self, store, ledgers, run_uid: str, community_uid: str, report, live, book: LogBook): + self.store = store + self.ledgers = ledgers + self.run_uid = run_uid + self.community_uid = community_uid + self.report = report + self.live = live + self.book = book + self.saved = 0 + self.writes = 0 + + def flush(self) -> None: + ledger = self.report.ledger + pushed = dict(self.live.counts) + self.saved += self.ledgers.append(self.run_uid, self.community_uid, ledger, self.saved) + self.store.update_run( + self.run_uid, + log=self.book.text(), + collected=self.report.collected, + pushed=pushed, + errors=list(self.report.errors), + tally=tally_of(pushed, ledger), + verdict=LEVEL_RUNNING, + headline="Running now.", + items_stored=self.saved, + items_dropped=ledger.dropped, + ) + self.writes += 1 + + async def loop(self) -> None: + while True: + await asyncio.sleep(LIMITS.checkpoint_seconds) + try: + self.flush() + except Exception: + logger.exception("run checkpoint failed, the run continues") diff --git a/devplacepy/services/community/discoverer.py b/devplacepy/services/community/discoverer.py index b3676180..64cb7b41 100644 --- a/devplacepy/services/community/discoverer.py +++ b/devplacepy/services/community/discoverer.py @@ -9,10 +9,10 @@ from dataclasses import dataclass, field from urllib.parse import urlsplit from devplacepy.services.community.ai import AiAssistant -from devplacepy.services.community.extract import canonical_link, field_text, find_login_form, parse_html, select_nodes +from devplacepy.services.community.extract import canonical_link, detect_captcha, field_text, find_login_form, json_or_none, leaf_paths, parse_html, select_nodes from devplacepy.services.community.feeds import FeedDetector from devplacepy.services.community.learner import MetadataLearner, StructureLearner -from devplacepy.services.community.runtime import Collector, LoginFailed, LoginRequired +from devplacepy.services.community.runtime import CaptchaRequired, Collector, LoginFailed, LoginRequired, SecondFactorRequired from devplacepy.services.community.search import SearchPhase from devplacepy.services.community.spec import ( LIMITS, @@ -57,6 +57,10 @@ def slug_for_query(query: str) -> str: return slug_of(origin_of(address)) if address else re.sub(r"[^a-z0-9]+", "-", query.lower()).strip("-")[:63] +JSON_LOGIN_PATHS = ("/api/login", "/api/auth/login", "/api/v1/auth/login", "/api/v1/login", "/auth/login", "/login.json", "/api/session", "/session", "/api/users/login") +TOKEN_KEYS = re.compile(r"^(token|access_token|accesstoken|jwt|id_token|auth_token|authtoken|bearer)$", re.I) +SUCCESS_KEYS = ("user", "data", "success", "ok", "session") +MAX_JSON_LOGIN_ATTEMPTS = 12 LOGIN_PATHS = ("/login", "/signin", "/sign-in", "/accounts/login", "/users/sign_in", "/auth/login", "/user/login") LOGIN_LABEL = re.compile(r"\b(log ?in|sign ?in)\b", re.I) MAX_LOGIN_PAGES = 6 @@ -78,9 +82,49 @@ class LoginLearner: return "" return text if status < 400 else "" + async def find_json(self, base_url: str, credentials: dict) -> dict | None: + origin = "{0.scheme}://{0.netloc}".format(urlsplit(base_url)) + by_email = bool(credentials.get("email")) and not credentials.get("username") + identity = credentials.get("email") if by_email else credentials.get("username") or credentials.get("email") + keys = ("email", "username", "login") if by_email else ("username", "login", "email") + attempts = 0 + for path in JSON_LOGIN_PATHS: + for key in keys: + if attempts >= MAX_JSON_LOGIN_ATTEMPTS: + return None + attempts += 1 + url = origin + path + try: + response = await self.fetcher.post(url, json={key: identity, "password": credentials["password"]}) + status, text = response.status, await response.text() + except Exception as exc: + logger.info("login: json probe %s failed: %s: %s", url, type(exc).__name__, exc) + break + if status in (404, 405) or status >= 500: + break + spec = self.json_spec(url, key, "email" if key == "email" else "username", status, text) + if spec: + logger.info("login: found a JSON login at %s", url) + return spec + return None + + @staticmethod + def json_spec(url: str, key: str, identity: str, status: int, text: str) -> dict | None: + document = json_or_none(text) + if status >= 400 or not isinstance(document, dict) or any(document.get(name) for name in ("error", "errors")): + return None + token = next((path for path, value in leaf_paths(document).items() if TOKEN_KEYS.match(path.rsplit(".", 1)[-1]) and isinstance(value, str) and value), "") + if not token and not any(document.get(name) for name in SUCCESS_KEYS): + return None + spec = {"mode": "json", "url": url, "username_field": key, "password_field": "password", "identity": identity} + if token: + spec.update(token_path=token, token_scheme="Bearer") + return spec + async def find(self, base_url: str, html: str) -> dict | None: spec = find_login_form(parse_html(html), base_url) if html else None if spec: + spec["captcha"] = detect_captcha(html) return spec urls = [] if html: @@ -92,9 +136,11 @@ class LoginLearner: urls = list(dict.fromkeys(urls))[:MAX_LOGIN_PAGES] for render in (False, True): for url in urls[: MAX_LOGIN_PAGES if not render else 2]: - spec = find_login_form(parse_html(await self.page(url, render)), url) + text = await self.page(url, render) + spec = find_login_form(parse_html(text), url) if spec: spec["render"] = render + spec["captcha"] = detect_captcha(text) logger.info("login: found a %s login form at %s%s", spec["mode"], url, " (needs JavaScript)" if render else "") return spec return None @@ -146,11 +192,20 @@ class CommunityDiscoverer: login = None if walled or self.has_credentials(): login = await self.logins.find(url, context.get("html", "")) + if login is None and self.has_credentials(): + login = await self.logins.find_json(url, self.credentials) if login and self.has_credentials(): probe = {"community": origin, "origin": origin, "base_url": url, "login": login, "limits": LIMITS.runtime()} try: await Collector(self.fetcher, probe, credentials=self.credentials).login() logger.info("discover: logged in to %s", origin) + except (CaptchaRequired, SecondFactorRequired) as exc: + result.error, result.status = str(exc), STATUS_LOGIN + result.url, result.origin, result.slug = url, origin, slug_of(origin) + result.name = result.name or origin + result.schema = self.schema(result, [], login_required=True, login=login) + logger.error("discover: %s", exc) + return False except LoginFailed as exc: result.error, result.status = str(exc), STATUS_FAILED logger.error("discover: %s", exc) @@ -175,7 +230,12 @@ class CommunityDiscoverer: if not feeds: if walled and login: result.status = STATUS_LOGIN - result.error = "the community requires a login; add the account's username or email and password and relearn" if not self.has_credentials() else "no feed could be learned even after logging in" + if login.get("captcha"): + result.error = "the login page of this community uses a captcha, which cannot be automated; sign in by hand or use a site that offers an API" + elif not self.has_credentials(): + result.error = "the community requires a login; add the account's username or email and password and relearn" + else: + result.error = "no feed could be learned even after logging in" result.schema = self.schema(result, [], login_required=True, login=login) logger.warning("discover: %s", result.error) else: diff --git a/devplacepy/services/community/extract.py b/devplacepy/services/community/extract.py index 3fbd01d2..fd30aec6 100644 --- a/devplacepy/services/community/extract.py +++ b/devplacepy/services/community/extract.py @@ -2,8 +2,13 @@ from __future__ import annotations +import base64 +import hashlib +import hmac import json import re +import struct +import time import xml.etree.ElementTree as ElementTree from datetime import datetime, timezone from email.utils import parsedate_to_datetime @@ -381,6 +386,16 @@ def block_text(node: Node) -> str: return "".join(parts) +SUBMITTER = re.compile(r"^\S+\s+(?:by|via)\s+(\S+?)(?:\s+\d{4}-\d{2}-\d{2}.*)?$", re.I) +BARE_SUBMITTER = re.compile(r"^(?:by|via)\s+(\S+?)(?:\s+\d{4}-\d{2}-\d{2}.*)?$", re.I) + + +def split_submitter(name: str) -> str: + text = " ".join(str(name or "").split()) + found = SUBMITTER.match(text) or BARE_SUBMITTER.match(text) + return found.group(1) if found else text + + def strip_tags(text: str) -> str: if "<" not in text: return text.strip() @@ -415,6 +430,84 @@ def leaf_paths(data, prefix: str = "$", depth: int = 0) -> dict[str, object]: return found +NEXT_PAGE_FIELDS = ( + "a[rel=next]::attr(href)", + "link[rel=next]::attr(href)", + "a.next::attr(href)", + "li.next a::attr(href)", + ".next a::attr(href)", + "a[class*=next]::attr(href)", + "a[aria-label^=Next]::attr(href)", +) +PAGINATION_CONTAINERS = ("nav", ".pagination", ".pager", ".paginator", "[class*=pagination]", "[class*=pager]") +NEXT_LABEL = re.compile( + r"^(?:next(?: page)?|older(?: comments| posts| replies)?|more(?: comments| replies)?|load more|volgende(?: pagina)?|weiter|suivant|siguiente|[\u203a\u00bb\u2192>]+|next\s*[\u203a\u00bb\u2192>]+|[\u203a\u00bb\u2192>]+\s*next)$", + re.I, +) +PAGE_NUMBER = re.compile(r"(?:[?&](?:page|p|pg|paged|pagenum|pageno|pn|cp)=(\d+))|(?:/page/(\d+))|(?:/p(\d+)/?$)", re.I) +PAGE_PARAM_NAMES = ("page", "p", "pg", "paged", "pagenum", "pageno", "pn", "cp") +UNUSABLE_HREF = ("#", "javascript:", "mailto:", "tel:") + + +def page_number(url: str) -> int | None: + match = PAGE_NUMBER.search(url) + if not match: + return None + return int(next(group for group in match.groups() if group)) + + +def with_page_number(url: str, number: int, param: str = "page") -> str: + if re.search(r"/page/\d+", url): + return re.sub(r"/page/\d+", f"/page/{number}", url, count=1) + parts = urlsplit(url) + query = [item for item in parts.query.split("&") if item and item.split("=")[0].lower() not in PAGE_PARAM_NAMES] + query.append(f"{param}={number}") + return parts._replace(query="&".join(query)).geturl() + + +def contained_in(node: Node, containers: list[Node]) -> bool: + wanted = {id(item) for item in containers} + cursor = node.parent + while cursor is not None: + if id(cursor) in wanted: + return True + cursor = cursor.parent + return False + + +def usable_href(href: str) -> bool: + return bool(href) and not href.strip().lower().startswith(UNUSABLE_HREF) + + +def find_next_page(root: Node, current: str, hint: dict | None = None) -> str: + hint = hint or {} + style = hint.get("style", "auto") + if style == "none": + return "" + if style == "next_link" and hint.get("next"): + value = html_field(root, hint["next"]) + return urljoin(current, value) if usable_href(value) else "" + if style == "page_param": + return with_page_number(current, (page_number(current) or 1) + 1, hint.get("param") or "page") + for spec in NEXT_PAGE_FIELDS: + value = html_field(root, spec) + if usable_href(value): + return urljoin(current, value) + anchors = [node for node in root.descendants() if node.tag == "a" and usable_href(node.attrs.get("href", ""))] + containers = [node for spec in PAGINATION_CONTAINERS for node in select_nodes(root, spec)] + for anchor in anchors: + label = anchor.text() or anchor.attrs.get("aria-label", "") or anchor.attrs.get("title", "") + if NEXT_LABEL.match(label.strip()) and (contained_in(anchor, containers) or not label.strip().isalpha()): + return urljoin(current, anchor.attrs["href"]) + here = urlsplit(current) + wanted = (page_number(current) or 1) + 1 + for anchor in anchors: + target = urljoin(current, anchor.attrs["href"]) + if urlsplit(target).path.rstrip("/").rsplit("/page/", 1)[0] == here.path.rstrip("/").rsplit("/page/", 1)[0] and page_number(target) == wanted: + return target + return "" + + IDENTITY_NAME = re.compile(r"(user|login|email|mail|account|identifier|handle|name)", re.I) SKIP_INPUT_TYPES = frozenset({"submit", "button", "image", "reset", "checkbox", "radio", "file"}) @@ -472,3 +565,139 @@ def find_login_form(root: Node, base_url: str) -> dict | None: "submit_selector": "button[type=submit], input[type=submit]", } return None + + +OTP_NAME = re.compile(r"(otp|totp|2fa|two[-_ ]?factor|one[-_ ]?time|verification[-_ ]?code|auth(?:enticator)?[-_ ]?code|security[-_ ]?code|mfa|token|^code$)", re.I) +CAPTCHA_MARKERS = re.compile(r"(g-recaptcha|grecaptcha|h-captcha|hcaptcha|cf-turnstile|turnstile\.js|captcha|funcaptcha|arkoselabs)", re.I) +TOTP_DIGITS = 6 +TOTP_STEP = 30 + + +def totp(secret: str, at: float | None = None, digits: int = TOTP_DIGITS, step: int = TOTP_STEP) -> str: + cleaned = re.sub(r"[\s-]", "", secret or "").upper() + cleaned += "=" * (-len(cleaned) % 8) + key = base64.b32decode(cleaned, casefold=True) + counter = int((time.time() if at is None else at) // step) + digest = hmac.new(key, struct.pack(">Q", counter), hashlib.sha1).digest() + offset = digest[-1] & 0x0F + value = struct.unpack(">I", digest[offset : offset + 4])[0] & 0x7FFFFFFF + return str(value % (10**digits)).zfill(digits) + + +def is_otp_input(node: Node) -> bool: + name = node.attrs.get("name", "") + if not name: + return False + if node.attrs.get("autocomplete", "").lower() == "one-time-code": + return True + kind = node.attrs.get("type", "text").lower() + return kind in ("text", "number", "tel") and bool(OTP_NAME.search(name) or OTP_NAME.search(node.attrs.get("id", ""))) + + +def find_otp_form(root: Node, base_url: str) -> dict | None: + for form in select_nodes(root, "form"): + inputs = [node for node in form.descendants() if node.tag == "input"] + field = next((node for node in inputs if is_otp_input(node)), None) + if field is None: + continue + return { + "action": urljoin(base_url, form.attrs.get("action") or base_url), + "method": (form.attrs.get("method") or "post").upper(), + "field": field.attrs["name"], + "fields": { + node.attrs["name"]: node.attrs.get("value", "") + for node in inputs + if node.attrs.get("name") + and node is not field + and node.attrs.get("type", "text").lower() not in SKIP_INPUT_TYPES + }, + } + return None + + +LD_POST_TYPES = frozenset( + {"article", "newsarticle", "blogposting", "discussionforumposting", "socialmediaposting", "techarticle", "report", "question", "answer", "qapage", "liveblogposting"} +) +LD_COMMENT_TYPES = frozenset({"comment", "answer"}) +LD_MAX_ITEMS = 200 +LD_MAX_DEPTH = 6 + + +def ld_types(node: dict) -> set[str]: + declared = node.get("@type") + values = declared if isinstance(declared, list) else [declared] + return {str(item).lower().rsplit("/", 1)[-1] for item in values if item} + + +def ld_text(value) -> str: + if isinstance(value, dict): + return ld_text(value.get("name") or value.get("@value") or value.get("text") or "") + if isinstance(value, list): + return ld_text(value[0]) if value else "" + return str(value or "").strip() + + +def ld_author(value) -> dict: + node = value[0] if isinstance(value, list) and value else value + if isinstance(node, dict): + name = ld_text(node.get("name") or node.get("alternateName") or node.get("identifier")) + identity = ld_text(node.get("@id") or node.get("url") or node.get("identifier")) + else: + name, identity = ld_text(node), "" + result = {} + if name: + result["username"] = name + if identity: + result["id"] = identity + return result + + +def ld_normalize(node: dict, depth: int = 0) -> dict: + content = ld_text(node.get("articleBody") or node.get("text") or node.get("description") or "") + normalized = { + "id": ld_text(node.get("@id") or node.get("identifier") or node.get("url")), + "url": ld_text(node.get("url") or node.get("mainEntityOfPage")), + "title": ld_text(node.get("headline") or node.get("name")), + "content": content, + "created_at": ld_text(node.get("datePublished") or node.get("dateCreated")), + "updated_at": ld_text(node.get("dateModified")), + "author": ld_author(node.get("author") or node.get("creator")), + } + raw_comments = node.get("comment") or node.get("suggestedAnswer") or node.get("acceptedAnswer") or [] + if isinstance(raw_comments, dict): + raw_comments = [raw_comments] + if depth < LD_MAX_DEPTH and isinstance(raw_comments, list): + normalized["comments"] = [ld_normalize(item, depth + 1) for item in raw_comments if isinstance(item, dict)][:LD_MAX_ITEMS] + return {key: value for key, value in normalized.items() if value not in ("", {}, [])} + + +def ld_walk(value, found: list, depth: int = 0) -> None: + if depth > LD_MAX_DEPTH or len(found) >= LD_MAX_ITEMS: + return + if isinstance(value, list): + for item in value: + ld_walk(item, found, depth + 1) + elif isinstance(value, dict): + if ld_types(value) & LD_POST_TYPES and not ld_types(value) <= LD_COMMENT_TYPES: + found.append(ld_normalize(value)) + return + for key in ("@graph", "itemListElement", "item", "mainEntity", "hasPart"): + if key in value: + ld_walk(value[key], found, depth + 1) + + +def extract_json_ld_posts(root: Node) -> list[dict]: + found: list[dict] = [] + for script in select_nodes(root, "script"): + if "ld+json" not in script.attrs.get("type", "").lower(): + continue + raw = "".join(chunk for chunk in script.chunks if isinstance(chunk, str)).strip() + try: + ld_walk(json.loads(raw), found) + except ValueError: + continue + return [item for item in found if item.get("id") and (item.get("title") or item.get("content"))] + + +def detect_captcha(text: str) -> bool: + return bool(CAPTCHA_MARKERS.search(text or "")) diff --git a/devplacepy/services/community/feeds.py b/devplacepy/services/community/feeds.py index 135e95d2..9d65b819 100644 --- a/devplacepy/services/community/feeds.py +++ b/devplacepy/services/community/feeds.py @@ -10,6 +10,7 @@ from urllib.parse import urljoin, urlsplit from devplacepy.services.community.extract import ( Node, + extract_json_ld_posts, json_or_none, parse_feed, parse_html, @@ -19,6 +20,7 @@ from devplacepy.services.community.spec import ( FEED_ATOM, FEED_HTML, FEED_JSON, + FEED_JSONLD, FEED_PROBE_PATHS, FEED_RSS, LIMITS, @@ -180,7 +182,7 @@ class FeedDetector: for path in FEED_PROBE_PATHS[: LIMITS.max_probe_paths]: await self.consider(urljoin(origin, path), found, context) if not found and root is not None: - candidate = await self.html_with_ai(base_url, text, root) + candidate = await self.html_with_ai(base_url, text, root) or self.ld_candidate(base_url, root) if candidate: found[base_url] = candidate if not found and not direct: @@ -202,7 +204,7 @@ class FeedDetector: return candidate = classify_text(url, body) if candidate is None and status < 300: - candidate = self.html_candidate(url, body) + candidate = self.html_candidate(url, body) or self.ld_candidate(url, parse_html(body) if "<" in body[:2000] else None) if candidate: logger.info("detect: %s feed at %s (%d sample items)", candidate.type, url, len(candidate.items)) found[url] = candidate @@ -225,7 +227,7 @@ class FeedDetector: return FeedCandidate(url, FEED_HTML, items=select_nodes(root, selector), document=root, item_selector=selector) def preference(self, candidate: FeedCandidate) -> int: - order = {FEED_JSON: 0, FEED_ATOM: 1, FEED_RSS: 2, FEED_HTML: 3} + order = {FEED_JSON: 0, FEED_ATOM: 1, FEED_RSS: 2, FEED_JSONLD: 3, FEED_HTML: 4} return order[candidate.type] def advertised(self, root: Node, base_url: str) -> list[str]: @@ -256,6 +258,14 @@ class FeedDetector: selector, nodes = found return FeedCandidate(url, FEED_HTML, items=nodes, document=root, item_selector=selector) + def ld_candidate(self, url: str, root: Node | None) -> FeedCandidate | None: + if root is None: + return None + items = extract_json_ld_posts(root) + if len(items) < 2: + return None + return FeedCandidate(url, FEED_JSONLD, items=items, document=items) + def looks_like_login(self, root: Node) -> bool: if select_nodes(root, "input[type=password]") and not find_item_selector(root): return True diff --git a/devplacepy/services/community/fetcher.py b/devplacepy/services/community/fetcher.py index e8a4eee7..9de74372 100644 --- a/devplacepy/services/community/fetcher.py +++ b/devplacepy/services/community/fetcher.py @@ -115,27 +115,35 @@ class Fetcher: if self.browser is None and self.client is None: await self.start() - async def get(self, url: str, params: dict | None = None, timeout: float | None = None, session: str | None = None): + async def get(self, url: str, params: dict | None = None, timeout: float | None = None, session: str | None = None, headers: dict | None = None): await self.ready() await guard_public_url(url) await self.polite(url) logger.info("GET %s params=%s", url, params or {}) if self.browser is not None: - response = await self.browser.get(url, params=params, timeout=timeout or LIMITS.timeout_seconds, session=session or self.session) + response = await self.browser.get(url, params=params, headers=headers, timeout=timeout or LIMITS.timeout_seconds, session=session or self.session) logger.info("GET %s -> %s", url, response.status) return response - raw = await self.client.get(url, params=params, timeout=timeout or LIMITS.timeout_seconds) + raw = await self.client.get(url, params=params, headers=headers, timeout=timeout or LIMITS.timeout_seconds) logger.info("GET %s -> %s", url, raw.status_code) return FetchResponse(raw) - async def post(self, url: str, data: dict | None = None, timeout: float | None = None, session: str | None = None): + async def post( + self, + url: str, + data: dict | None = None, + timeout: float | None = None, + session: str | None = None, + json: dict | None = None, + headers: dict | None = None, + ): await self.ready() await guard_public_url(url) await self.polite(url) - logger.info("POST %s fields=%s", url, sorted((data or {}).keys())) + logger.info("POST %s fields=%s", url, sorted((json or data or {}).keys())) if self.browser is not None: - return await self.browser.post(url, data=data, timeout=timeout or LIMITS.timeout_seconds, session=session or self.session) - return FetchResponse(await self.client.post(url, data=data, timeout=timeout or LIMITS.timeout_seconds)) + return await self.browser.post(url, data=data, json=json, headers=headers, timeout=timeout or LIMITS.timeout_seconds, session=session or self.session) + return FetchResponse(await self.client.post(url, data=data, json=json, headers=headers, timeout=timeout or LIMITS.timeout_seconds)) @asynccontextmanager async def page(self, session: str | None = None): diff --git a/devplacepy/services/community/files.py b/devplacepy/services/community/files.py index 28e89d4d..ae05b2e7 100644 --- a/devplacepy/services/community/files.py +++ b/devplacepy/services/community/files.py @@ -27,6 +27,10 @@ class CommunityFiles: path = self.collectors_path(slug) if not path.exists(): self.write(slug, schema) + else: + source = path.read_text() + if not CodeGenerator.is_current(source): + self.write(slug, CodeGenerator.embedded_schema(source) or schema) spec = importlib.util.spec_from_file_location(f"community_collectors_{slug.replace('-', '_')}", path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) diff --git a/devplacepy/services/community/generator.py b/devplacepy/services/community/generator.py index a9aaa529..222f6c5e 100644 --- a/devplacepy/services/community/generator.py +++ b/devplacepy/services/community/generator.py @@ -14,8 +14,8 @@ HEADER_LINE = re.compile(r"^# retoor <[^>]*>\n", re.M) ENTRYPOINT = ''' -async def collect_all(stealth, state=None, only=None, sleep=None, credentials=None): - collector = Collector(stealth, SCHEMA, state, sleep, credentials) +async def collect_all(stealth, state=None, only=None, sleep=None, credentials=None, options=None): + collector = Collector(stealth, SCHEMA, state, sleep, credentials, options) return await collector.run(only) {wrappers} @@ -53,19 +53,40 @@ if __name__ == "__main__": WRAPPER = ''' -async def collect_{name}(stealth, state=None, sleep=None, credentials=None): +async def collect_{name}(stealth, state=None, sleep=None, credentials=None, options=None): # {kind} feed {url} # post fields: {fields} # comments: {comments} # pagination: {pagination} - return await collect_all(stealth, state, only="{feed_id}", sleep=sleep, credentials=credentials) + return await collect_all(stealth, state, only="{feed_id}", sleep=sleep, credentials=credentials, options=options) ''' +EMBEDDED_SCHEMA = re.compile(r"^SCHEMA = json\.loads\(r'''(.*?)'''\)$", re.M | re.S) +VERSION_MARK = re.compile(r"^# generator-version: (\d+)$", re.M) + + class CodeGenerator: + VERSION = 3 + def __init__(self, schema: dict): self.schema = schema + @classmethod + def is_current(cls, source: str) -> bool: + found = VERSION_MARK.search(source) + return bool(found) and int(found.group(1)) == cls.VERSION + + @staticmethod + def embedded_schema(source: str) -> dict | None: + found = EMBEDDED_SCHEMA.search(source) + if not found: + return None + try: + return json.loads(found.group(1).replace("\\u0027", "'")) + except ValueError: + return None + @staticmethod def module_source(module) -> str: text = Path(module.__file__).read_text() @@ -94,6 +115,7 @@ class CodeGenerator: header = ( "# Generated by DevPlace community discovery. Community: " f"{self.schema.get('community')} (origin {self.schema.get('origin')}).\n" + f"# generator-version: {self.VERSION}\n" "# Self-contained: standard library plus stealthii when run directly.\n" "# A login-protected community reads COMMUNITY_USERNAME / COMMUNITY_EMAIL / COMMUNITY_PASSWORD from the environment.\n" "# Adjust SCHEMA below to change a selector or path; collect_all() reads it at runtime.\n" diff --git a/devplacepy/services/community/learner.py b/devplacepy/services/community/learner.py index 4bf48724..5dd54016 100644 --- a/devplacepy/services/community/learner.py +++ b/devplacepy/services/community/learner.py @@ -8,6 +8,7 @@ import statistics from urllib.parse import urljoin, urlsplit from devplacepy.services.community.extract import ( + NEXT_PAGE_FIELDS, NOW_TOKEN, Node, field_text, @@ -26,6 +27,7 @@ from devplacepy.services.community.feeds import FeedCandidate, find_comment_sele from devplacepy.services.community.spec import ( FEED_HTML, FEED_JSON, + FEED_JSONLD, LIMITS, ) @@ -56,6 +58,11 @@ COMMENT_LINK_TEXT = re.compile(r"\bcomments?\b|\breplies\b|\bdiscuss", re.I) MIN_TITLE_LENGTH = 5 STATIC_LINK_SPECS = ("h1 a::attr(href)", "h2 a::attr(href)", "h3 a::attr(href)", "a.title::attr(href)", "a.u-url::attr(href)") STATIC_TITLE_SPECS = ("h1", "h2", "h3", "a.title", "a.u-url") +VOTE_LIST_KEYS = re.compile(r"^(votes|upvotes|upvoters|likes|likers|voters)$", re.I) +REACTION_LIST_KEYS = re.compile(r"^(reactions|emoji_reactions|emojis)$", re.I) +VOTE_VALUE_KEYS = re.compile(r"^(value|direction|vote|dir|type|score)$", re.I) +EMOJI_KEYS = re.compile(r"^(emoji|reaction|reaction_type|reaction_emoji)$", re.I) +ENGAGEMENT_MIN_ENTRIES = 2 COMMENT_PROBES = ("{feed}/{id}/comments", "{feed}/{id}", "/api/posts/{id}/comments", "/api/comments?post_id={id}") @@ -255,6 +262,45 @@ class HtmlInferrer: return {"fields": {"username": [name]} if name else {}} +def learn_engagement(samples: list) -> dict: + inferrer = FieldInferrer([sample for sample in samples if isinstance(sample, dict)]) + found: dict = {} + for kind, pattern in (("votes", VOTE_LIST_KEYS), ("reactions", REACTION_LIST_KEYS)): + for path in inferrer.list_paths(pattern): + entries = [entry for sample in inferrer.samples for match in select(sample, path) if isinstance(match, list) for entry in match if isinstance(entry, dict)] + if len(entries) < ENGAGEMENT_MIN_ENTRIES: + continue + sub = FieldInferrer(entries) + author = sub.author_spec() + if not author["fields"].get("username"): + continue + fields: dict = {} + if kind == "reactions": + emoji = [candidate for candidate in sub.candidates(EMOJI_KEYS.pattern) if sub.coverage_of(candidate) >= sub.coverage] + if not emoji: + continue + fields["emoji"] = emoji[:1] + else: + value = [candidate for candidate in sub.candidates(VOTE_VALUE_KEYS.pattern) if sub.coverage_of(candidate) >= sub.coverage] + if value: + fields["value"] = value[:1] + moment = sub.pick_all("created_at", lambda text: normalize_instant(text) is not None)[:1] + if moment: + fields["created_at"] = moment + found[kind] = {"list_path": path, "fields": fields, "author": author} + break + return found + + +def json_next_path(document) -> str | None: + if not isinstance(document, dict): + return None + for path, value in leaf_paths(document).items(): + if NEXT_KEYS.match(last_segment(path)) and isinstance(value, str) and value.startswith(("http", "/", "?")): + return path + return None + + class PaginationProbe: def __init__(self, fetcher): self.fetcher = fetcher @@ -327,13 +373,14 @@ class CommentProbe: if any(isinstance(match, list) and match for sample in entries for match in select(sample, path)): spec["replies_path"] = path break + spec.update(learn_engagement(entries)) return spec def usable(self, spec: dict) -> bool: return "id" in spec["fields"] and "content" in spec["fields"] async def learn(self, candidate: FeedCandidate, post_fields: dict, inferrer: FieldInferrer | None) -> dict: - if candidate.type != FEED_JSON or inferrer is None or not candidate.items: + if candidate.type not in (FEED_JSON, FEED_JSONLD) or inferrer is None or not candidate.items: return {"mode": "none", "reason": f"comments are not learned for {candidate.type} feeds"} for path in inferrer.comment_lists(): entries = [entry for sample in candidate.items for match in select(sample, path) if isinstance(match, list) for entry in match if isinstance(entry, dict)] @@ -380,7 +427,11 @@ class CommentProbe: spec = self.comment_fields([entry for entry in entries if isinstance(entry, dict)]) if not self.usable(spec): return None - return {"mode": "url", "url_template": template, "list_path": path, **spec} + learned = {"mode": "url", "url_template": template, "list_path": path, **spec} + next_path = json_next_path(document) + if next_path: + learned["pagination"] = {"style": "json_next", "path": next_path} + return learned class PageProbe: @@ -486,9 +537,19 @@ class PageProbe: comments = self.ai_comment_spec(proposal, render) detail = await self.post_detail(candidate, post, post_links, links, roots, render) if comments["mode"] == "page": + declared = self.comment_pagination(roots) + if declared: + comments["pagination"] = declared logger.info("pages: comments via %s (%s)", comments["item_selector"], sorted(comments["fields"])) return comments, detail + @staticmethod + def comment_pagination(roots: list[Node]) -> dict | None: + for spec in NEXT_PAGE_FIELDS: + if any(html_field(root, spec) for root in roots): + return {"style": "next_link", "next": spec} + return None + async def post_detail(self, candidate: FeedCandidate, post: dict, post_links: list[str], links: list[str], roots: list[Node], render: bool) -> dict | None: if post_links == links: return await self.detail(candidate, post, roots, render) @@ -630,6 +691,8 @@ class StructureLearner: feed = self.as_feed(candidate, post, author, pagination) if detail: feed["post"]["detail"] = detail + if inferrer is not None: + feed["post"].update(learn_engagement(samples)) feed["id"] = f"feed{index + 1}" feed["comments"] = comments feed["post"]["source_footer"] = True diff --git a/devplacepy/services/community/ledger.py b/devplacepy/services/community/ledger.py index 9644daa5..4733f146 100644 --- a/devplacepy/services/community/ledger.py +++ b/devplacepy/services/community/ledger.py @@ -10,7 +10,16 @@ from devplacepy.services.sync.spec import ( KIND_COMMENTS, KIND_POSTS, KIND_USERS, + KINDS, + REASON_ACK_DELETED, REASON_CONTENT_REFUSED, + REASON_IMMUTABLE, + REASON_QUEUE_FULL, + REASON_SELF_FOLLOW, + REASON_UNKNOWN_OPTION, + REASON_UNKNOWN_POLL, + REASON_UNKNOWN_PROJECT, + REASON_UNKNOWN_TARGET, REASON_INTERNAL, REASON_INVALID, REASON_LOCALLY_DELETED, @@ -22,6 +31,7 @@ from devplacepy.services.sync.spec import ( REASON_UNKNOWN_PARENT, REASON_UNKNOWN_POST, SKIP_IDENTITY, + SKIP_ALIAS_ENGAGEMENT, SKIP_IDENTITY_CAP, SKIP_MIN_COMMENTS, SKIP_NO_AUTHOR, @@ -34,7 +44,7 @@ from devplacepy.services.sync.spec import ( ) KIND_FEED = "feed" -LEDGER_KINDS = (KIND_USERS, KIND_POSTS, KIND_COMMENTS, KIND_FEED) +LEDGER_KINDS = (*KINDS, KIND_FEED) OUTCOME_CREATED = "created" OUTCOME_UPDATED = "updated" @@ -43,6 +53,7 @@ OUTCOME_STALE = "stale" OUTCOME_SKIPPED = "skipped" OUTCOME_REJECTED = "rejected" OUTCOME_FAILED = "failed" +OUTCOME_DEFERRED = "deferred" OUTCOME_WARNING = "warning" OUTCOME_ERROR = "error" @@ -69,6 +80,8 @@ LEVEL_TITLES = { REASON_ARTICLE_UNREACHABLE = "article_unreachable" REASON_TRUNCATED = "truncated" REASON_COLLECT_ERROR = "collect_error" +REASON_ATTACHMENT_UNREACHABLE = "attachment_unreachable" +REASON_ATTACHMENT_TOO_LARGE = "attachment_too_large" @dataclass(frozen=True) @@ -101,6 +114,7 @@ OUTCOMES: tuple[Outcome, ...] = ( Outcome(OUTCOME_UPDATED, "Updated", TONE_GOOD, True, "Changed at the origin and updated here."), Outcome(OUTCOME_UNCHANGED, "Unchanged", TONE_MUTED, True, "Already synced and identical (only counted), or held for a reason such as deleted here (listed with the reason)."), Outcome(OUTCOME_STALE, "Stale", TONE_MUTED, True, "The origin sent an older version than the one already stored."), + Outcome(OUTCOME_DEFERRED, "Waiting", TONE_WARN, True, "Parked until a record it depends on arrives, then applied by itself."), Outcome(OUTCOME_SKIPPED, "Skipped", TONE_WARN, True, "Held back before sending by the selection, aliasing or translation rules."), Outcome(OUTCOME_WARNING, "Warning", TONE_WARN, True, "Synced, but something degraded on the way."), Outcome(OUTCOME_REJECTED, "Rejected", TONE_BAD, True, "Refused by validation, the content filter or an earlier moderation decision."), @@ -123,6 +137,14 @@ REASON_TEXT: dict[str, tuple[str, str]] = { REASON_LOCALLY_DELETED: ("Deleted here", "A moderator or the owner deleted it on this side. Moderation wins over the origin."), REASON_OLDER_THAN_SYNCED: ("Older than stored", "The origin sent a version older than the one already stored, so it was ignored."), REASON_TOMBSTONE_UNKNOWN: ("Deleted at the origin, never synced", "The origin reports a deletion for a record that was never synced here. Nothing to remove."), + REASON_UNKNOWN_TARGET: ("Target not synced", "The post, comment or project this vote, reaction or attachment belongs to is not synced here yet."), + REASON_UNKNOWN_POLL: ("Poll not synced", "The poll this ballot belongs to is not synced here yet."), + REASON_UNKNOWN_OPTION: ("Option unknown", "The poll has no such option, so the ballot cannot be counted."), + REASON_UNKNOWN_PROJECT: ("Project not synced", "The project this post links to is not synced here yet."), + REASON_SELF_FOLLOW: ("Follows itself", "An account cannot follow itself."), + REASON_IMMUTABLE: ("Cannot change", "An attachment cannot change. Send a new id, or a deletion."), + REASON_ACK_DELETED: ("Deleted here, acknowledged", "This record is gone on this side and the origin has acknowledged it, so it is not reported again."), + REASON_QUEUE_FULL: ("Waiting list full", "The waiting list for this origin is full, so the record was not parked."), REASON_INTERNAL: ("Internal error", "Something unexpected broke while processing this record. The run log has the traceback."), SKIP_SAMPLED_OUT: ("Sampled out", "Outside the percentage of new posts this community accepts. The same post always gets the same answer."), SKIP_MIN_COMMENTS: ("Too few comments", "Fewer comments were collected this run than the minimum you set."), @@ -132,9 +154,12 @@ REASON_TEXT: dict[str, tuple[str, str]] = { SKIP_IDENTITY_CAP: ("Identity cap reached", "The per-run limit of new pseudonyms was reached. Retried on the next run."), SKIP_TRANSLATION: ("Translation failed", "The translation did not come back or the per-run translation limit was reached. Retried on the next run."), SKIP_REWRITE: ("Rewrite failed", "Replacing original names failed. The item is dropped rather than stored under a real name. Retried on the next run."), + SKIP_ALIAS_ENGAGEMENT: ("Not sent under pseudonyms", "Votes and reactions name the people who gave them, so they are not sent while identity aliasing is on."), SKIP_PARENT: ("Parent was skipped", "The post or comment this depends on was skipped, so this one was too."), REASON_ARTICLE_UNREACHABLE: ("Article unreachable", "The original article could not be fetched. The post was imported with the text from the listing."), REASON_TRUNCATED: ("Cut to the limit", "The text was longer than the platform allows and was shortened. The source link is kept."), + REASON_ATTACHMENT_UNREACHABLE: ("Attachment unreachable", "A file linked from the post could not be downloaded, so it is not attached here."), + REASON_ATTACHMENT_TOO_LARGE: ("Attachment too large", "A file linked from the post is over the size limit, so it is not attached here."), REASON_COLLECT_ERROR: ("Source error", "A page or feed could not be read. The detail has the address and the answer."), } RETRIED = frozenset(TRANSIENT_SKIPS) @@ -156,7 +181,10 @@ def label_for(kind: str, record: dict) -> str: return clip(record.get("username"), LEDGER.label_chars) if kind == KIND_POSTS: return clip(record.get("title") or record.get("content"), LEDGER.label_chars) - return clip(record.get("content") or record.get("title") or record.get("username"), LEDGER.label_chars) + return clip( + record.get("content") or record.get("title") or record.get("username") or record.get("question") or record.get("filename") or record.get("emoji") or record.get("origin_uid"), + LEDGER.label_chars, + ) @dataclass @@ -250,8 +278,10 @@ def verdict(status: str, dry_run: bool, tally: dict) -> dict: return {"level": LEVEL_ATTENTION, "headline": "Not attempted: community sync is switched off or the origin is not allowed." + tail} if status == "failed": return {"level": LEVEL_BAD, "headline": "The run failed before anything could be synced." + tail} + if status == "interrupted": + return {"level": LEVEL_BAD, "headline": "Interrupted: the process ended before this run finished. What was synced up to then is listed." + tail} bad = [f"{tally[key]} {key}" for key in (OUTCOME_FAILED, OUTCOME_ERROR, OUTCOME_REJECTED) if tally.get(key)] - warn = [f"{tally[key]} {key}" for key in (OUTCOME_SKIPPED, OUTCOME_WARNING) if tally.get(key)] + warn = [f"{tally[key]} {key}" for key in (OUTCOME_DEFERRED, OUTCOME_SKIPPED, OUTCOME_WARNING) if tally.get(key)] if tally.get(OUTCOME_FAILED) or tally.get(OUTCOME_ERROR): return {"level": LEVEL_BAD, "headline": ", ".join(bad + warn).capitalize() + "." + tail} if bad or warn: diff --git a/devplacepy/services/community/ledger_store.py b/devplacepy/services/community/ledger_store.py index d1c3a1d7..c3cba079 100644 --- a/devplacepy/services/community/ledger_store.py +++ b/devplacepy/services/community/ledger_store.py @@ -10,6 +10,7 @@ from devplacepy.services.community.ledger import ( REASON_TEXT, Ledger, ) +from devplacepy.services.community.spec import LIMITS from devplacepy.services.community.store import ITEMS_TABLE, iso, now from devplacepy.utils.text import generate_uid @@ -21,9 +22,26 @@ def escape_like(text: str) -> str: return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") +SEARCH_TABLE = "community_run_items_fts" + + class LedgerStore: + def __init__(self) -> None: + self._search: bool | None = None + + def search_ready(self) -> bool: + if not self._search: + self._search = TABLE in db.tables and bool( + list(db.query("SELECT name FROM sqlite_master WHERE name = :name", name=SEARCH_TABLE)) + ) + return self._search + def save(self, run_uid: str, community_uid: str, ledger: Ledger) -> int: - if not ledger.items: + return self.append(run_uid, community_uid, ledger, 0) + + def append(self, run_uid: str, community_uid: str, ledger: Ledger, offset: int) -> int: + drafts = ledger.items[offset:] + if not drafts: return 0 stamp = iso(now()) rows = [ @@ -31,7 +49,7 @@ class LedgerStore: "uid": generate_uid(), "run_uid": run_uid, "community_uid": community_uid, - "seq": seq, + "seq": offset + seq, "rank": draft.rank, "kind": draft.kind, "outcome": draft.outcome, @@ -43,7 +61,7 @@ class LedgerStore: "detail": draft.detail, "created_at": stamp, } - for seq, draft in enumerate(ledger.items) + for seq, draft in enumerate(drafts) ] get_table(TABLE).insert_many(rows, chunk_size=INSERT_CHUNK) return len(rows) @@ -93,7 +111,10 @@ class LedgerStore: clauses.append("reason = :reason") params["reason"] = str(filters["reason"])[:80] text = str(filters.get("q") or "").strip()[:80] - if text: + if text and not run_uid and len(text) >= LIMITS.search_min_chars and self.search_ready(): + clauses.append(f"id IN (SELECT rowid FROM {SEARCH_TABLE} WHERE {SEARCH_TABLE} MATCH :q)") + params["q"] = '"' + text.replace('"', '""') + '"' + elif text: clauses.append("(label LIKE :q ESCAPE '\\' OR origin_uid LIKE :q ESCAPE '\\' OR detail LIKE :q ESCAPE '\\')") params["q"] = f"%{escape_like(text)}%" narrowed = "outcome" in params diff --git a/devplacepy/services/community/pusher.py b/devplacepy/services/community/pusher.py index ab2711fb..d05cc46a 100644 --- a/devplacepy/services/community/pusher.py +++ b/devplacepy/services/community/pusher.py @@ -10,7 +10,7 @@ from devplacepy.services.community.ledger import OUTCOME_CREATED, OUTCOME_UNCHAN from devplacepy.services.community.spec import LIMITS from devplacepy.services.sync.batch import BatchRunner from devplacepy.services.sync.records import SyncBatchForm -from devplacepy.services.sync.spec import KINDS, REASON_TOMBSTONE_UNKNOWN, STATUSES +from devplacepy.services.sync.spec import DEFERRABLE_REASONS, KINDS, REASON_TOMBSTONE_UNKNOWN, STATUSES logger = logging.getLogger("community.push") @@ -25,18 +25,12 @@ class PushReport: return sum(per.get(status, 0) for per in self.counts.values()) -DEPENDENCIES = { - "unknown_author": ("users", "author_uid"), - "unknown_post": ("posts", "post_uid"), - "unknown_parent": ("comments", "parent_uid"), -} - - class SyncPusher: - def __init__(self, origin: str, dry_run: bool = False, ledger: Ledger | None = None): + def __init__(self, origin: str, dry_run: bool = False, ledger: Ledger | None = None, report: PushReport | None = None): self.origin = origin self.dry_run = dry_run self.ledger = ledger + self.live = report def note(self, kind: str, result, raw: dict, outcome: str, reason: str | None = None) -> None: if self.ledger is None: @@ -56,23 +50,27 @@ class SyncPusher: @staticmethod def satisfied_by_projection(result, raw: dict, projected: dict) -> bool: - if result.status != "rejected" or result.reason not in DEPENDENCIES: + if result.status != "rejected" or result.reason not in DEFERRABLE_REASONS: return False - kind, field_name = DEPENDENCIES[result.reason] - return str(raw.get(field_name)) in projected[kind] + return result.needs_uid in projected.get(result.needs_kind, ()) def push(self, records: dict[str, list[dict]]) -> PushReport: - report = PushReport(dry_run=self.dry_run) + report = self.live or PushReport(dry_run=self.dry_run) + report.dry_run = self.dry_run mode = "DRY RUN" if self.dry_run else "LIVE" projected: dict[str, set[str]] = {kind: set() for kind in KINDS} for kind in KINDS: rows = records.get(kind, []) + if not rows: + report.counts[kind] = {} + continue tally: Counter = Counter() logger.info("push[%s]: %s %d %s record(s) in batches of %d", mode, self.origin, len(rows), kind, LIMITS.batch_records) for start in range(0, len(rows), LIMITS.batch_records): chunk = rows[start : start + LIMITS.batch_records] - outcome = BatchRunner(kind).run(SyncBatchForm(origin=self.origin, records=chunk, dry_run=self.dry_run)) + outcome = BatchRunner(kind, humanize=True).run(SyncBatchForm(origin=self.origin, records=chunk, dry_run=self.dry_run)) tally.update(outcome.counts) + report.counts[kind] = {status: tally.get(status, 0) for status in STATUSES if tally.get(status, 0)} for result, raw in zip(outcome.results, chunk): if result.status == "created": projected[kind].add(result.origin_uid) diff --git a/devplacepy/services/community/runner.py b/devplacepy/services/community/runner.py index e980ddc4..21a5ce43 100644 --- a/devplacepy/services/community/runner.py +++ b/devplacepy/services/community/runner.py @@ -7,16 +7,19 @@ import logging from dataclasses import dataclass, field from devplacepy.services.sync.aliaser import AliasRefused, Aliaser +from devplacepy.services.sync.healing import AliasHealer from devplacepy.services.sync.options import SyncOptions, alias_allowed from devplacepy.services.sync.selection import Selector +from devplacepy.services.community.checkpoint import RunCheckpoint from devplacepy.services.community.discoverer import CommunityDiscoverer, Discovery from devplacepy.services.community.fetcher import Fetcher from devplacepy.services.community.files import CommunityFiles, files as default_files from devplacepy.services.community.ledger import Ledger, tally_of, verdict from devplacepy.services.community.ledger_store import LedgerStore, ledger_store as default_ledger_store from devplacepy.services.community.logbook import LogBook -from devplacepy.services.community.pusher import SyncPusher -from devplacepy.services.community.runtime import LoginRequired +from devplacepy.services.community.pusher import PushReport, SyncPusher +from devplacepy.services.community.runtime import Collector, LoginRequired +from devplacepy.services.community.schemacheck import sample_limits from devplacepy.services.community.spec import ( LIMITS, RUN_DRY, @@ -29,6 +32,7 @@ from devplacepy.services.community.spec import ( ) from devplacepy.services.community.store import CommunityStore, iso, now, store as default_store from devplacepy.services.sync.gate import refusal +from devplacepy.services.sync.spec import SKIP_ALIAS_ENGAGEMENT, SKIP_PARENT logger = logging.getLogger("community.run") @@ -36,6 +40,20 @@ TRIGGER_CLI = "cli" TRIGGER_ADMIN = "admin" TRIGGER_SERVICE = "service" KINDS = ("users", "posts", "comments") +COLLECTED_KINDS = (*KINDS, "votes", "reactions", "attachments") +ENGAGEMENT_KINDS = ("votes", "reactions", "attachments") + + +def options_alias_active(row: dict) -> bool: + return SyncOptions.from_row(row).alias + + +@dataclass +class SchemaVerdict: + ok: bool + posts: int + comments: int + problems: list @dataclass @@ -49,6 +67,7 @@ class RunReport: run_uid: str = "" watermark: dict = field(default_factory=dict) ledger: Ledger = field(default_factory=Ledger) + live: PushReport = field(default_factory=lambda: PushReport(dry_run=False)) class CommunityRunner: @@ -120,6 +139,7 @@ class CommunityRunner: login_username=credentials.get("username", ""), login_email=credentials.get("email", ""), login_password=credentials.get("password", ""), + login_totp_secret=credentials.get("totp", ""), ) if existing: row = self.store.update(existing["uid"], **fields, enabled=1 if result.status == STATUS_READY else 0) @@ -130,9 +150,13 @@ class CommunityRunner: return row async def run(self, row: dict, *, dry_run: bool, trigger: str) -> RunReport: + self.store.reap_interrupted() run_uid = self.store.start_run(row["uid"], trigger, dry_run) report = RunReport(status=RUN_FAILED, dry_run=dry_run, run_uid=run_uid) + report.live.dry_run = dry_run with LogBook() as book: + checkpoint = RunCheckpoint(self.store, self.ledgers, run_uid, row["uid"], report, report.live, book) + ticker = asyncio.create_task(checkpoint.loop()) mode = "DRY RUN" if dry_run else "LIVE" logger.info("run[%s]: %s (%s) trigger=%s", mode, row["name"], row["origin"], trigger) try: @@ -152,12 +176,14 @@ class CommunityRunner: for error in report.errors: logger.error("run[%s]: %s", mode, error) logger.info("run[%s]: finished with status %s", mode, report.status) + ticker.cancel() + await asyncio.gather(ticker, return_exceptions=True) report.log = book.text() if not dry_run: self.settle(row, report) tally = tally_of(report.pushed, report.ledger) outcome = verdict(report.status, report.dry_run, tally) - stored = self.ledgers.save(run_uid, row["uid"], report.ledger) + stored = checkpoint.saved + self.ledgers.append(run_uid, row["uid"], report.ledger, checkpoint.saved) self.store.finish_run( run_uid, report.status, @@ -174,6 +200,27 @@ class CommunityRunner: self.store.prune_runs(row["uid"]) return report + async def verify_schema(self, row: dict, schema: dict) -> "SchemaVerdict": + sample = {**schema, "limits": sample_limits(schema)} + try: + async with self.fetcher_factory() as fetcher: + result = await asyncio.wait_for( + Collector(fetcher, sample, None, sleep=self.sleep, credentials=self.store.credentials(row)).run(), + timeout=LIMITS.run_timeout_seconds, + ) + except asyncio.TimeoutError: + return SchemaVerdict(False, 0, 0, [f"the verification run exceeded {LIMITS.run_timeout_seconds}s"]) + except LoginRequired as exc: + return SchemaVerdict(False, 0, 0, [str(exc)]) + except Exception as exc: + logger.exception("schema verification crashed") + return SchemaVerdict(False, 0, 0, [f"{type(exc).__name__}: {exc}"]) + posts, comments = len(result["records"]["posts"]), len(result["records"]["comments"]) + problems = list(result.get("errors", [])) + if not posts: + problems.append("the schema collected no posts from the live source") + return SchemaVerdict(bool(result["ok"]) and posts > 0, posts, comments, problems) + async def execute(self, row: dict, report: RunReport) -> None: refused = refusal(row["origin"]) if refused and not report.dry_run: @@ -183,10 +230,17 @@ class CommunityRunner: if refused: logger.warning("run[DRY RUN]: %s; continuing because nothing is written", refused) module = self.files.load(row["slug"], row["schema"]) + options = SyncOptions.from_row(row) async with self.fetcher_factory() as fetcher: - result = await module.collect_all(fetcher, row.get("watermark") or None, sleep=self.sleep, credentials=self.store.credentials(row)) + result = await module.collect_all( + fetcher, + row.get("watermark") or None, + sleep=self.sleep, + credentials=self.store.credentials(row), + options={"attachments": options.collect_attachments}, + ) records = result["records"] - report.collected = {kind: len(records[kind]) for kind in KINDS} + report.collected = {kind: len(records.get(kind, [])) for kind in COLLECTED_KINDS if records.get(kind) or kind in KINDS} logger.info("run: collected %s", report.collected) report.errors.extend(result.get("errors", [])) for warning in result.get("warnings", []): @@ -199,13 +253,14 @@ class CommunityRunner: if not result["ok"]: report.errors.append("no feed could be collected") return - options = SyncOptions.from_row(row) prepared = await self.prepare(row, options, records, report) if prepared is None: report.status = RUN_SKIPPED return records, skipped, held = prepared - push = await asyncio.to_thread(SyncPusher(row["origin"], dry_run=report.dry_run, ledger=report.ledger).push, records) + push = await asyncio.to_thread(SyncPusher(row["origin"], dry_run=report.dry_run, ledger=report.ledger, report=report.live).push, records) + if options_alias_active(row) and not report.dry_run: + self.heal(row["origin"]) report.pushed = push.counts | ({"skipped": dict(skipped)} if skipped else {}) report.errors.extend(push.problems[:20]) report.watermark = {} if held else result["state"] @@ -215,8 +270,20 @@ class CommunityRunner: failed = push.total("failed") report.status = RUN_DRY if report.dry_run else (RUN_PARTIAL if report.errors or failed else RUN_OK) + @staticmethod + def heal(origin: str) -> None: + healed = AliasHealer(origin).heal() + if healed.renamed: + logger.info("alias: %d alias(es) followed their clone, %d stored text(s) rewritten", len(healed.renamed), healed.rewritten) + if healed.conflicts: + logger.warning("alias: %d alias(es) could not follow their clone: %s", len(healed.conflicts), ", ".join(healed.conflicts)) + async def prepare(self, row: dict, options: SyncOptions, records: dict, report: RunReport): - selector = Selector(row["origin"], options) + selector = Selector(row["origin"], options, persist=not report.dry_run) + if options.alias and not report.dry_run: + self.heal(row["origin"]) + extras = {kind: records.get(kind, []) for kind in ENGAGEMENT_KINDS} + collected_users = list(records.get("users", [])) records, skipped = selector.select(records) for kind, record, reason in selector.dropped: report.ledger.add_dropped_record(kind, record, reason) @@ -235,10 +302,35 @@ class CommunityRunner: report.ledger.add_dropped_record(kind, record, reason) skipped.update(outcome.skipped) logger.info("alias: %d substitution(s), %d translated, %d new identities", outcome.substitutions, outcome.translated, outcome.identities_created) + records = self.attach_engagement(records, extras, collected_users, options, skipped, report) if skipped: logger.info("select: skipped %s", dict(skipped)) return records, skipped, held + @staticmethod + def attach_engagement(records: dict, extras: dict, collected_users: list, options: SyncOptions, skipped, report: RunReport) -> dict: + authors = {("post", str(post.get("origin_uid"))): post.get("author_uid") for post in records.get("posts", [])} + authors |= {("comment", str(comment.get("origin_uid"))): comment.get("author_uid") for comment in records.get("comments", [])} + merged = dict(records) + for kind in ENGAGEMENT_KINDS: + kept = [] + for record in extras.get(kind, []): + target = (record.get("target_kind"), str(record.get("target_uid"))) + if target not in authors: + skipped[SKIP_PARENT] += 1 + report.ledger.add_dropped_record(kind, record, SKIP_PARENT) + elif options.alias and kind != "attachments": + skipped[SKIP_ALIAS_ENGAGEMENT] += 1 + report.ledger.add_dropped_record(kind, record, SKIP_ALIAS_ENGAGEMENT) + else: + kept.append({**record, "author_uid": authors[target]} if kind == "attachments" and options.alias else record) + merged[kind] = kept + needed = {str(item.get(field)) for kind in ("votes", "reactions") for item in merged.get(kind, []) for field in ("voter_uid", "author_uid")} + if needed: + known = {str(user.get("origin_uid")) for user in merged.get("users", [])} + merged["users"] = [*merged.get("users", []), *[user for user in collected_users if str(user.get("origin_uid")) in needed - known]] + return merged + def settle(self, row: dict, report: RunReport) -> None: current = self.store.get(row["uid"]) if current is None: diff --git a/devplacepy/services/community/runtime.py b/devplacepy/services/community/runtime.py index f44a1ea3..1d259322 100644 --- a/devplacepy/services/community/runtime.py +++ b/devplacepy/services/community/runtime.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import base64 import hashlib import json import re @@ -12,8 +13,13 @@ from urllib.parse import urljoin from devplacepy.services.community.extract import ( Node, canonical_link, + detect_captcha, + extract_json_ld_posts, + split_submitter, field_text, find_login_form, + find_next_page, + find_otp_form, html_field, json_or_none, normalize_instant, @@ -23,6 +29,7 @@ from devplacepy.services.community.extract import ( select_nodes, strip_tags, text_at, + totp, ) UID_PATTERN = re.compile(r"^[A-Za-z0-9._:@-]+$") @@ -35,7 +42,11 @@ MAX_COMMENT_LENGTH = 125000 MAX_REPLY_DEPTH = 5 RETRY_STATUSES = frozenset({408, 425, 429, 500, 502, 503, 504}) RENDER_SETTLE_MS = 800 -NEXT_PAGE_FIELDS = ("a[rel=next]::attr(href)", "link[rel=next]::attr(href)") +ATTACHMENT_EXTENSIONS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf") +ATTACHMENT_TYPES = ("image/", "application/pdf") +HTML_FILE_URL = re.compile(r"""(?:src|href|data-src|data-original)\s*=\s*["']([^"'\s>]+)["']""", re.I) +DOWN_VOTES = re.compile(r"^(-1|-|down|downvote|dislike|negative|minus)$", re.I) +DEFAULT_OPTIONS = {"attachments": False, "max_attachments": 40, "max_attachments_per_post": 4, "max_attachment_bytes": 2_000_000, "max_attachment_total": 25_000_000} DEFAULT_LIMITS = { "max_pages": 3, "max_posts": 150, @@ -47,6 +58,7 @@ DEFAULT_LIMITS = { "backoff_seconds": 2.0, "timeout_seconds": 30, "max_bytes": 5_000_000, + "max_remembered_links": 20_000, } @@ -62,10 +74,34 @@ class LoginFailed(LoginRequired): pass +class CaptchaRequired(LoginRequired): + pass + + +class SecondFactorRequired(LoginFailed): + pass + + class PageMissing(CollectError): pass +CODE_KEYS = ("otp_required", "totp_required", "two_factor", "two_factor_required", "requires_2fa", "mfa_required", "second_factor", "needs_otp") +CODE_PHRASES = re.compile(r"(one[- ]time (?:code|password)|verification code|authenticator|two[- ]factor|2fa code|otp)", re.I) + + +def code_challenge(document, text: str, status: int) -> bool: + if isinstance(document, dict): + lowered = {str(key).lower(): value for key, value in document.items()} + if any(lowered.get(key) for key in CODE_KEYS): + return True + return status in (400, 401, 403, 422) and bool(CODE_PHRASES.search(text or "")) + + +def url_hint(feed: dict) -> str: + return feed.get("url") or "" + + def uid_for(value) -> str: text = str(value if value is not None else "").strip() if text and len(text) <= UID_MAX_LENGTH and UID_PATTERN.match(text): @@ -88,10 +124,16 @@ def flatten_matches(matches: list) -> list: class Collector: - def __init__(self, stealth, schema: dict, state: dict | None = None, sleep=None, credentials: dict | None = None): + def __init__(self, stealth, schema: dict, state: dict | None = None, sleep=None, credentials: dict | None = None, options: dict | None = None): self.stealth = stealth + self.options = {**DEFAULT_OPTIONS, **(options or {})} + self.votes: dict[str, dict] = {} + self.reactions: dict[str, dict] = {} + self.attachments: list[dict] = [] + self.file_candidates: list[dict] = [] self.schema = schema - self.state = {"feeds": dict((state or {}).get("feeds", {}))} + remembered = (state or {}).get("links") or {} + self.state = {"feeds": dict((state or {}).get("feeds", {})), "links": dict(remembered)} self.sleep = sleep or asyncio.sleep self.credentials = credentials or {} self.limits = {**DEFAULT_LIMITS, **schema.get("limits", {})} @@ -105,8 +147,13 @@ class Collector: self.warnings: list[str] = [] self.notices: list[dict] = [] self.sources: dict[str, str] = {} - self.link_owner: dict[str, tuple[str, str]] = {} + self.link_owner: dict[str, tuple[str, str]] = { + key: (value[0], value[1]) + for key, value in remembered.items() + if isinstance(value, (list, tuple)) and len(value) == 2 + } self.aliases: dict[str, str] = {} + self.headers: dict[str, str] = {} async def attempt(self, label: str, action) -> str: last = "no attempt" @@ -138,18 +185,22 @@ class Collector: return await self.attempt(url, lambda: self.render_html(url)) async def action(): - response = await self.stealth.get(url, params=params, timeout=self.limits["timeout_seconds"], session=self.session) + response = await self.stealth.get(url, params=params, timeout=self.limits["timeout_seconds"], session=self.session, headers=self.headers or None) return int(response.status), await response.text() return await self.attempt(url, action) async def post(self, url: str, data: dict) -> str: async def action(): - response = await self.stealth.post(url, data=data, timeout=self.limits["timeout_seconds"], session=self.session) + response = await self.stealth.post(url, data=data, timeout=self.limits["timeout_seconds"], session=self.session, headers=self.headers or None) return int(response.status), await response.text() return await self.attempt(url, action) + async def post_json(self, url: str, body: dict) -> tuple[int, str]: + response = await self.stealth.post(url, json=body, timeout=self.limits["timeout_seconds"], session=self.session, headers=self.headers or None) + return int(response.status), await response.text() + async def render_html(self, url: str): async with self.stealth.page(session=self.session) as page: response = await page.goto(url, wait_until="domcontentloaded") @@ -165,6 +216,9 @@ class Collector: def has_credentials(self) -> bool: return bool(self.credentials.get("password") and (self.credentials.get("username") or self.credentials.get("email"))) + def community_name(self) -> str: + return self.schema.get("community", "community") + async def login(self) -> bool: spec = self.schema.get("login") if not spec or not self.has_credentials(): @@ -172,26 +226,77 @@ class Collector: identity = self.credentials.get("email") if spec.get("identity") == "email" else self.credentials.get("username") identity = identity or self.credentials.get("username") or self.credentials.get("email") render = bool(spec.get("render")) + if spec.get("captcha"): + raise CaptchaRequired(f"{self.community_name()} protects its login with a captcha, which cannot be automated; sign in by hand or use a site that offers an API") + if spec.get("mode") == "json": + await self.login_json(spec, identity) + return True if spec.get("mode") == "form": - root = parse_html(await self.fetch(spec["url"], render=render)) - form = find_login_form(root, spec["url"]) or spec - data = {**form.get("fields", {}), form["username_field"]: identity, form["password_field"]: self.credentials["password"]} - if str(form.get("method", "POST")).upper() == "GET": - await self.fetch(form["action"], params=data) - else: - await self.post(form["action"], data) + await self.login_form(spec, identity, render) else: async with self.stealth.page(session=self.session) as page: await page.goto(spec["url"], wait_until="domcontentloaded") + if detect_captcha(await page.content()): + raise CaptchaRequired(f"{self.community_name()} protects its login with a captcha, which cannot be automated") await page.fill(spec["username_selector"], identity) await page.fill(spec["password_selector"], self.credentials["password"]) await page.click(spec["submit_selector"]) await page.wait_for_timeout(RENDER_SETTLE_MS * 2) - after = parse_html(await self.fetch(spec.get("check_url") or self.schema.get("base_url") or spec["url"], render=render)) - if select_nodes(after, "input[type=password]"): - raise LoginFailed(f"{self.schema.get('community', 'community')} rejected the stored credentials") + after_text = await self.fetch(spec.get("check_url") or self.schema.get("base_url") or spec["url"], render=render) + if select_nodes(parse_html(after_text), "input[type=password]"): + if detect_captcha(after_text): + raise CaptchaRequired(f"{self.community_name()} asked for a captcha while signing in, which cannot be automated") + raise LoginFailed(f"{self.community_name()} rejected the stored credentials") return True + async def login_form(self, spec: dict, identity: str, render: bool) -> None: + page_text = await self.fetch(spec["url"], render=render) + if detect_captcha(page_text): + raise CaptchaRequired(f"{self.community_name()} protects its login with a captcha, which cannot be automated; sign in by hand or use a site that offers an API") + form = find_login_form(parse_html(page_text), spec["url"]) or spec + data = {**form.get("fields", {}), form["username_field"]: identity, form["password_field"]: self.credentials["password"]} + if str(form.get("method", "POST")).upper() == "GET": + answer = await self.fetch(form["action"], params=data) + else: + answer = await self.post(form["action"], data) + await self.second_factor(answer, form["action"]) + + async def second_factor(self, answer: str, base_url: str) -> None: + form = find_otp_form(parse_html(answer), base_url) + if form is None: + return + secret = self.credentials.get("totp") + if not secret: + raise SecondFactorRequired( + f"{self.community_name()} asks for a one-time code; add the account's authenticator secret (TOTP) to the community. A code sent by SMS or push cannot be automated" + ) + data = {**form["fields"], form["field"]: totp(secret)} + reply = await (self.fetch(form["action"], params=data) if form["method"] == "GET" else self.post(form["action"], data)) + if find_otp_form(parse_html(reply), form["action"]): + raise SecondFactorRequired(f"{self.community_name()} rejected the one-time code; check the authenticator secret and the clock of this server") + + async def login_json(self, spec: dict, identity: str) -> None: + body = {**spec.get("fields", {}), spec["username_field"]: identity, spec["password_field"]: self.credentials["password"]} + status, text = await self.post_json(spec["url"], body) + document = json_or_none(text) + if code_challenge(document, text, status): + secret = self.credentials.get("totp") + if not secret: + raise SecondFactorRequired( + f"{self.community_name()} asks for a one-time code; add the account's authenticator secret (TOTP) to the community. A code sent by SMS or push cannot be automated" + ) + status, text = await self.post_json(spec["url"], {**body, spec.get("otp_field") or "otp": totp(secret)}) + document = json_or_none(text) + if code_challenge(document, text, status): + raise SecondFactorRequired(f"{self.community_name()} rejected the one-time code; check the authenticator secret and the clock of this server") + if status >= 400: + raise LoginFailed(f"{self.community_name()} rejected the stored credentials (status {status})") + token = text_at(document, spec["token_path"]) if spec.get("token_path") and document is not None else "" + if spec.get("token_path") and not token: + raise LoginFailed(f"{self.community_name()} answered the login without the expected token") + if token: + self.headers["Authorization"] = f"{spec.get('token_scheme') or 'Bearer'} {token}" + def parse(self, feed: dict, text: str): kind = feed["type"] if kind == "json_api": @@ -199,6 +304,9 @@ class Collector: if document is None: raise CollectError(f"{feed['url']} did not return JSON") return document, flatten_matches(select(document, feed.get("list_path") or "$")) + if kind == "json_ld": + items = extract_json_ld_posts(parse_html(text)) + return items, items if kind in ("rss", "atom"): _, meta, items = parse_feed(text) return {"meta": meta}, items @@ -347,6 +455,7 @@ class Collector: return owner[0] if key: self.link_owner.setdefault(key, (post_uid, feed["id"])) + self.remember_link(key, self.link_owner[key]) record = { "origin_uid": post_uid, "author_uid": self.add_author(feed.get("author") or {}, item, created), @@ -357,15 +466,24 @@ class Collector: if updated: record["updated_at"] = updated self.posts[post_uid] = record + self.add_engagement(feed["post"], item, "post", post_uid, created) + self.note_files(item, detail, "post", post_uid, record["author_uid"], created, link or url_hint(feed)) if link: self.sources[post_uid] = link if cut: self.notice("posts", post_uid, "truncated", title, f"{cut} characters over the {MAX_POST_LENGTH} limit were cut") return post_uid + def remember_link(self, key: str, owner: tuple[str, str]) -> None: + links = self.state["links"] + links.pop(key, None) + links[key] = list(owner) + while len(links) > int(self.limits["max_remembered_links"]): + links.pop(next(iter(links))) + def add_author(self, spec: dict, item, moment: str) -> str: fields = spec.get("fields", {}) - username = strip_tags(field_text(item, fields.get("username"))) + username = split_submitter(strip_tags(field_text(item, fields.get("username")))) identifier = field_text(item, fields.get("id")) if not username and not identifier: username, identifier = "anonymous", "anonymous" @@ -395,9 +513,7 @@ class Collector: self.add_comments(spec, flatten_matches(select(item, spec["list_path"])), post_uid, None, 0) elif mode == "url": target = spec["url_template"].replace("{id}", identifier).replace("{uid}", post_uid) - document = json_or_none(await self.fetch(urljoin(base, target), render=bool(spec.get("render")))) - entries = flatten_matches(select(document, spec.get("list_path") or "$")) if document is not None else [] - self.add_comments(spec, entries, post_uid, None, 0) + await self.collect_comment_documents(spec, urljoin(base, target), post_uid) elif mode == "page": link = self.discussion_link(feed, item, base) if link: @@ -409,6 +525,22 @@ class Collector: except CollectError as exc: self.errors.append(f"comments {identifier}: {exc}") + async def collect_comment_documents(self, spec: dict, link: str, post_uid: str) -> None: + visited: set[str] = set() + pagination = spec.get("pagination") or {} + for _ in range(int(self.limits["max_comment_pages"])): + if link in visited: + break + visited.add(link) + document = json_or_none(await self.fetch(link, render=bool(spec.get("render")))) + entries = flatten_matches(select(document, spec.get("list_path") or "$")) if document is not None else [] + before = len(self.comments) + self.add_comments(spec, entries, post_uid, None, 0) + target = text_at(document, pagination["path"]) if pagination.get("style") == "json_next" and document is not None else "" + if len(self.comments) == before or not target: + break + link = urljoin(link, target) + async def collect_comment_pages(self, feed: dict, spec: dict, link: str, post_uid: str) -> None: render = bool(feed.get("render") or spec.get("render")) visited: set[str] = set() @@ -417,11 +549,13 @@ class Collector: break visited.add(link) root = await self.page_root(link, render) + before = len(self.comments) self.add_page_comments(spec, root, post_uid) - target = next((value for value in (html_field(root, field) for field in NEXT_PAGE_FIELDS) if value), "") - if not target: + if len(self.comments) == before: + break + link = find_next_page(root, link, spec.get("pagination")) + if not link: break - link = urljoin(link, target) def add_comments(self, spec: dict, entries: list, post_uid: str, parent_uid: str | None, depth: int) -> None: fields = spec["fields"] @@ -482,6 +616,90 @@ class Collector: if updated: record["updated_at"] = updated self.comments[comment_uid] = record + self.add_engagement(spec, entry, "comment", comment_uid, created) + self.note_files(entry, None, "comment", comment_uid, record["author_uid"], created, "") + + def add_engagement(self, spec: dict, item, target_kind: str, target_uid: str, created: str) -> None: + if isinstance(item, Node): + return + for kind, bucket in (("votes", self.votes), ("reactions", self.reactions)): + engagement = spec.get(kind) + if not engagement: + continue + for entry in flatten_matches(select(item, engagement["list_path"])): + if not isinstance(entry, dict): + continue + fields = engagement.get("fields", {}) + voter = self.add_author(engagement.get("author") or {}, entry, created) + moment = normalize_instant(field_text(entry, fields.get("created_at"))) or created + if kind == "votes": + value = -1 if DOWN_VOTES.match(strip_tags(field_text(entry, fields.get("value"))).strip()) else 1 + key = f"vote:{target_kind}:{target_uid}:{voter}" + record = {"origin_uid": uid_for(key), "voter_uid": voter, "target_kind": target_kind, "target_uid": target_uid, "value": value, "created_at": moment} + else: + emoji = strip_tags(field_text(entry, fields.get("emoji"))).strip() + if not emoji or len(emoji) > 16 or emoji.isalnum(): + continue + key = f"reaction:{target_kind}:{target_uid}:{voter}:{emoji}" + record = {"origin_uid": uid_for(key), "author_uid": voter, "target_kind": target_kind, "target_uid": target_uid, "emoji": emoji, "created_at": moment} + bucket[record["origin_uid"]] = record + + def note_files(self, item, detail, target_kind: str, target_uid: str, author_uid: str, created: str, base: str) -> None: + if not self.options.get("attachments"): + return + found: list[str] = [] + for source in (item, detail): + if isinstance(source, Node): + for node in source.descendants(): + for attribute in ("src", "href", "data-src"): + if node.tag in ("img", "a", "source") and node.attrs.get(attribute): + found.append(node.attrs[attribute]) + elif isinstance(source, (dict, str)): + text = json.dumps(source, default=str) if isinstance(source, dict) else source + found.extend(HTML_FILE_URL.findall(text.replace("\\/", "/").replace('\\"', '"'))) + taken = 0 + seen: set[str] = set() + for raw in found: + absolute = urljoin(base, raw.strip()) if base else raw.strip() + path = absolute.split("?", 1)[0].lower() + if not absolute.startswith(("http://", "https://")) or not path.endswith(ATTACHMENT_EXTENSIONS) or absolute in seen: + continue + seen.add(absolute) + self.file_candidates.append({"url": absolute, "target_kind": target_kind, "target_uid": target_uid, "author_uid": author_uid, "created_at": created}) + taken += 1 + if taken >= int(self.options["max_attachments_per_post"]): + break + + async def download_files(self) -> None: + total = 0 + for candidate in self.file_candidates[: int(self.options["max_attachments"])]: + url = candidate["url"] + try: + response = await self.stealth.get(url, timeout=self.limits["timeout_seconds"], session=self.session, headers=self.headers or None) + kind = str(getattr(response, "headers", {}).get("content-type", "")).split(";")[0].strip().lower() if getattr(response, "headers", None) else "" + if int(response.status) >= 400 or (kind and not kind.startswith(ATTACHMENT_TYPES)): + self.notice(candidate["target_kind"] + "s", candidate["target_uid"], "attachment_unreachable", url, f"{url}: status {response.status}") + continue + data = await response.read() + except Exception as exc: + self.notice(candidate["target_kind"] + "s", candidate["target_uid"], "attachment_unreachable", url, f"{url}: {type(exc).__name__}: {exc}") + continue + if not data or len(data) > int(self.options["max_attachment_bytes"]) or total + len(data) > int(self.options["max_attachment_total"]): + self.notice(candidate["target_kind"] + "s", candidate["target_uid"], "attachment_too_large", url, f"{url}: {len(data)} bytes is over the limit") + continue + total += len(data) + filename = url.split("?", 1)[0].rstrip("/").rsplit("/", 1)[-1] or "file" + self.attachments.append( + { + "origin_uid": uid_for(f"file:{url}"), + "author_uid": candidate["author_uid"], + "target_kind": candidate["target_kind"], + "target_uid": candidate["target_uid"], + "filename": filename[:200], + "content_b64": base64.b64encode(data).decode(), + "created_at": candidate["created_at"], + } + ) def ordered_comments(self) -> list[dict]: ordered: list[dict] = [] @@ -518,6 +736,8 @@ class Collector: except CollectError as exc: self.errors.append(f"{feed['id']}: {exc}") reports[feed["id"]] = {"error": str(exc)} + if self.options.get("attachments"): + await self.download_files() failed = [name for name, report in reports.items() if "error" in report] return { "community": community, @@ -527,6 +747,9 @@ class Collector: "users": list(self.users.values()), "posts": list(self.posts.values()), "comments": self.ordered_comments(), + "votes": list(self.votes.values()), + "reactions": list(self.reactions.values()), + "attachments": self.attachments, }, "state": self.state, "feeds": reports, diff --git a/devplacepy/services/community/schemacheck.py b/devplacepy/services/community/schemacheck.py new file mode 100644 index 00000000..f498281f --- /dev/null +++ b/devplacepy/services/community/schemacheck.py @@ -0,0 +1,254 @@ +# retoor + +from __future__ import annotations + +import re +from urllib.parse import urlsplit + +from devplacepy.services.community.extract import parse_path, parse_selector +from devplacepy.services.community.runtime import DEFAULT_LIMITS +from devplacepy.services.community.spec import FEED_HTML, FEED_TYPES, LIMITS + +FEED_ID = re.compile(r"^[a-z0-9_-]{1,40}$") +ORIGIN = re.compile(r"^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$") +COMMENT_MODES = ("none", "inline", "url", "page") +PAGINATION_STYLES = ("none", "page", "offset", "cursor", "next_link") +COMMENT_PAGINATION_STYLES = ("none", "next_link", "page_param", "json_next") +LOGIN_MODES = ("form", "page", "json") +MAX_FEEDS = 8 +MAX_SPEC_CHARS = 500 +CEILINGS = { + "max_pages": 20, + "max_posts": 1000, + "max_comment_posts": 100, + "max_comment_pages": 10, + "comment_recheck": 50, + "retries": 5, + "timeout_seconds": 120, + "max_bytes": 20_000_000, + "max_remembered_links": 50_000, + "backoff_seconds": 30, +} +FLOORS = {"delay_seconds": 0.5} +MAX_DELAY = 60.0 +REQUIRED_POST_FIELDS = ("id",) +TEXT_POST_FIELDS = ("content", "title") + + +class SchemaError(ValueError): + def __init__(self, errors: list[str]): + super().__init__("; ".join(errors[:3])) + self.errors = errors + + +def check_spec(spec, html: bool, where: str, errors: list[str]) -> None: + if not isinstance(spec, str) or not spec.strip() or len(spec) > MAX_SPEC_CHARS: + errors.append(f"{where}: a non-empty path or selector of at most {MAX_SPEC_CHARS} characters is required") + return + if spec == "$now": + return + try: + if html: + selector, _, accessor = spec.partition("::") + if selector.strip(): + parse_selector(selector) + if accessor and accessor != "text" and not re.fullmatch(r"attr\([\w:-]+\)", accessor): + errors.append(f"{where}: unsupported accessor {accessor!r}, use text or attr(name)") + else: + parse_path(spec) + except ValueError as exc: + errors.append(f"{where}: {exc}") + + +def check_fields(fields, html: bool, where: str, errors: list[str]) -> None: + if not isinstance(fields, dict): + errors.append(f"{where}: must be an object of field names to lists of paths") + return + for name, specs in fields.items(): + if isinstance(specs, str): + specs = [specs] + if not isinstance(specs, list) or not specs: + errors.append(f"{where}.{name}: must be a non-empty list of paths") + continue + for index, spec in enumerate(specs): + check_spec(spec, html, f"{where}.{name}[{index}]", errors) + + +def check_author(author, html: bool, where: str, errors: list[str]) -> None: + if author in (None, {}): + return + if not isinstance(author, dict): + errors.append(f"{where}: must be an object") + return + check_fields(author.get("fields", {}), html, f"{where}.fields", errors) + + +def check_comments(comments, feed_html: bool, where: str, errors: list[str]) -> None: + if comments in (None, {}): + return + if not isinstance(comments, dict) or comments.get("mode") not in COMMENT_MODES: + errors.append(f"{where}.mode: must be one of {', '.join(COMMENT_MODES)}") + return + mode = comments["mode"] + if mode == "none": + return + page = mode == "page" + check_fields(comments.get("fields", {}), page, f"{where}.fields", errors) + fields = comments.get("fields") or {} + for required in ("id", "content") if mode != "page" else ("content",): + if required not in fields: + errors.append(f"{where}.fields.{required}: is required for comments in {mode} mode") + check_author(comments.get("author"), page, f"{where}.author", errors) + if mode in ("inline", "url") and comments.get("list_path") is not None: + check_spec(comments["list_path"], False, f"{where}.list_path", errors) + if mode == "inline" and not comments.get("list_path"): + errors.append(f"{where}.list_path: is required for inline comments") + if mode == "url": + template = comments.get("url_template") + if not isinstance(template, str) or not ("{id}" in template or "{uid}" in template): + errors.append(f"{where}.url_template: must contain {{id}} or {{uid}}") + if page: + if not comments.get("item_selector"): + errors.append(f"{where}.item_selector: is required for page comments") + else: + check_spec(comments["item_selector"], True, f"{where}.item_selector", errors) + hint = comments.get("pagination") + if hint: + if not isinstance(hint, dict) or hint.get("style") not in COMMENT_PAGINATION_STYLES: + errors.append(f"{where}.pagination.style: must be one of {', '.join(COMMENT_PAGINATION_STYLES)}") + elif hint["style"] == "next_link": + check_spec(hint.get("next"), True, f"{where}.pagination.next", errors) + elif hint["style"] == "json_next": + check_spec(hint.get("path"), False, f"{where}.pagination.path", errors) + + +def check_pagination(pagination, html: bool, where: str, errors: list[str]) -> None: + if pagination in (None, {}): + return + style = pagination.get("style") if isinstance(pagination, dict) else None + if style not in PAGINATION_STYLES: + errors.append(f"{where}.style: must be one of {', '.join(PAGINATION_STYLES)}") + return + if style in ("page", "offset", "cursor") and not pagination.get("param"): + errors.append(f"{where}.param: is required for {style} pagination") + if style == "offset" and not pagination.get("per_page"): + errors.append(f"{where}.per_page: is required for offset pagination") + if style == "next_link": + check_spec(pagination.get("next"), html, f"{where}.next", errors) + if style == "cursor": + check_spec(pagination.get("cursor_path"), False, f"{where}.cursor_path", errors) + + +def check_feed(feed, index: int, seen: set[str], errors: list[str]) -> None: + where = f"feeds[{index}]" + if not isinstance(feed, dict): + errors.append(f"{where}: must be an object") + return + identifier = feed.get("id") + if not isinstance(identifier, str) or not FEED_ID.match(identifier): + errors.append(f"{where}.id: 1-40 characters of a-z, 0-9, _ and -") + elif identifier in seen: + errors.append(f"{where}.id: {identifier!r} is used twice") + else: + seen.add(identifier) + parts = urlsplit(str(feed.get("url") or "")) + if parts.scheme not in ("http", "https") or not parts.hostname: + errors.append(f"{where}.url: must be an http or https address") + kind = feed.get("type") + if kind not in FEED_TYPES: + errors.append(f"{where}.type: must be one of {', '.join(FEED_TYPES)}") + return + html = kind == FEED_HTML + post = feed.get("post") + if not isinstance(post, dict): + errors.append(f"{where}.post: is required") + return + check_fields(post.get("fields"), html, f"{where}.post.fields", errors) + fields = post.get("fields") or {} + for required in REQUIRED_POST_FIELDS: + if required not in fields: + errors.append(f"{where}.post.fields.{required}: is required") + if not any(name in fields for name in TEXT_POST_FIELDS): + errors.append(f"{where}.post.fields: needs content or title") + detail = post.get("detail") + if detail: + check_fields({"content": detail.get("content")}, True, f"{where}.post.detail", errors) + if html: + if not feed.get("item_selector"): + errors.append(f"{where}.item_selector: is required for html_list feeds") + else: + check_spec(feed["item_selector"], True, f"{where}.item_selector", errors) + elif kind in ("json_api", "json_ld") and feed.get("list_path") is not None: + check_spec(feed["list_path"], False, f"{where}.list_path", errors) + check_author(feed.get("author"), html, f"{where}.author", errors) + check_pagination(feed.get("pagination"), html, f"{where}.pagination", errors) + check_comments(feed.get("comments"), html, f"{where}.comments", errors) + + +def check_limits(limits, errors: list[str]) -> None: + if limits in (None, {}): + return + if not isinstance(limits, dict): + errors.append("limits: must be an object") + return + for name, value in limits.items(): + if name not in DEFAULT_LIMITS: + errors.append(f"limits.{name}: unknown limit") + elif isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0: + errors.append(f"limits.{name}: must be a positive number") + elif name in CEILINGS and value > CEILINGS[name]: + errors.append(f"limits.{name}: at most {CEILINGS[name]}") + elif name in FLOORS and value < FLOORS[name]: + errors.append(f"limits.{name}: at least {FLOORS[name]} seconds, requests must stay polite") + elif name == "delay_seconds" and value > MAX_DELAY: + errors.append(f"limits.delay_seconds: at most {MAX_DELAY} seconds") + + +def check_login(login, errors: list[str]) -> None: + if login in (None, {}): + return + if not isinstance(login, dict) or login.get("mode") not in LOGIN_MODES: + errors.append(f"login.mode: must be one of {', '.join(LOGIN_MODES)}") + return + if login["mode"] == "json": + for required in ("url", "username_field", "password_field"): + if not login.get(required): + errors.append(f"login.{required}: is required for a JSON login") + if login.get("token_path"): + check_spec(login["token_path"], False, "login.token_path", errors) + elif not login.get("url"): + errors.append("login.url: is required") + + +def validate_schema(schema) -> list[str]: + errors: list[str] = [] + if not isinstance(schema, dict): + return ["the schema must be a JSON object"] + origin = schema.get("origin") + if not isinstance(origin, str) or not ORIGIN.match(origin): + errors.append("origin: a host name of a-z, 0-9, dots and hyphens") + if not isinstance(schema.get("community"), str) or not schema["community"].strip(): + errors.append("community: a name is required") + feeds = schema.get("feeds") + if not isinstance(feeds, list) or not feeds: + errors.append("feeds: at least one feed is required") + elif len(feeds) > MAX_FEEDS: + errors.append(f"feeds: at most {MAX_FEEDS}") + else: + seen: set[str] = set() + for index, feed in enumerate(feeds): + check_feed(feed, index, seen, errors) + check_limits(schema.get("limits"), errors) + check_login(schema.get("login"), errors) + return errors + + +def sample_limits(schema: dict) -> dict: + return { + **schema.get("limits", {}), + "max_pages": 1, + "max_posts": LIMITS.sample_posts, + "max_comment_posts": LIMITS.sample_comment_posts, + "max_comment_pages": 1, + "delay_seconds": max(float(schema.get("limits", {}).get("delay_seconds", LIMITS.delay_seconds)), LIMITS.delay_seconds), + } diff --git a/devplacepy/services/community/service.py b/devplacepy/services/community/service.py index 616f242d..a45865c7 100644 --- a/devplacepy/services/community/service.py +++ b/devplacepy/services/community/service.py @@ -73,6 +73,9 @@ class CommunityService(BaseService): dry = config[DRY_RUN_KEY] budget = config[MAX_PER_RUN_KEY] mode = "DRY RUN" if dry else "LIVE" + reaped = store.reap_interrupted() + if reaped: + self.log(f"marked {reaped} interrupted run(s)") for row in store.pending(budget): budget -= 1 self.log(f"[{mode}] discovering pending community {row['query']!r}") diff --git a/devplacepy/services/community/spec.py b/devplacepy/services/community/spec.py index 28a863de..be08f40f 100644 --- a/devplacepy/services/community/spec.py +++ b/devplacepy/services/community/spec.py @@ -8,7 +8,8 @@ FEED_RSS = "rss" FEED_ATOM = "atom" FEED_JSON = "json_api" FEED_HTML = "html_list" -FEED_TYPES = (FEED_RSS, FEED_ATOM, FEED_JSON, FEED_HTML) +FEED_JSONLD = "json_ld" +FEED_TYPES = (FEED_RSS, FEED_ATOM, FEED_JSON, FEED_HTML, FEED_JSONLD) STATUS_PENDING = "pending" STATUS_LEARNING = "learning" @@ -30,6 +31,7 @@ RUN_PARTIAL = "partial" RUN_FAILED = "failed" RUN_DRY = "dry_run" RUN_SKIPPED = "skipped" +RUN_INTERRUPTED = "interrupted" SCHEMA_VERSION = 1 RSEARCH_URL = "https://rsearch.app.molodetz.nl/search" @@ -60,8 +62,12 @@ class CommunityLimits: max_communities_per_run: int = 3 runs_kept: int = 30 ledger_runs_kept: int = 10 + checkpoint_seconds: int = 15 + interrupted_grace_seconds: int = 60 + search_min_chars: int = 3 batch_records: int = 50 min_field_coverage: float = 0.6 + max_remembered_links: int = 20_000 def runtime(self) -> dict: return { @@ -75,6 +81,7 @@ class CommunityLimits: "backoff_seconds": self.backoff_seconds, "timeout_seconds": self.timeout_seconds, "max_bytes": self.max_bytes, + "max_remembered_links": self.max_remembered_links, } diff --git a/devplacepy/services/community/store.py b/devplacepy/services/community/store.py index 49cf6502..2941b9f7 100644 --- a/devplacepy/services/community/store.py +++ b/devplacepy/services/community/store.py @@ -9,6 +9,7 @@ from devplacepy.database import db, get_table from devplacepy.database.soft_delete import soft_delete from devplacepy.services.community.spec import ( LIMITS, + RUN_INTERRUPTED, STATUS_FAILING, STATUS_LEARNING, STATUS_PENDING, @@ -17,7 +18,7 @@ from devplacepy.services.community.spec import ( from devplacepy.services.sync.selection import jittered from devplacepy.utils.text import generate_uid -JSON_COLUMNS = ("schema", "watermark", "totals", "sync_options") +JSON_COLUMNS = ("schema", "schema_previous", "watermark", "totals", "sync_options") RUN_JSON_COLUMNS = ("collected", "pushed", "errors", "tally") RUN_SUMMARY_COLUMNS = "uid, community_uid, started_at, finished_at, status, dry_run, trigger, pushed, tally, verdict, headline, items_stored, items_dropped" ITEMS_TABLE = "community_run_items" @@ -30,6 +31,7 @@ EDITABLE = ( "user_count", "status", "schema", + "schema_previous", "watermark", "enabled", "interval_seconds", @@ -46,6 +48,7 @@ EDITABLE = ( "login_username", "login_email", "login_password", + "login_totp_secret", "sync_options", "claimed_at", ) @@ -105,6 +108,7 @@ class CommunityStore: "username": row.get("login_username") or "", "email": row.get("login_email") or "", "password": row.get("login_password") or "", + "totp": row.get("login_totp_secret") or "", } def due(self, limit: int) -> list[dict]: @@ -167,6 +171,23 @@ class CommunityStore: ) return uid + def update_run(self, uid: str, **fields) -> None: + get_table("community_runs").update({"uid": uid, **encode(fields, RUN_JSON_COLUMNS)}, ["uid"]) + + def reap_interrupted(self) -> int: + if "community_runs" not in db.tables: + return 0 + horizon = iso(now() - timedelta(seconds=LIMITS.run_timeout_seconds + LIMITS.interrupted_grace_seconds)) + stale = [row["uid"] for row in get_table("community_runs").find(status="running") if row["started_at"] < horizon] + for uid in stale: + self.finish_run( + uid, + RUN_INTERRUPTED, + verdict="bad", + headline="Interrupted: the process ended before this run finished. What was synced up to then is listed.", + ) + return len(stale) + def finish_run(self, uid: str, status: str, **fields) -> None: get_table("community_runs").update( {"uid": uid, "status": status, "finished_at": iso(now()), **encode(fields, RUN_JSON_COLUMNS)}, ["uid"] diff --git a/devplacepy/services/devii/actions/catalog/community.py b/devplacepy/services/devii/actions/catalog/community.py index 9032fcac..0445be63 100644 --- a/devplacepy/services/devii/actions/catalog/community.py +++ b/devplacepy/services/devii/actions/catalog/community.py @@ -10,6 +10,7 @@ ACCOUNT = ( body("login_username", "Username of an account on that community, for a login-protected community."), body("login_email", "Email of that account, when the community logs in by email."), body("login_password", "Password of that account. Stored for the scheduled collectors, never shown again."), + body("login_totp_secret", "Authenticator (TOTP) secret of that account when it uses two-factor codes. Stored for the collectors, never shown again."), ) COMMUNITY_ACTIONS: tuple[Action, ...] = ( @@ -61,6 +62,34 @@ COMMUNITY_ACTIONS: tuple[Action, ...] = ( ), requires_admin=True, ), + Action( + name="community_schema_get", + method="GET", + path="/admin/communities/{slug}/schema", + summary="Show the full learned schema of a community as JSON (admin only)", + description="The schema lists the feeds and the paths or selectors that hold each field. Read it before proposing a change with community_schema_set.", + params=(SLUG,), + requires_admin=True, + read_only=True, + ), + Action( + name="community_schema_set", + method="POST", + path="/admin/communities/{slug}/schema", + summary="Replace the learned schema of a community after it passed a live check (admin only)", + description=( + "The structure is validated, then a small live sample must collect at least one post; otherwise nothing is " + "stored and the reasons are returned. The old schema is kept for a one-click restore. Show the administrator " + "the exact change, get explicit confirmation, then call again with confirm=true." + ), + params=( + SLUG, + body("schema", "The complete schema as one JSON object string.", required=True), + confirm(), + ), + requires_admin=True, + ajax=True, + ), Action( name="community_add", method="POST", diff --git a/devplacepy/services/devii/actions/catalog/game.py b/devplacepy/services/devii/actions/catalog/game.py index dc62b442..c2880ce9 100644 --- a/devplacepy/services/devii/actions/catalog/game.py +++ b/devplacepy/services/devii/actions/catalog/game.py @@ -75,6 +75,23 @@ GAME_ACTIONS: tuple[Action, ...] = ( Param(name="slot", location="body", description="Plot slot index.", required=True, type="integer"), ), ), + Action( + name="game_plot_stop", + method="POST", + path="/game/plot/stop", + summary="Pause or resume Continuous Delivery auto-replant on one Code Farm plot", + description=( + "Needs the Continuous Delivery Mastery upgrade. stop=true makes the next harvest of " + "that plot leave it empty so a different crop can be planted; the pause clears " + "itself once the plot is planted again. stop=false cancels a pending pause." + ), + handler="http", + requires_auth=True, + params=( + Param(name="slot", location="body", description="Plot slot index.", required=True, type="integer"), + body("stop", "true to skip the replant after the next harvest, false to cancel.", required=True, type="boolean"), + ), + ), Action( name="game_buy_plot", method="POST", diff --git a/devplacepy/services/devii/actions/catalog/nodes.py b/devplacepy/services/devii/actions/catalog/nodes.py index 90182809..4f9ce836 100644 --- a/devplacepy/services/devii/actions/catalog/nodes.py +++ b/devplacepy/services/devii/actions/catalog/nodes.py @@ -130,6 +130,24 @@ NODE_ACTIONS: tuple[Action, ...] = ( requires_admin=True, params=(ADMIN_DEVICE,), ), + Action( + name="node_admin_class_override", + method="POST", + path="/admin/nodes/devices/{device_uid}/class", + summary="Allow or stop allowing one donor device regardless of its address class", + description=( + "Donors on datacenter or unknown addresses are not scheduled unless an administrator overrides " + "the device. value=allow lets this device serve jobs; value=clear removes the override. Show the " + "administrator the device, its owner and its address class, get explicit confirmation, then call " + "again with confirm=true." + ), + requires_admin=True, + params=( + ADMIN_DEVICE, + body("value", "allow to override the address class rule, clear to remove the override.", required=True), + confirm(), + ), + ), Action( name="node_admin_killswitch", method="POST", diff --git a/devplacepy/services/devii/actions/catalog/sync.py b/devplacepy/services/devii/actions/catalog/sync.py index b40bf264..0b4e482b 100644 --- a/devplacepy/services/devii/actions/catalog/sync.py +++ b/devplacepy/services/devii/actions/catalog/sync.py @@ -3,7 +3,7 @@ from __future__ import annotations from ..spec import Action -from ._shared import body +from ._shared import body, path, query ENVELOPE = ( body("origin", "Stable id of the sending community, for example other.example.org.", required=True), @@ -41,4 +41,85 @@ SYNC_ACTIONS: tuple[Action, ...] = ( "Upsert comments copied from another community", "Creates or updates comments; author_uid, post_uid and any parent_uid are origin ids that must already be synced.", ), + sync_action( + "projects", + "Upsert public projects copied from another community", + "Creates or updates projects as public and writable; author_uid is the author's origin id. Posts reference a project through project_uid.", + ), + sync_action( + "polls", + "Upsert polls copied from another community", + "One poll per post with 2 to 6 options, each option carrying its own origin_uid and label.", + ), + sync_action( + "votes", + "Upsert votes copied from another community", + "A vote needs a synced voter and a synced target (target_kind post, comment or project, value 1 or -1). The target's stars are derived from synced votes only.", + ), + sync_action( + "reactions", + "Upsert emoji reactions copied from another community", + "A reaction needs a synced author and a synced target; emoji must be a single emoji.", + ), + sync_action( + "follows", + "Upsert follow relations copied from another community", + "follower_uid and following_uid are origin ids of synced users. A user cannot follow themselves.", + ), + sync_action( + "poll_votes", + "Upsert poll ballots copied from another community", + "One ballot per voter and poll; option_uid is the option's origin id within poll_uid.", + ), + sync_action( + "attachments", + "Upsert attachments copied from another community", + "The sender downloads the file and sends it as content_b64 (at most 5 MB). Attachments cannot change; only a tombstone or a restore is accepted for a known origin_uid.", + ), + Action( + name="sync_export", + method="GET", + path="/sync/export/{kind}", + summary="Export this community's own users, projects, posts, comments, polls, follows or attachments in sync record form", + description=( + "Administrator pull feed for another community. Pages of at most 50 records, keyset paginated: pass the " + "returned next_cursor back until has_more is false. Rows that were themselves synced in are never exported. " + "Votes, reactions and ballots are never exported because who voted is not public." + ), + params=( + path("kind", "One of users, projects, posts, comments, polls, follows, attachments."), + query("since", "Only records changed after this ISO 8601 instant."), + query("cursor", "The next_cursor of the previous page."), + query("limit", "Records per page, 1 to 50."), + ), + requires_admin=True, + read_only=True, + ), + Action( + name="sync_deletions", + method="GET", + path="/sync/deletions", + summary="List synced records that were deleted or purged on this side and not yet acknowledged by the origin", + description="Lets a sending community learn which of its records are gone here. Page with next_cursor, then acknowledge with sync_deletions_ack.", + params=( + query("origin", "The sending community's origin id.", required=True), + query("cursor", "The next_cursor of the previous page."), + query("limit", "Entries per page, 1 to 50."), + ), + requires_admin=True, + read_only=True, + ), + Action( + name="sync_deletions_ack", + method="POST", + path="/sync/deletions/ack", + summary="Acknowledge deletions so the origin is no longer told about them", + description="After acknowledging, a re-sent record answers unchanged with reason acknowledged_deleted instead of a rejection.", + ajax=True, + params=( + body("origin", "The sending community's origin id.", required=True), + body("items", "JSON array string of at most 50 objects like {\"kind\": \"posts\", \"origin_uid\": \"p-1\"}.", required=True), + ), + requires_admin=True, + ), ) diff --git a/devplacepy/services/devii/actions/dispatcher.py b/devplacepy/services/devii/actions/dispatcher.py index 346f7a7d..150e901a 100644 --- a/devplacepy/services/devii/actions/dispatcher.py +++ b/devplacepy/services/devii/actions/dispatcher.py @@ -34,8 +34,10 @@ MUTATING_METHODS = ("POST", "DELETE", "PUT", "PATCH") CONFIRM_REQUIRED = { "community_delete", + "community_schema_set", "node_device_revoke", "node_admin_suspend", + "node_admin_class_override", "node_admin_killswitch", "join_battle", "fight_battle", @@ -251,6 +253,12 @@ def confirmation_error(name: str, arguments: dict[str, Any]) -> ToolInputError | f"Deleting community '{slug}' stops its synchronisation and moves it to the trash; content already " "synchronised stays. Show the user the community, get explicit confirmation, then call again with confirm=true." ) + if name == "community_schema_set": + slug = str(arguments.get("slug", "")).strip() or "(unspecified)" + return ToolInputError( + f"Replacing the learned schema of community '{slug}' changes what every future run collects. " + "Show the administrator the exact change, get explicit confirmation, then call again with confirm=true." + ) if name == "node_device_revoke": uid = str(arguments.get("device_uid", "")).strip() or "(unspecified)" return ToolInputError( @@ -264,6 +272,13 @@ def confirmation_error(name: str, arguments: dict[str, Any]) -> ToolInputError | "the whole period. Show the administrator the exact device and owner, get explicit confirmation, " "then call again with confirm=true." ) + if name == "node_admin_class_override": + uid = str(arguments.get("device_uid", "")).strip() or "(unspecified)" + return ToolInputError( + f"Overriding the address class rule for donor device '{uid}' lets it serve jobs although its " + "address is not a residential or mobile one. Show the administrator the device, its owner and its " + "address class, get explicit confirmation, then call again with confirm=true." + ) if name == "node_admin_killswitch": return ToolInputError( "The community donation switch is global: turning it off disconnects every donor node within " diff --git a/devplacepy/services/game/CLAUDE.md b/devplacepy/services/game/CLAUDE.md index 2b86735f..17700d49 100644 --- a/devplacepy/services/game/CLAUDE.md +++ b/devplacepy/services/game/CLAUDE.md @@ -4,7 +4,7 @@ This file documents the Code Farm idle game. Claude Code auto-loads it when a fi ## Overview -`game/` package (routers, mounted at `/game`) is the **Code Farm** idle game (`index.py` + `farm.py`): `GET /game` (page), `GET /game/state`, `GET /game/leaderboard?board=`, `POST /game/{plant,harvest,buy-plot,upgrade,fertilize,daily,grant,perk,prestige,legacy,mastery,quests/claim}`, `POST /game/{defense/upgrade,defense/downgrade,infrastructure/buy,cosmetics/buy,cosmetics/equip}`, plus social `GET /game/farm/{username}` and `POST /game/farm/{username}/{water,steal}`. Customer-facing documentation is the `docs_api.py` **Code Farm** group (`docs_api/groups/game.py`, enum `options` on every key param) and the prose guide `templates/docs/code-farm.html` (`/docs/code-farm.html` - the complete rules, every exact formula and catalog table, the full farm/plot field reference, the JSON error shape, and a stdlib automation client); keep BOTH in lockstep with `economy.py` whenever a constant, formula, catalog entry, field, or endpoint changes. +`game/` package (routers, mounted at `/game`) is the **Code Farm** idle game (`index.py` + `farm.py`): `GET /game` (page), `GET /game/state`, `GET /game/leaderboard?board=`, `POST /game/{plant,harvest,plot/stop,buy-plot,upgrade,fertilize,daily,grant,perk,prestige,legacy,mastery,quests/claim}`, `POST /game/{defense/upgrade,defense/downgrade,infrastructure/buy,cosmetics/buy,cosmetics/equip}`, plus social `GET /game/farm/{username}` and `POST /game/farm/{username}/{water,steal}`. Customer-facing documentation is the `docs_api.py` **Code Farm** group (`docs_api/groups/game.py`, enum `options` on every key param) and the prose guide `templates/docs/code-farm.html` (`/docs/code-farm.html` - the complete rules, every exact formula and catalog table, the full farm/plot field reference, the JSON error shape, and a stdlib automation client); keep BOTH in lockstep with `economy.py` whenever a constant, formula, catalog entry, field, or endpoint changes. A cooperative-and-competitive idle game (Farmville-style) mounted at `/game`, member-only to play, public to view another farm. Cooperative loop = watering a neighbour's growing build; competitive loop = stealing a neighbour's ready build. @@ -100,6 +100,15 @@ Orthogonal to prestige, modeled directly on the existing Legacy shape (leveled, Three new high-tier crops (`distsys`/`mlpipe`/`secfort` in `economy.CROPS`) extend the `Crop` dataclass with three new **trailing, defaulted** fields (`min_mastery: int = 0`, `steal_immune: bool = False`, `era_key: str | None = None`) - safe because every existing `Crop(...)` construction is positional over the original 8 non-defaulted fields. They set `min_level=MAX_LEVEL` (so, like every other crop, they re-lock after each refactor until the player re-levels to 20 - consistent with how Kernel etc. already behave post-prestige) and `min_mastery=1` (gated on `mastery_points_earned_total`, checked both in `store/actions.py::plant()` - real enforcement - and in `economy.crop_payload`'s `locked` flag - display). `secfort` (Security Fortress) is `steal_immune=True`: `serialize_plot` forces `can_steal=False`/`steal_reason="immune"` and `store/actions.py::steal()` raises immediately, before any grace/cooldown check, when the target crop is immune. +### Per-plot auto-replant pause (`game_plots.stop_after_harvest`) + +Continuous Delivery used to be a one-way lock: the harvested crop was replanted at once, `plant()` refuses an occupied plot and nothing uproots, so a crop could only change at a Refactor. `POST /game/plot/stop` (`store.set_plot_stop`, `GamePlotStopForm{slot, stop}`, Devii `game_plot_stop`, audit `game.plot.stop`) sets a **one-shot** flag on one plot. Invariants: + +- **The flag is read AFTER `clear_plot` wins.** `_try_autoreplant` re-reads the plot, so a toggle either landed before the clear (replant skipped) or finds an empty plot (`set_plot_stop`'s `WHERE crop_key != ''` fails, loud error). Never read the flag from the pre-clear plot dict and never reset it inside `clear_plot`. +- **Every writer that puts a crop on a plot resets it:** the `plant()` and `_try_autoreplant` claim SETs write `stop_after_harvest = 0`, and `prestige()`'s plot reset writes it too. A stale `1` on an empty plot is harmless and is never serialized (`serialize_plot` gates `stop_after_harvest` on owner + Continuous Delivery + a crop present). +- Column is `COALESCE`-read (`_lvl(plot, ...)`); existing rows are NULL. +- `_crop_lock(farm, crop)` is the single lock predicate used by `plant()` and `_try_autoreplant`, so autoreplant no longer replants a crop that is locked (an Era crop after its Era ends). Saturation-based skipping was deliberately NOT built: at prestige 50+ (the only owners of Continuous Delivery) even a 0.4 market factor pays 11x or more the planting cost for every crop, so skipping only forfeits coins, XP and quest progress. + ### 4. Secondary leaderboards (`economy.py` scoring + `store/farm.py`) `GET /game/leaderboard` gained an optional `board` query param (default `"score"`, so every existing caller/Devii/docs entry that omits it sees byte-identical output to before this shipped). `store/farm.py::leaderboard_for(board, limit)` dispatches through `LEADERBOARD_BOARDS` (`score` = the original `farm_score` board, `prestige`, `harvests` = this week's `harvests_week` counter, `raids` = average coins per **successful** raid with a minimum of `MIN_RAIDS_FOR_EFFICIENCY_BOARD` (3) qualifying raids - deliberately not "attempts," since failed steals are never persisted today and adding that write would be a spam vector for no real value, `time_to_kernel` = seconds between a farm's last `prestiged_at` and its next Kernel harvest, `fair_play` = `economy.fair_play_score(harvests_week, coins)` which rewards recent activity and penalizes hoarding) plus `era` (dispatches to `store/era.py::leaderboard_era`, always empty when no Era is running). `GameFarm.js._loadLeaderboard` renders a board-appropriate value per row instead of always showing the raw composite score - `raid_avg` (formatted `"Nc/raid"`) for the `raids` board and a formatted duration for `time_to_kernel`, falling back to `score` for every other board. New `game_farms` columns backing these: `harvests_week`/`harvests_week_start` (**incremented inside `credit_farm`'s atomic UPDATE** whenever `harvests > 0` - a CASE that rolls the counter over to the new ISO week when `harvests_week_start` differs, else adds; also reset lazily in `serialize_farm` via `_reset_harvests_week` for idle weeks, same lazy-no-tick idiom as everything else. The increment is load-bearing: it feeds grant eligibility, the `harvests` board, and `fair_play` - it was originally missing entirely, leaving the community grant permanently unclaimable and the weekly board all zeros, a liveness bug invisible to pure-safety fuzzing and caught only by the layer-3 grant race setup), `prestiged_at` (stamped every `prestige()`), `last_kernel_harvest_prestige`/`time_to_kernel_seconds` (updated in `harvest`/`_auto_harvest` the first time a Kernel is harvested since the last refactor). diff --git a/devplacepy/services/game/store/__init__.py b/devplacepy/services/game/store/__init__.py index 32521739..c0055bb9 100644 --- a/devplacepy/services/game/store/__init__.py +++ b/devplacepy/services/game/store/__init__.py @@ -18,6 +18,7 @@ from .actions import ( harvest, plant, prestige, + set_plot_stop, steal, upgrade_legacy, upgrade_perk, diff --git a/devplacepy/services/game/store/actions.py b/devplacepy/services/game/store/actions.py index c86ef31b..a0c218db 100644 --- a/devplacepy/services/game/store/actions.py +++ b/devplacepy/services/game/store/actions.py @@ -38,16 +38,20 @@ from .quests import advance_quests, ensure_quests from .serialize import _daily_available, _plot_state, _watered_by +def _crop_lock(farm: dict, crop) -> tuple[str, str] | None: + progress = economy.level_progress(int(farm.get("xp", 0))) + era_locked = bool(crop.era_key and crop.era_key != active_era_name()) + return economy.crop_lock_reason( + crop, progress["level"], _lvl(farm, "mastery_points_earned_total"), era_locked + ) + + def plant(user: dict, slot: int, crop_key: str) -> dict: farm = ensure_farm(user["uid"]) crop = economy.crop_for(crop_key) if not crop: raise GameError("Unknown crop type.") - progress = economy.level_progress(int(farm.get("xp", 0))) - era_locked = bool(crop.era_key and crop.era_key != active_era_name()) - lock = economy.crop_lock_reason( - crop, progress["level"], _lvl(farm, "mastery_points_earned_total"), era_locked - ) + lock = _crop_lock(farm, crop) if lock: raise GameError(f"{crop.name} {lock[1]}.") plot = _plot_at(farm["uid"], slot) @@ -77,7 +81,7 @@ def plant(user: dict, slot: int, crop_key: str) -> dict: claimed = conditional_update_row( "game_plots", plot["uid"], - "crop_key = :crop, planted_at = :planted, ready_at = :ready, watered_by = '[]', raided_fraction = 0", + "crop_key = :crop, planted_at = :planted, ready_at = :ready, watered_by = '[]', raided_fraction = 0, stop_after_harvest = 0", "crop_key = '' OR crop_key IS NULL", {"crop": crop.key, "planted": _iso(now), "ready": _iso(ready_at)}, ) @@ -168,6 +172,8 @@ def _try_autoreplant(farm: dict, slot: int, crop, now) -> None: plot = _plot_at(farm["uid"], slot) if not plot or plot.get("crop_key"): return + if _lvl(plot, "stop_after_harvest") or _crop_lock(farm, crop): + return cost = economy.effective_plant_cost(crop, _lvl(farm, "perk_discount")) if int(farm.get("coins", 0)) < cost: return @@ -187,7 +193,7 @@ def _try_autoreplant(farm: dict, slot: int, crop, now) -> None: claimed = conditional_update_row( "game_plots", plot["uid"], - "crop_key = :crop, planted_at = :planted, ready_at = :ready, watered_by = '[]', raided_fraction = 0", + "crop_key = :crop, planted_at = :planted, ready_at = :ready, watered_by = '[]', raided_fraction = 0, stop_after_harvest = 0", "crop_key = '' OR crop_key IS NULL", {"crop": crop.key, "planted": _iso(now), "ready": _iso(ready_at)}, ) @@ -195,6 +201,25 @@ def _try_autoreplant(farm: dict, slot: int, crop, now) -> None: refund_farm(farm["uid"], cost) +def set_plot_stop(user: dict, slot: int, stop: bool) -> dict: + farm = ensure_farm(user["uid"]) + if not _lvl(farm, MASTERY_COLUMN["autoreplant"]): + raise GameError("Continuous Delivery is not unlocked.") + plot = _plot_at(farm["uid"], slot) + if not plot or not plot.get("crop_key"): + raise GameError("That plot is empty.") + updated = conditional_update_row( + "game_plots", + plot["uid"], + "stop_after_harvest = :stop", + "crop_key != '' AND crop_key IS NOT NULL", + {"stop": 1 if stop else 0}, + ) + if updated == 0: + raise GameError("That plot is empty.") + return {"slot": slot, "stop": bool(stop), "crop": plot.get("crop_key", "")} + + def water(visitor: dict, owner: dict, slot: int) -> dict: if visitor["uid"] == owner["uid"]: raise GameError("You cannot water your own build.") @@ -615,6 +640,7 @@ def prestige(user: dict) -> dict: "planted_at": "", "ready_at": "", "watered_by": "[]", + "stop_after_harvest": 0, "updated_at": now, }, ["uid"], diff --git a/devplacepy/services/game/store/serialize.py b/devplacepy/services/game/store/serialize.py index dd3c2edb..41d16fab 100644 --- a/devplacepy/services/game/store/serialize.py +++ b/devplacepy/services/game/store/serialize.py @@ -65,6 +65,7 @@ def serialize_plot( canary: bool = False, underdog: bool = False, contract_boost: bool = False, + autoreplant: bool = False, ) -> dict: state = _plot_state(plot, now) crop = economy.crop_for(plot.get("crop_key", "")) @@ -133,6 +134,7 @@ def serialize_plot( if can_steal else 0 ) + can_stop_replant = is_owner and autoreplant and bool(crop) fertilize_cost = 0 if state == "growing" and is_owner and crop: full_grow = economy.grow_seconds_for( @@ -172,6 +174,8 @@ def serialize_plot( "fertilize_cost": fertilize_cost, "raided_fraction": round(raided_fraction, 4), "raided_pct": round(raided_fraction * 100), + "can_stop_replant": can_stop_replant, + "stop_after_harvest": can_stop_replant and bool(plot.get("stop_after_harvest")), } @@ -238,6 +242,7 @@ def serialize_farm( canary=owns_infrastructure(farm, "canary"), underdog=underdog_active, contract_boost=contract_active, + autoreplant=bool(_lvl(farm, "mastery_autoreplant")), ) for plot in plots ] diff --git a/devplacepy/services/jobs/deepsearch/crawl.py b/devplacepy/services/jobs/deepsearch/crawl.py index b7da1fea..ddcf7217 100644 --- a/devplacepy/services/jobs/deepsearch/crawl.py +++ b/devplacepy/services/jobs/deepsearch/crawl.py @@ -18,6 +18,8 @@ from devplacepy import stealth from devplacepy.config import INTERNAL_BASE_URL from devplacepy.net_guard import BlockedAddressError, guard_public_url, guarded_async_client +from devplacepy.services.nodes.policy import DestinationPolicy, PolicyError + from .extract import extract_html, relevant_links from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf @@ -319,11 +321,62 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None: ) -async def _resolve_candidate(candidate: dict, depth: int, browser=None) -> CrawledPage | None: +NODE_FETCH_TIMEOUT_SECONDS = 40.0 +NODE_FETCH_WAIT_SECONDS = 25 + + +def node_servable(url: str) -> bool: + try: + DestinationPolicy().check(url) + except PolicyError: + return False + return True + + +async def node_page(url: str, depth: int, api_key: str) -> CrawledPage | None: + if not api_key or not node_servable(url): + return None + try: + async with stealth.stealth_async_client( + base_url=INTERNAL_BASE_URL, + headers={"Authorization": f"Bearer {api_key}", "Accept": "application/json"}, + timeout=httpx.Timeout(NODE_FETCH_TIMEOUT_SECONDS, connect=10.0), + ) as client: + response = await client.post( + "/nodes/internal/fetch", data={"url": url, "timeout": NODE_FETCH_WAIT_SECONDS} + ) + if response.status_code >= 400: + return None + data = response.json() + except (httpx.HTTPError, ValueError) as exc: + logger.info("deepsearch node fetch failed for %s: %s", url, exc) + return None + if int(data.get("status") or 0) >= 400: + return None + extracted = extract_html(data.get("text") or "", base_url=url) + if len(extracted.text) < MIN_PAGE_CHARS: + return None + return CrawledPage( + url=url, + title=extracted.title or url, + text=extracted.text, + source=f"node-{data.get('via') or 'fetch'}", + status=int(data.get("status") or 200), + depth=depth, + links=extracted.links, + ) + + +async def _resolve_candidate( + candidate: dict, depth: int, browser=None, api_key: str = "" +) -> CrawledPage | None: url = candidate["url"] snippet_page = _snippet_page(candidate, depth) if _is_hostile(url): - return snippet_page + page = await node_page(url, depth, api_key) + if page and snippet_page: + return page if len(page.text) >= len(snippet_page.text) else snippet_page + return page or snippet_page page = await fetch_page(url, depth, browser) if page and snippet_page: return page if len(page.text) >= len(snippet_page.text) else snippet_page @@ -339,6 +392,7 @@ async def crawl( query: str = "", depth: int = 1, seen_hashes: set[str] | None = None, + api_key: str = "", ) -> CrawlOutcome: from playwright.async_api import async_playwright @@ -387,7 +441,7 @@ async def crawl( emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"}) fetch_start = time.perf_counter() results = await asyncio.gather( - *(_resolve_candidate(candidate, level, browser) for candidate in batch), + *(_resolve_candidate(candidate, level, browser, api_key) for candidate in batch), return_exceptions=True, ) elapsed_ms = int((time.perf_counter() - fetch_start) * 1000) diff --git a/devplacepy/services/jobs/deepsearch/worker.py b/devplacepy/services/jobs/deepsearch/worker.py index 0b965319..a4a52f50 100644 --- a/devplacepy/services/jobs/deepsearch/worker.py +++ b/devplacepy/services/jobs/deepsearch/worker.py @@ -189,6 +189,7 @@ async def _run(payload: dict, output_dir: Path) -> dict: should_stop, query=query, depth=depth, + api_key=api_key, ) round_no = 0 @@ -227,6 +228,7 @@ async def _run(payload: dict, output_dir: Path) -> dict: query=query, depth=REFINE_CRAWL_DEPTH, seen_hashes=outcome.seen_hashes, + api_key=api_key, ) if not extra.pages: break diff --git a/devplacepy/services/nodes/CLAUDE.md b/devplacepy/services/nodes/CLAUDE.md index 38b3ab22..ee3bd9d1 100644 --- a/devplacepy/services/nodes/CLAUDE.md +++ b/devplacepy/services/nodes/CLAUDE.md @@ -121,20 +121,31 @@ For a `tunnel` or `any` job the scheduler prefers a node whose `hello.modes` inc `SocialSearch.search` fans one query out to adapters in parallel (`asyncio.gather`, 20 s each) and merges hits round-robin. Direct adapters (`adapters/direct.py`: Hacker News, Stack Overflow, GitHub issues, dev.to by tag, Mastodon by hashtag) need no donors. `adapters/reddit.py` is node-backed and only used when `nodes_social_via_nodes` is `1`; otherwise it is reported as `nodes_not_used`. Every adapter parses a response structurally and raises `AdapterError` on a forged or wrong shape; platforms that fail are listed under `unavailable` with a code and never filled with invented results. **Measured on this machine (2026-10-15, datacenter address):** Hacker News Algolia, Stack Exchange, GitHub search, dev.to tag API and Mastodon hashtag timelines answer; Bluesky `searchPosts` returns 403 without auth, Mastodon `/api/v2/search` returns nothing without login, Lobsters search sits behind a bot wall, dev.to has no usable text search, and Reddit returns 403 to this address. Those are why Bluesky and Lobsters are not adapters. Surfaces: `GET /nodes/social/search` (daily per-member quota `nodes_daily_social_user`, default 300), Devii `social_search`, the DeepSearch `deepsearch_social` option (off by default, signed-in owners only, goes through the HTTP route with the owner's key). +## Server mechanisms added after the first build + +- **Address class (D6 enforced).** `geo.py` keeps DB-IP lite ASN and country data in a SQLite range file under `config.NODE_GEO_DIR` (`devplace nodes geo-sync [--dry-run] [--month]`, `geo-status`; CC BY 4.0, attribution on `/docs/privacy.html`). Measured: build about 10 s, 477362 ASN and 710834 country rows, 48 MB, lookup 0.17 ms. `addressclass.classify` maps an ASN to `residential`, `mobile`, `datacenter`, `business` or `unknown`. The scheduler only assigns jobs to `ELIGIBLE_CLASSES` (residential, mobile) unless a device carries `class_override == "allow"` (admin, `POST /admin/nodes/devices/{uid}/class`, audit `node.device.override`, Devii `node_admin_class_override` confirm gated) or `nodes_allow_unknown_class=1` admits unknown. Refusals are counted in `NodeScheduler.class_refusals`. A missing geo file is `counters.geo_unavailable` plus one warning, never a silent pass. +- **Durable state.** `last_seen_at` writes are batched (`touch_devices`, `last_seen_write_seconds`). Counters flush as deltas into `node_counters` every `counter_flush_seconds` (lifetime totals survive restart). Circuit breaker state is saved to `node_circuits` (`snapshot`/`restore`, capped at `circuit_snapshot_max`). The fabric cache is `SharedResultCache`: an in-process L1 over the `node_cache` table, so a result fetched by one worker serves the others. +- **Fast path.** `wake.py` `WakeHub` wakes the dispatch loop and a waiting caller in the same process the moment a job is queued or done; the DB poll remains the cross-process fallback. Measured end to end (one donor, 300 jobs): polling p50 209 ms, p95 214 ms; fast path p50 25 ms, p95 about 30 ms, max about 38 ms. A single 6.2 s cold-start outlier appeared once and did not reproduce. +- **Sampled redundancy.** `redundancy_percent` (5, ceiling `redundancy_max_percent` 20) of fetch-mode jobs are shadowed to a second donor (`shadow_of`, `exclude_device`); `shape.py` compares structure, and a mismatch raises the flag `redundancy_mismatch` for a human. Counters: `redundancy_checked`, `redundancy_matched`, `redundancy_mismatches`, `redundancy_inconclusive`. +- **Outcome counters per address class.** `outcome:::` in `node_counters`, shown on `/admin/nodes`. This is the instrument for the residential success rate; it reads `n = 0` until real donors exist. +- **DeepSearch** routes hostile URLs through `POST /nodes/internal/fetch` with the owner's api key (`crawl.node_servable`, `node_page`) when `nodes_social_via_nodes` is on. +- **Battery.** `on_battery` is a donor limit code (pause 300 s, retryable). `welcome.device_label` and `kick.until` are optional protocol fields; a suspended ticket 403 carries `until`. + +**Measured capacity (one worker, scratch server, 2000 simulated donors on this machine):** 2000 sockets connected in 8 s; RSS 156 MB at 250, 190 MB at 500, 241 MB at 1000, 341 MB at 2000; idle dispatch tick p50 67 ms (max 160 ms); event loop lag p99 78 ms idle; 14.2 jobs/s completed at 15 offered, 0 client failures. A soak at 2000 sockets, 10 jobs/s and 3 percent a minute churn ran 4844 s (about 81 minutes, not 24 hours): RSS 346 to 361 MB, 42909 of 42913 jobs done, 1 failed, 0 integrity failures, 2026 of 2026 redundancy checks matched, loop lag p99 73 to 106 ms. Races, with separate OS processes: 16 workers x 20 counter flushes summed exactly (1600 of 1600); 12 racing cache puts left 1 row and all 12 readers hit. + ## Clients -`nodes/client/` is the Go donor client (standard library only, `build.sh` cross-compiles five targets and writes `manifest.json`). `nodes/extension/` is the MV3 extension (Chrome and Firefox, fetch mode only, 49 `node --test` tests, one of which reads `spec.py` and fails on drift). `make node-client` builds both into `data/node_clients/`, which `GET /nodes/client-version` and `GET /nodes/download/{artifact}` serve (names must match a strict pattern, symlinks and the manifest are never served, the SHA-256 is computed from the file). **Verified end to end on 2026-10-15** against a scratch server on this machine: the Go client logged in, connected, answered a `tunnel` job for `reddit.com` (`via: tunnel`, status 403 because the donor was this same datacenter address) and stayed connected across repeated jobs. Not verified: a residential donor, the extension in a real browser, Windows and macOS builds running. +`nodes/client/` is the Go donor client (standard library only, `build.sh` cross-compiles five targets and writes `manifest.json`). It reads battery state on Linux, macOS and Windows (`RefuseBattery`, `PauseOnBattery`) and holds a timed suspension until `until` (clamped 60 s to 24 h). `nodes/extension/` is the MV3 extension (Chrome and Firefox, fetch mode only; icons 16/32/48/128; a suspended state with `suspendedUntil`; active hours in the server timezone; the manifest sets `content_security_policy.extension_pages` explicitly because Firefox's default MV3 policy adds `upgrade-insecure-requests`, which rewrote `ws://127.0.0.1` to `wss://` and broke loopback connections). `make node-client` builds both into `data/node_clients/`, served by `GET /nodes/client-version` and `GET /nodes/download/{artifact}` (strict names, no symlinks, SHA-256 from the file). **Verified in real browsers:** Chromium via `tests/e2e/nodes/extension.py` (disclosure, connect, online, a fetch-mode reddit job answered with 403 via `fetch`, suspend, disable) and Firefox 158 with geckodriver (connected, job done with 403 and 190240 bytes, suspended with `until`, disable). The Firefox script is not in the repository; the method is in `nodes/extension/README.md`. Go: `go vet` and `go test` pass, cross-compiles clean. ## Gaps (what is not built or not measured) -- **Reddit viability is unmeasured from a residential line.** The only real run used a datacenter donor and got 403, which proves the path, not the success rate. X, Instagram, LinkedIn, Facebook and TikTok stay unreachable (login walls); nothing here claims them. -- `country`, `asn` and `asn_class` on a device are never filled, so the datacenter exclusion (D6) is not enforced: the scheduler treats every node as eligible. It needs an offline ASN database file. -- `last_seen_at` is written on connect only. Socket counters are per process. The fabric cache is per process. Circuit breaker state is in memory. The same-process fast path is not built, callers poll at `poll_interval_ms`. -- Fetch-mode results are not cross-checked (sampled redundancy between two donors is not built). They are structurally checked and labelled `via: fetch`, and `tunnel` is preferred whenever a node can do it. -- `DeepSearch` still skips hostile domains with a static list; Reddit URLs from social search are used through their snippet. Routing a hostile URL fetch through `/nodes/internal/fetch` is not built. -- The Go client has no battery-aware pause (`pause_on_battery` is stored and sent but ignored by it), and treats a timed suspension as permanent until restarted. The extension enforces its own local hours in the browser clock, not the server timezone. -- Not measured: Reddit and X success rates per address class, WebSocket idle timeouts through `pravda.education` and `devplace.net`, socket capacity of one worker, a 24 hour soak. The caps in `LimitSpec` are starting ceilings, not measurements. -- **Launch is blocked on a recorded legal decision** (platform terms, donor terms, ISP terms, jurisdiction). Nothing here is enabled for any account: `nodes_enabled` is `0` until an administrator switches it on. +- **Reddit success from a residential line is unmeasured.** The counters exist and read `n = 0`; every real run so far used a datacenter donor and got 403. X, Instagram, LinkedIn, Facebook and TikTok stay unreachable (login walls); nothing here claims them. +- **No 24 hour soak.** The longest run was about 81 minutes. The caps in `LimitSpec` are ceilings backed by that run, not by a day. +- **WebSocket idle timeout through the public hostname is unmeasured.** It needs a production connection, which needs the owner's go-ahead. +- The extension has no battery pause (browsers expose no battery state to extensions); the Go client has one. +- The geo file needs a monthly `geo-sync`; between syncs a new ASN is `unknown`. +- Windows and macOS client builds compile but were not run. +- **Launch is blocked on a recorded legal decision** (platform terms, donor terms, ISP terms, jurisdiction). Nothing here is enabled for any account: `nodes_enabled` is `0` until an administrator switches it on. This is a human decision and no code closes it. ## Rules for work in this directory diff --git a/devplacepy/services/nodes/addressclass.py b/devplacepy/services/nodes/addressclass.py new file mode 100644 index 00000000..cf4365bf --- /dev/null +++ b/devplacepy/services/nodes/addressclass.py @@ -0,0 +1,66 @@ +# retoor + +from __future__ import annotations + +import re + +CLASS_RESIDENTIAL, CLASS_MOBILE, CLASS_DATACENTER, CLASS_UNKNOWN = ( + "residential", + "mobile", + "datacenter", + "unknown", +) +ELIGIBLE_CLASSES = (CLASS_RESIDENTIAL, CLASS_MOBILE) + +DATACENTER_ASNS = frozenset( + { + 16509, 14618, 8987, 7224, + 15169, 396982, 19527, + 8075, 8068, 8069, + 24940, 213230, + 16276, + 14061, 393406, + 63949, 20940, 16625, + 20473, + 31898, 792, + 45102, 37963, + 45090, 132203, + 13335, + 51167, + 12876, + 60781, 16265, 30633, + 9009, + 36351, + 27357, 19994, + 54113, + 47583, + 197540, + 8560, + 26496, + 22612, + } +) + +DATACENTER_WORDS = re.compile( + r"\b(hosting|cloud|datacenter|data[ -]cent(?:er|re)|vps|servers?|colo(?:cation)?|dedicated|cdn|vpn|proxy|hoster|webhost\w*)\b", + re.IGNORECASE, +) +MOBILE_WORDS = re.compile( + r"\b(mobile|mobil|mobilfunk|wireless|cellular|lte|gsm|3g|4g|5g)\b", re.IGNORECASE +) + + +def classify(asn: int | str | None, org: str | None) -> str: + try: + number = int(asn) if asn not in (None, "") else 0 + except (TypeError, ValueError): + number = 0 + name = (org or "").strip() + if number <= 0 and not name: + return CLASS_UNKNOWN + if number in DATACENTER_ASNS or DATACENTER_WORDS.search(name): + return CLASS_DATACENTER + if MOBILE_WORDS.search(name): + return CLASS_MOBILE + return CLASS_RESIDENTIAL + diff --git a/devplacepy/services/nodes/broker.py b/devplacepy/services/nodes/broker.py index 35f3bf40..96b71517 100644 --- a/devplacepy/services/nodes/broker.py +++ b/devplacepy/services/nodes/broker.py @@ -12,6 +12,7 @@ from dataclasses import dataclass from devplacepy.database import nodes as store from devplacepy.services.nodes.policy import Target from devplacepy.services.nodes.spec import SPEC +from devplacepy.services.nodes.wake import HUB LIMITS = SPEC.limits @@ -72,19 +73,37 @@ def decode_body(body_b64: str) -> tuple[bytes, bool]: class JobBroker: - def enqueue(self, caller_kind: str, caller_id: str, target: Target, mode: str) -> str: - return store.enqueue_request( - caller_kind, caller_id, target.url, target.method, target.adapter, mode + def enqueue( + self, + caller_kind: str, + caller_id: str, + target: Target, + mode: str, + *, + shadow_of: str = "", + exclude_device: str = "", + ) -> str: + uid = store.enqueue_request( + caller_kind, + caller_id, + target.url, + target.method, + target.adapter, + mode, + shadow_of=shadow_of, + exclude_device=exclude_device, ) + HUB.notify_queued() + return uid def queued(self, limit: int = LIMITS.dispatch_batch) -> list[dict]: return store.list_queued(limit) - def claim(self, uid: str, device_uid: str) -> bool: - return store.claim_request(uid, device_uid) + def claim(self, uid: str, device_uid: str, asn_class: str = "") -> bool: + return store.claim_request(uid, device_uid, asn_class) def complete(self, uid: str, http_status: int, headers: dict, body: bytes, truncated: bool) -> bool: - return store.finish_request( + done = store.finish_request( uid, result_status=http_status, result_headers=json.dumps(headers), @@ -92,9 +111,15 @@ class JobBroker: result_bytes=len(body), truncated=truncated, ) + if done: + HUB.job_done(uid) + return done def fail(self, uid: str, error: str, *, expired: bool = False) -> bool: - return store.fail_request(uid, error, expired=expired) + failed = store.fail_request(uid, error, expired=expired) + if failed: + HUB.job_done(uid) + return failed def expire(self) -> dict[str, int]: return store.expire_requests() @@ -117,11 +142,22 @@ class JobBroker: async def wait(self, uid: str, timeout: float) -> BrokerResult: deadline = time.monotonic() + timeout pause = LIMITS.poll_interval_ms / 1000 - while True: - row = store.get_request(uid) - if row is not None and row["status"] in TERMINAL: - return self.read(uid) - if time.monotonic() >= deadline: - self.fail(uid, "caller_timeout") - return self.read(uid) - await asyncio.sleep(pause) + event = HUB.job_event(uid) + try: + while True: + row = store.get_request(uid) + if row is not None and row["status"] in TERMINAL: + return self.read(uid) + if time.monotonic() >= deadline: + self.fail(uid, "caller_timeout") + return self.read(uid) + if event is None: + await asyncio.sleep(pause) + continue + event.clear() + try: + await asyncio.wait_for(event.wait(), pause) + except asyncio.TimeoutError: + pass + finally: + HUB.forget(uid) diff --git a/devplacepy/services/nodes/cache.py b/devplacepy/services/nodes/cache.py index e0c24649..455f6f72 100644 --- a/devplacepy/services/nodes/cache.py +++ b/devplacepy/services/nodes/cache.py @@ -2,12 +2,16 @@ from __future__ import annotations +import base64 import hashlib +import json import time +from datetime import datetime, timezone from collections import OrderedDict from dataclasses import dataclass from typing import Callable +from devplacepy.database import nodes as store from devplacepy.services.nodes.spec import SPEC LIMITS = SPEC.limits @@ -70,7 +74,14 @@ class ResultCache: return entry def put( - self, method: str, url: str, status: int, headers: dict, body: bytes, truncated: bool + self, + method: str, + url: str, + status: int, + headers: dict, + body: bytes, + truncated: bool, + age_seconds: float = 0.0, ) -> bool: if self.ttl_seconds <= 0 or status >= 500 or truncated: return False @@ -78,7 +89,9 @@ class ResultCache: return False key = cache_key(method, url) self._drop(key) - self._entries[key] = CachedResult(status, dict(headers), body, truncated, self._clock()) + self._entries[key] = CachedResult( + status, dict(headers), body, truncated, self._clock() - age_seconds + ) self._bytes += len(body) while self._bytes > self.max_bytes and self._entries: oldest = next(iter(self._entries)) @@ -88,3 +101,56 @@ class ResultCache: def clear(self) -> None: self._entries.clear() self._bytes = 0 + + +class SharedResultCache(ResultCache): + def get(self, method: str, url: str) -> CachedResult | None: + local = super().get(method, url) + if local is not None: + return local + row = store.cache_get(cache_key(method, url)) + if row is None: + return None + try: + headers = json.loads(row["headers"] or "{}") + body = base64.b64decode(row["body"] or "") + stored = datetime.fromisoformat(row["stored_at"]) + except (ValueError, TypeError): + return None + age = max(0.0, (datetime.now(timezone.utc) - stored).total_seconds()) + if age >= self.ttl_seconds: + return None + super().put(method, url, int(row["status"]), headers, body, False, age_seconds=age) + entry = self._entries.get(cache_key(method, url)) + if entry is not None: + self.misses -= 1 + self.hits += 1 + return entry + + def put( + self, + method: str, + url: str, + status: int, + headers: dict, + body: bytes, + truncated: bool, + age_seconds: float = 0.0, + ) -> bool: + kept = super().put(method, url, status, headers, body, truncated, age_seconds) + if kept: + store.cache_put( + cache_key(method, url), + method.upper(), + url, + status, + json.dumps(dict(headers)), + base64.b64encode(body).decode(), + len(body), + self.ttl_seconds, + ) + return kept + + def clear(self) -> None: + super().clear() + store.cache_clear() diff --git a/devplacepy/services/nodes/circuit.py b/devplacepy/services/nodes/circuit.py index 44019ea3..0a1f8bfb 100644 --- a/devplacepy/services/nodes/circuit.py +++ b/devplacepy/services/nodes/circuit.py @@ -28,11 +28,14 @@ class CircuitBreaker: failures: int = LIMITS.breaker_failures, open_seconds: float = LIMITS.breaker_open_seconds, clock: Callable[[], float] = time.monotonic, + wall: Callable[[], float] = time.time, ) -> None: self.failures = max(1, failures) self.open_seconds = open_seconds self._clock = clock + self._wall = wall self._circuits: dict[Hashable, _Circuit] = {} + self.dirty = False def state(self, key: Hashable) -> str: circuit = self._circuits.get(key) @@ -55,7 +58,8 @@ class CircuitBreaker: self._circuits[key].probing = True def success(self, key: Hashable) -> None: - self._circuits.pop(key, None) + if self._circuits.pop(key, None) is not None: + self.dirty = True def failure(self, key: Hashable) -> None: circuit = self._circuits.setdefault(key, _Circuit()) @@ -63,9 +67,49 @@ class CircuitBreaker: circuit.failures += 1 if circuit.failures >= self.failures: circuit.opened_at = self._clock() + self.dirty = True def forget(self, key: Hashable) -> None: - self._circuits.pop(key, None) + if self._circuits.pop(key, None) is not None: + self.dirty = True def open_count(self) -> int: return sum(1 for key in list(self._circuits) if self.state(key) != CLOSED) + + def snapshot(self) -> list[dict]: + now_clock, now_wall = self._clock(), self._wall() + rows = [] + for key, circuit in list(self._circuits.items()): + state = self.state(key) + rows.append( + { + "key": ":".join(str(part) for part in key) if isinstance(key, tuple) else str(key), + "failures": circuit.failures, + "state": state, + "opened_at": now_wall - (now_clock - circuit.opened_at) if circuit.failures >= self.failures else 0.0, + "seconds_left": max(0.0, self.open_seconds - (now_clock - circuit.opened_at)) + if state == OPEN + else 0.0, + } + ) + rows.sort(key=lambda item: (-item["failures"], item["key"])) + return rows + + def restore(self, rows: list[dict]) -> int: + now_clock, now_wall = self._clock(), self._wall() + restored = 0 + for row in rows: + key = tuple(str(row["key"]).split(":", 1)) + if len(key) != 2: + continue + failures = int(row.get("failures") or 0) + opened_wall = float(row.get("opened_at") or 0.0) + if failures < self.failures or opened_wall <= 0: + continue + age = max(0.0, now_wall - opened_wall) + if age >= self.open_seconds * 2: + continue + circuit = _Circuit(failures=failures, opened_at=now_clock - age) + self._circuits[key] = circuit + restored += 1 + return restored diff --git a/devplacepy/services/nodes/dispatcher.py b/devplacepy/services/nodes/dispatcher.py index 3187c886..c86b61ae 100644 --- a/devplacepy/services/nodes/dispatcher.py +++ b/devplacepy/services/nodes/dispatcher.py @@ -3,16 +3,21 @@ from __future__ import annotations import asyncio +import base64 +import hashlib +import json import logging import time from typing import Callable +from devplacepy.database import get_int_setting from devplacepy.database import nodes as store from devplacepy.services.nodes.broker import BodyError, JobBroker, clean_headers, decode_body from devplacepy.services.nodes.circuit import CircuitBreaker from devplacepy.services.nodes.ledger import UsageLedger from devplacepy.services.nodes.policy import DestinationPolicy, PolicyError, Target from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry +from devplacepy.services.nodes.shape import INCONCLUSIVE, MATCH, compare, shape_of from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler from devplacepy.services.nodes.session import Inflight, NodeSession from devplacepy.services.nodes.spec import MODE_FETCH, MODE_TUNNEL, SPEC @@ -31,9 +36,13 @@ REQUEST_OVERHEAD_BYTES = 256 DONOR_LIMIT_PAUSE_SECONDS = 300 CONCURRENCY_PAUSE_SECONDS = 5 THROTTLED_STATUSES = (429,) +BLOCKED_STATUSES = (401, 403, 429, 451) +SHADOW_KIND = "shadow" +REDUNDANCY_SETTING = "nodes_redundancy_percent" DONOR_LIMIT_CODES = { "bandwidth_exhausted": None, "outside_hours": DONOR_LIMIT_PAUSE_SECONDS, + "on_battery": DONOR_LIMIT_PAUSE_SECONDS, "concurrency_exceeded": CONCURRENCY_PAUSE_SECONDS, } REFUSAL_CODES = frozenset(item.code for item in SPEC.refusals) - set(DONOR_LIMIT_CODES) - { @@ -91,16 +100,24 @@ class NodeDispatcher: self.broker.fail(uid, exc.code) self.counters.inc("jobs_refused") return 0 - key = (target.destination.suffix, row["mode"]) + excluded = row.get("exclude_device") or "" + key = (target.destination.suffix, row["mode"], excluded) if key in saturated: return 0 owner = row["caller_id"] if row["caller_kind"] == "user" else "" - found = self.scheduler.candidates(target, row["mode"]) + if excluded: + prior = self.registry.get(excluded) + owner = prior.user_uid if prior is not None else owner + found = [ + item + for item in self.scheduler.candidates(target, row["mode"]) + if item.device_uid != excluded + ] if not found: saturated.add(key) return 0 session = self.scheduler.choose(found, owner) - if session is None or not self.broker.claim(uid, session.device_uid): + if session is None or not self.broker.claim(uid, session.device_uid, session.asn_class): return 0 job = Inflight( job_id=uid, @@ -110,10 +127,13 @@ class NodeDispatcher: url=target.url, started=self._clock(), bytes_up=len(target.url.encode()) + REQUEST_OVERHEAD_BYTES, + adapter=target.adapter, + shadow_of=row.get("shadow_of") or "", ) session.inflight[uid] = job self.scheduler.mark_started(session, target) transport = self.scheduler.transport_of(session, target, row["mode"]) or MODE_FETCH + job.transport = transport store.set_request_transport(uid, transport) if transport == MODE_TUNNEL: self._tunnel_tasks[uid] = asyncio.create_task(self._run_tunnel(session, job, target)) @@ -140,6 +160,7 @@ class NodeDispatcher: self.counters.inc("jobs_failed") if node_fault: self.breaker.failure(("node", session.device_uid)) + self._observe(session, job, "failed") store.add_event("job_failed", device_uid=session.device_uid, user_uid=session.user_uid, detail=code) self.ledger.record_failure(session.device_uid, session.user_uid) @@ -196,6 +217,7 @@ class NodeDispatcher: if not self.broker.complete(job.job_id, status, headers, body, truncated): return self._account(session, job, status, job.bytes_up, len(body)) + self._after_result(session, job, status, headers, body) async def _run_tunnel(self, session: NodeSession, job: Inflight, target: Target) -> None: try: @@ -215,9 +237,8 @@ class NodeDispatcher: return if result.truncated: self.counters.inc("results_truncated") - if not self.broker.complete( - job.job_id, result.status, clean_headers(result.headers), result.body, result.truncated - ): + headers = clean_headers(result.headers) + if not self.broker.complete(job.job_id, result.status, headers, result.body, result.truncated): return self._account(session, job, result.status, result.bytes_up, result.bytes_down) @@ -237,12 +258,84 @@ class NodeDispatcher: if not accepted: session.used_today = int(session.settings.get("bandwidth_mb_day") or 0) * MEGABYTE self.breaker.success(("node", session.device_uid)) + self._observe(session, job, "blocked" if status >= 500 or status in BLOCKED_STATUSES else "ok") destination = ("destination", job.destination) if status >= 500 or status in THROTTLED_STATUSES: self.breaker.failure(destination) else: self.breaker.success(destination) + def _observe(self, session: NodeSession, job: Inflight, kind: str) -> None: + if job.shadow_of or not job.adapter: + return + try: + self.counters.observe(job.adapter, session.asn_class, kind) + except KeyError: + logger.warning("outcome counter name rejected for %s", job.adapter) + + @staticmethod + def redundancy_percent() -> int: + configured = get_int_setting(REDUNDANCY_SETTING, LIMITS.redundancy_percent) + return max(0, min(configured, LIMITS.redundancy_max_percent)) + + @staticmethod + def sampled(uid: str, percent: int) -> bool: + if percent <= 0: + return False + return int(hashlib.sha256(uid.encode()).hexdigest()[:8], 16) % 100 < percent + + def _after_result(self, session: NodeSession, job: Inflight, status: int, headers: dict, body: bytes) -> None: + if job.shadow_of: + self._compare_shadow(session, job, status, headers, body) + return + if job.transport != MODE_FETCH or not self.sampled(job.job_id, self.redundancy_percent()): + return + row = store.get_request(job.job_id) or {} + try: + target = self.policy.check(job.url, job.method, MODE_FETCH) + except PolicyError: + return + try: + self.broker.enqueue( + SHADOW_KIND, + job.job_id, + target, + MODE_FETCH, + shadow_of=job.job_id, + exclude_device=row.get("device_uid") or session.device_uid, + ) + except store.QueueFull: + return + + def _compare_shadow(self, session: NodeSession, job: Inflight, status: int, headers: dict, body: bytes) -> None: + original = store.get_request(job.shadow_of) + if original is None or original.get("status") != "done": + self.counters.inc("redundancy_inconclusive") + return + try: + first_body = base64.b64decode(original.get("result_body") or "") + first_headers = json.loads(original.get("result_headers") or "{}") + except (ValueError, TypeError): + self.counters.inc("redundancy_inconclusive") + return + verdict = compare( + shape_of(int(original.get("result_status") or 0), first_headers, first_body), + shape_of(status, headers, body), + ) + self.counters.inc("redundancy_checked") + if verdict == MATCH: + self.counters.inc("redundancy_matched") + elif verdict == INCONCLUSIVE: + self.counters.inc("redundancy_inconclusive") + else: + self.counters.inc("redundancy_mismatches") + store.raise_flag( + "redundancy_mismatch", + device_uid=session.device_uid, + user_uid=session.user_uid, + detail=f"{job.destination} first={original.get('device_uid')} second={session.device_uid}", + ) + async def sweep_inflight(self) -> int: limit = LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds now = self._clock() diff --git a/devplacepy/services/nodes/fabric.py b/devplacepy/services/nodes/fabric.py index 205c882a..cb3e0a37 100644 --- a/devplacepy/services/nodes/fabric.py +++ b/devplacepy/services/nodes/fabric.py @@ -11,7 +11,7 @@ from devplacepy.database import get_int_setting, get_table from devplacepy.database import nodes as store from devplacepy.services.base import BaseService from devplacepy.services.nodes.broker import FAILED, JobBroker -from devplacepy.services.nodes.cache import ResultCache +from devplacepy.services.nodes.cache import ResultCache, SharedResultCache from devplacepy.services.nodes.consent import ConsentGate from devplacepy.services.nodes.policy import DestinationPolicy, PolicyError, Target from devplacepy.services.nodes.spec import SPEC @@ -27,6 +27,7 @@ RETRYABLE = frozenset( "send_failed", "concurrency_exceeded", "outside_hours", + "on_battery", "bandwidth_exhausted", } ) @@ -81,7 +82,7 @@ class NodeFabric: ) -> None: self.policy = policy or DestinationPolicy() self.broker = broker or JobBroker() - self.cache = cache or ResultCache() + self.cache = cache or SharedResultCache() self.gate = gate or ConsentGate() def available(self) -> FabricStatus: diff --git a/devplacepy/services/nodes/geo.py b/devplacepy/services/nodes/geo.py new file mode 100644 index 00000000..60918ba3 --- /dev/null +++ b/devplacepy/services/nodes/geo.py @@ -0,0 +1,293 @@ +# retoor + +from __future__ import annotations + +import csv +import gzip +import ipaddress +import logging +import os +import sqlite3 +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from devplacepy import config +from devplacepy.services.nodes.addressclass import classify + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class GeoSpec: + asn_url: str = "https://download.db-ip.com/free/dbip-asn-lite-{month}.csv.gz" + country_url: str = "https://download.db-ip.com/free/dbip-country-lite-{month}.csv.gz" + attribution: str = "IP geolocation by DB-IP.com (CC BY 4.0)" + max_rows: int = 2_000_000 + min_asn_rows: int = 100_000 + min_country_rows: int = 100_000 + insert_chunk: int = 5000 + download_timeout_seconds: int = 180 + max_download_bytes: int = 64 * 1024 * 1024 + months_back: int = 2 + checks: tuple[tuple[str, int], ...] = (("1.1.1.1", 13335), ("8.8.8.8", 15169)) + + +SPEC = GeoSpec() + +SCHEMA = ( + "CREATE TABLE geo_asn (family INTEGER NOT NULL, first BLOB NOT NULL, last BLOB NOT NULL, " + "asn INTEGER NOT NULL, org TEXT NOT NULL, PRIMARY KEY (family, first)) WITHOUT ROWID", + "CREATE TABLE geo_country (family INTEGER NOT NULL, first BLOB NOT NULL, last BLOB NOT NULL, " + "country TEXT NOT NULL, PRIMARY KEY (family, first)) WITHOUT ROWID", + "CREATE TABLE geo_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)", +) + + +UNKNOWN_COUNTRY = "ZZ" + + +class GeoError(RuntimeError): + pass + + +@dataclass(frozen=True) +class GeoResult: + country: str = "" + asn: int = 0 + org: str = "" + asn_class: str = "unknown" + + +def pack(address: str) -> tuple[int, bytes]: + ip = ipaddress.ip_address(address.strip().strip("[]")) + if ip.version == 6 and ip.ipv4_mapped is not None: + ip = ip.ipv4_mapped + return ip.version, ip.packed + + +class GeoDatabase: + def __init__(self, path: Path | None = None) -> None: + self.path = Path(path) if path else config.NODE_GEO_DB + self.warned = False + + def available(self) -> bool: + return self.path.is_file() and self.path.stat().st_size > 0 + + def _connect(self) -> sqlite3.Connection: + return sqlite3.connect(f"file:{self.path}?mode=ro", uri=True, timeout=5) + + def _range(self, conn: sqlite3.Connection, table: str, columns: str, family: int, packed: bytes): + row = conn.execute( + f"SELECT {columns}, last FROM {table} WHERE family = ? AND first <= ? " + "ORDER BY first DESC LIMIT 1", + (family, packed), + ).fetchone() + if row is None or row[-1] < packed: + return None + return row[:-1] + + def lookup(self, address: str) -> GeoResult: + if not address or not self.available(): + return GeoResult() + try: + family, packed = pack(address) + except ValueError: + return GeoResult() + try: + conn = self._connect() + except sqlite3.Error: + return GeoResult() + try: + asn_row = self._range(conn, "geo_asn", "asn, org", family, packed) + country_row = self._range(conn, "geo_country", "country", family, packed) + except sqlite3.Error: + return GeoResult() + finally: + conn.close() + asn, org = (int(asn_row[0]), str(asn_row[1])) if asn_row else (0, "") + country = str(country_row[0]) if country_row else "" + country = "" if country.upper() == UNKNOWN_COUNTRY else country + return GeoResult(country=country[:2].upper(), asn=asn, org=org, asn_class=classify(asn, org)) + + def status(self) -> dict: + if not self.available(): + return {"available": False, "path": str(self.path)} + conn = self._connect() + try: + meta = dict(conn.execute("SELECT key, value FROM geo_meta").fetchall()) + except sqlite3.Error: + meta = {} + finally: + conn.close() + return {"available": True, "path": str(self.path), **meta, "attribution": SPEC.attribution} + + +def months(now: datetime | None = None) -> list[str]: + moment = now or datetime.now(timezone.utc) + found = [] + for back in range(SPEC.months_back + 1): + year, month = moment.year, moment.month - back + while month < 1: + month += 12 + year -= 1 + found.append(f"{year:04d}-{month:02d}") + return found + + +def parse_asn_rows(lines): + for row in csv.reader(lines): + if len(row) < 4: + continue + first, last, asn, org = row[0], row[1], row[2], row[3] + try: + family, low = pack(first) + _, high = pack(last) + number = int(asn) + except ValueError: + continue + yield family, low, high, number, org[:200] + + +def parse_country_rows(lines): + for row in csv.reader(lines): + if len(row) < 3: + continue + try: + family, low = pack(row[0]) + _, high = pack(row[1]) + except ValueError: + continue + yield family, low, high, row[2][:2].upper() + + +def _insert(conn: sqlite3.Connection, sql: str, rows) -> int: + count = 0 + batch = [] + for row in rows: + batch.append(row) + count += 1 + if count > SPEC.max_rows: + raise GeoError(f"more than {SPEC.max_rows} rows, refusing to load") + if len(batch) >= SPEC.insert_chunk: + conn.executemany(sql, batch) + batch = [] + if batch: + conn.executemany(sql, batch) + return count + + +def build(asn_lines, country_lines, target: Path, month: str) -> dict: + temporary = target.with_suffix(".tmp") + temporary.unlink(missing_ok=True) + conn = sqlite3.connect(temporary) + try: + for statement in SCHEMA: + conn.execute(statement) + asn_rows = _insert(conn, "INSERT OR REPLACE INTO geo_asn VALUES (?,?,?,?,?)", parse_asn_rows(asn_lines)) + country_rows = _insert(conn, "INSERT OR REPLACE INTO geo_country VALUES (?,?,?,?)", parse_country_rows(country_lines)) + if asn_rows < SPEC.min_asn_rows or country_rows < SPEC.min_country_rows: + raise GeoError( + f"dataset too small (asn {asn_rows}, country {country_rows}), refusing to replace the current one" + ) + conn.executemany( + "INSERT INTO geo_meta VALUES (?,?)", + [ + ("month", month), + ("loaded_at", datetime.now(timezone.utc).isoformat()), + ("asn_rows", str(asn_rows)), + ("country_rows", str(country_rows)), + ], + ) + conn.commit() + except Exception: + conn.close() + temporary.unlink(missing_ok=True) + raise + conn.close() + verify(temporary) + return {"month": month, "asn_rows": asn_rows, "country_rows": country_rows, "temporary": temporary} + + +def verify(path: Path) -> None: + probe = GeoDatabase(path) + for address, expected in SPEC.checks: + found = probe.lookup(address).asn + if found != expected: + path.unlink(missing_ok=True) + raise GeoError(f"sanity check failed: {address} resolved to AS{found}, expected AS{expected}") + + +def install(built: dict, target: Path) -> None: + os.replace(built["temporary"], target) + + +async def _download(url: str, destination: Path) -> bool: + from devplacepy.stealth import stealth_async_client + + async with stealth_async_client(timeout=SPEC.download_timeout_seconds, follow_redirects=True) as client: + async with client.stream("GET", url) as response: + if response.status_code == 404: + return False + if response.status_code != 200: + raise GeoError(f"{url} answered {response.status_code}") + total = 0 + with open(destination, "wb") as handle: + async for chunk in response.aiter_bytes(): + total += len(chunk) + if total > SPEC.max_download_bytes: + raise GeoError(f"{url} exceeds {SPEC.max_download_bytes} bytes") + handle.write(chunk) + return True + + +async def fetch_month(month: str, workdir: Path) -> tuple[Path, Path] | None: + asn_file = workdir / f"asn-{month}.csv.gz" + country_file = workdir / f"country-{month}.csv.gz" + if not await _download(SPEC.asn_url.format(month=month), asn_file): + return None + if not await _download(SPEC.country_url.format(month=month), country_file): + return None + return asn_file, country_file + + +def build_from_files(asn_file: Path, country_file: Path, target: Path, month: str) -> dict: + with gzip.open(asn_file, "rt", encoding="utf-8", newline="") as asn_handle, gzip.open( + country_file, "rt", encoding="utf-8", newline="" + ) as country_handle: + return build(asn_handle, country_handle, target, month) + + +async def geo_sync(*, dry_run: bool = False, month: str = "", target: Path | None = None, workdir: Path | None = None) -> dict: + destination = Path(target) if target else config.NODE_GEO_DB + destination.parent.mkdir(parents=True, exist_ok=True) + scratch = Path(workdir) if workdir else destination.parent + scratch.mkdir(parents=True, exist_ok=True) + candidates = [month] if month else months() + for candidate in candidates: + files = await fetch_month(candidate, scratch) + if files is None: + continue + try: + built = build_from_files(files[0], files[1], destination, candidate) + finally: + for item in files: + item.unlink(missing_ok=True) + summary = {key: value for key, value in built.items() if key != "temporary"} + if dry_run: + built["temporary"].unlink(missing_ok=True) + return {**summary, "installed": False} + install(built, destination) + return {**summary, "installed": True} + raise GeoError(f"no dataset found for {', '.join(candidates)}") + + +_shared: GeoDatabase | None = None + + +def shared_geo() -> GeoDatabase: + global _shared + if _shared is None or _shared.path != config.NODE_GEO_DB: + _shared = GeoDatabase() + return _shared + diff --git a/devplacepy/services/nodes/registry.py b/devplacepy/services/nodes/registry.py index f3ceb487..2e4e56b8 100644 --- a/devplacepy/services/nodes/registry.py +++ b/devplacepy/services/nodes/registry.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from collections import Counter from devplacepy.database import get_int_setting @@ -26,21 +27,57 @@ COUNTER_NAMES: tuple[str, ...] = ( "jobs_refused", "integrity_failures", "results_truncated", + "geo_unavailable", + "redundancy_checked", + "redundancy_matched", + "redundancy_mismatches", + "redundancy_inconclusive", ) +OUTCOME_PREFIX = "outcome:" +OUTCOME_KINDS = ("ok", "blocked", "failed") +OUTCOME_PATTERN = re.compile(r"^outcome:[a-z0-9_]{1,24}:[a-z]{1,12}:(ok|blocked|failed)$") + + +def outcome_name(adapter: str, asn_class: str, kind: str) -> str: + name = f"{OUTCOME_PREFIX}{adapter}:{asn_class or 'unknown'}:{kind}" + if not OUTCOME_PATTERN.match(name): + raise KeyError(name) + return name + class NodeCounters: def __init__(self) -> None: self._counts: Counter[str] = Counter({name: 0 for name in COUNTER_NAMES}) + self._outcomes: Counter[str] = Counter() + self._pending: Counter[str] = Counter() def inc(self, name: str) -> None: if name not in COUNTER_NAMES: raise KeyError(name) self._counts[name] += 1 + self._pending[name] += 1 + + def observe(self, adapter: str, asn_class: str, kind: str) -> None: + name = outcome_name(adapter, asn_class, kind) + self._outcomes[name] += 1 + self._pending[name] += 1 def snapshot(self) -> dict[str, int]: return {name: self._counts[name] for name in COUNTER_NAMES} + def outcomes(self) -> dict[str, int]: + return dict(self._outcomes) + + def take_pending(self) -> dict[str, int]: + taken = {name: value for name, value in self._pending.items() if value > 0} + self._pending.clear() + return taken + + def restore_pending(self, taken: dict[str, int]) -> None: + for name, value in taken.items(): + self._pending[name] += value + class NodeRegistry: def __init__(self, counters: NodeCounters) -> None: @@ -78,10 +115,10 @@ class NodeRegistry: if self._sessions.get(session.device_uid) is session: del self._sessions[session.device_uid] - async def kick(self, session: NodeSession, reason: str) -> None: + async def kick(self, session: NodeSession, reason: str, until: str = "") -> None: self.release(session) self.counters.inc("kicked") - await session.kick(reason) + await session.kick(reason, until) async def kick_all(self, reason: str) -> int: sessions = self.sessions() diff --git a/devplacepy/services/nodes/scheduler.py b/devplacepy/services/nodes/scheduler.py index 3b1708a0..7e93e1bf 100644 --- a/devplacepy/services/nodes/scheduler.py +++ b/devplacepy/services/nodes/scheduler.py @@ -9,6 +9,8 @@ from datetime import datetime, timezone from typing import Callable from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from devplacepy.database import get_int_setting +from devplacepy.services.nodes.addressclass import CLASS_UNKNOWN, ELIGIBLE_CLASSES from devplacepy.services.nodes.circuit import CircuitBreaker from devplacepy.services.nodes.policy import Target from devplacepy.services.nodes.registry import NodeRegistry @@ -20,6 +22,8 @@ MEGABYTE = 1024 * 1024 MINUTES_PER_DAY = 24 * 60 SUPPORTED_MODES = (MODE_TUNNEL, MODE_FETCH) +OVERRIDE_ALLOW = "allow" +ALLOW_UNKNOWN_SETTING = "nodes_allow_unknown_class" def _minutes(value: str) -> int | None: @@ -73,6 +77,7 @@ class NodeScheduler: self._wall = wall self._rng = rng or random.Random() self._destination_started: dict[str, float] = {} + self.class_refusals = 0 @staticmethod def node_key(session: NodeSession) -> tuple[str, str]: @@ -102,9 +107,30 @@ class NodeScheduler: last = self._destination_started.get(target.destination.suffix) return last is None or self._clock() - last >= interval - def node_ready(self, session: NodeSession, target: Target, transport: str) -> bool: + @staticmethod + def allow_unknown_class() -> bool: + return get_int_setting(ALLOW_UNKNOWN_SETTING, 0) == 1 + + @staticmethod + def class_allowed(session: NodeSession, allow_unknown: bool) -> bool: + if session.class_override == OVERRIDE_ALLOW or session.asn_class in ELIGIBLE_CLASSES: + return True + return allow_unknown and session.asn_class == CLASS_UNKNOWN + + def node_ready( + self, + session: NodeSession, + target: Target, + transport: str, + allow_unknown: bool | None = None, + ) -> bool: if session.closed or self._clock() < session.paused_until: return False + if allow_unknown is None: + allow_unknown = self.allow_unknown_class() + if not self.class_allowed(session, allow_unknown): + self.class_refusals += 1 + return False if transport not in session.modes: return False if len(session.inflight) >= session.max_concurrent: @@ -125,11 +151,12 @@ class NodeScheduler: def candidates(self, target: Target, mode: str = MODE_ANY) -> list[NodeSession]: if not self.destination_ready(target) or not self.breaker.allows(self.destination_key(target)): return [] + allow_unknown = self.allow_unknown_class() for transport in self.transports_for(target, mode): found = [ session for session in self.registry.sessions() - if self.node_ready(session, target, transport) + if self.node_ready(session, target, transport, allow_unknown) ] if found: return found diff --git a/devplacepy/services/nodes/service.py b/devplacepy/services/nodes/service.py index 2b3ba33e..a34a1634 100644 --- a/devplacepy/services/nodes/service.py +++ b/devplacepy/services/nodes/service.py @@ -9,9 +9,14 @@ import time from devplacepy.database.nodes import ( STATUS_ACTIVE, add_event, + cache_prune, effective_status, + flush_counters, get_devices_by_uids, get_settings_by_users, + load_circuits, + save_circuits, + touch_devices, ) from devplacepy.services.base import BaseService, ConfigField from devplacepy.services.nodes.broker import JobBroker @@ -23,6 +28,7 @@ from devplacepy.services.nodes.ledger import UsageLedger from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry from devplacepy.services.nodes.scheduler import NodeScheduler from devplacepy.services.nodes.spec import SPEC +from devplacepy.services.nodes.wake import HUB logger = logging.getLogger(__name__) @@ -84,6 +90,8 @@ class NodesService(BaseService): ) self.fabric = NodeFabric(broker=self.broker, gate=self.gate) self._last_prune = 0.0 + self._last_touch = 0.0 + self._last_flush = 0.0 self._dispatch_task: asyncio.Task | None = None def ensure_dispatching(self) -> None: @@ -91,6 +99,7 @@ class NodesService(BaseService): task = self._dispatch_task if task is not None and not task.done() and task.get_loop() is loop: return + HUB.attach() self._dispatch_task = loop.create_task(self._dispatch_loop()) async def _dispatch_loop(self) -> None: @@ -102,9 +111,12 @@ class NodesService(BaseService): raise except Exception: logger.exception("node dispatch tick failed") - await asyncio.sleep(pause) + await HUB.wait_queued(pause) async def on_enable(self) -> None: + restored = self.breaker.restore(load_circuits()) + if restored: + self.log(f"restored {restored} open circuit(s)") self.ensure_dispatching() async def _refresh_settings(self, sessions) -> None: @@ -127,11 +139,13 @@ class NodesService(BaseService): verdicts = self.gate.allows_many(item.user_uid for item in sessions) kicked = 0 for session in sessions: - reason = self._reason_to_kick( - switched_on, devices.get(session.device_uid), verdicts[session.user_uid] - ) + device = devices.get(session.device_uid) + if device is not None: + session.class_override = device.get("class_override") or "" + reason = self._reason_to_kick(switched_on, device, verdicts[session.user_uid]) if reason: - await self.registry.kick(session, reason) + until = (device.get("suspended_until") or "").strip() if reason == "suspended" and device else "" + await self.registry.kick(session, reason, until) add_event( "kick", device_uid=session.device_uid, @@ -155,13 +169,37 @@ class NodesService(BaseService): def _prune_due(self) -> bool: return time.monotonic() - self._last_prune >= LIMITS.prune_interval_seconds + def persist_state(self, *, force: bool = False) -> None: + now = time.monotonic() + if force or now - self._last_touch >= LIMITS.last_seen_write_seconds: + self._last_touch = now + touch_devices(item.device_uid for item in self.registry.sessions()) + if force or now - self._last_flush >= LIMITS.counter_flush_seconds: + self._last_flush = now + pending = self.counters.take_pending() + try: + flush_counters(pending) + except Exception: + self.counters.restore_pending(pending) + raise + if self.breaker.dirty: + self.breaker.dirty = False + try: + save_circuits(self.breaker.snapshot()) + except Exception: + self.breaker.dirty = True + raise + async def run_once(self) -> None: kicked = await self.reconcile() if kicked: self.log(f"disconnected {kicked} node(s)") + self.persist_state() if self._prune_due(): self._last_prune = time.monotonic() pruned = self.ledger.prune() + removed = cache_prune(LIMITS.cache_max_bytes) + pruned["cache"] = removed["expired"] + removed["evicted"] if any(pruned.values()): self.log(f"pruned {pruned}") @@ -170,6 +208,7 @@ class NodesService(BaseService): self._dispatch_task.cancel() self._dispatch_task = None await self.registry.kick_all("service_stopped") + self.persist_state(force=True) def collect_metrics(self) -> dict: counts = self.counters.snapshot() diff --git a/devplacepy/services/nodes/session.py b/devplacepy/services/nodes/session.py index 5a33d462..98a4c31d 100644 --- a/devplacepy/services/nodes/session.py +++ b/devplacepy/services/nodes/session.py @@ -35,6 +35,9 @@ class Inflight: url: str started: float bytes_up: int + adapter: str = "" + shadow_of: str = "" + transport: str = "" JobSink = Callable[["NodeSession", Inflight, dict], Awaitable[None]] @@ -46,6 +49,8 @@ class NodeSession: self.device = device self.device_uid: str = device["uid"] self.user_uid: str = device["user_uid"] + self.asn_class: str = device.get("asn_class") or "unknown" + self.class_override: str = device.get("class_override") or "" self.connected_at = time.monotonic() self.missed_pings = 0 self.hello: dict = {} @@ -96,11 +101,14 @@ class NodeSession: except Exception: pass - async def kick(self, reason: str) -> None: + async def kick(self, reason: str, until: str = "") -> None: if self.closed: return try: - await self.send("kick", reason=reason) + if until: + await self.send("kick", reason=reason, until=until) + else: + await self.send("kick", reason=reason) except Exception: pass await self.close(CLOSE_KICKED, reason) @@ -135,6 +143,7 @@ class NodeSession: limits=LIMITS.describe(), destinations=[item.describe() for item in SPEC.destinations], settings=self.settings, + device_label=str(self.device.get("label") or ""), ) return frame diff --git a/devplacepy/services/nodes/shape.py b/devplacepy/services/nodes/shape.py new file mode 100644 index 00000000..310a7388 --- /dev/null +++ b/devplacepy/services/nodes/shape.py @@ -0,0 +1,70 @@ +# retoor + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass + +MATCH, MISMATCH, INCONCLUSIVE = "match", "mismatch", "inconclusive" +SIZE_BAND = 4.0 +SMALL_BODY_BYTES = 512 +NESTED_KEY_LIMIT = 64 +TITLE = re.compile(rb"]*>(.*?)", re.IGNORECASE | re.DOTALL) +HEADING = re.compile(rb"]*>(.*?)", re.IGNORECASE | re.DOTALL) +TAGS = re.compile(rb"<[^>]+>") +SPACE = re.compile(rb"\s+") + + +@dataclass(frozen=True) +class Shape: + status: int + kind: str + signature: frozenset | str + size: int + + @property + def clean(self) -> bool: + return 200 <= self.status < 400 + + +def _keys(value, depth: int = 0) -> frozenset: + if not isinstance(value, dict): + return frozenset() + found = set(value) + if depth < 2: + for key, item in list(value.items())[:NESTED_KEY_LIMIT]: + found.update(f"{key}.{inner}" for inner in _keys(item, depth + 1)) + return frozenset(found) + + +def _text(raw: bytes) -> str: + return SPACE.sub(b" ", TAGS.sub(b"", raw)).strip().decode("utf-8", "replace")[:200] + + +def shape_of(status: int, headers: dict, body: bytes) -> Shape: + kind = str(headers.get("content-type", "")).split(";")[0].strip().lower() + size = len(body) + if "json" in kind: + try: + return Shape(status, "json", _keys(json.loads(body)), size) + except ValueError: + return Shape(status, "json-invalid", "", size) + if "html" in kind: + match = TITLE.search(body) or HEADING.search(body) + return Shape(status, "html", _text(match.group(1)) if match else "", size) + return Shape(status, kind or "unknown", "", size) + + +def compare(first: Shape, second: Shape) -> str: + if not (first.clean and second.clean): + return INCONCLUSIVE + if first.kind != second.kind: + return MISMATCH + if first.signature != second.signature: + return MISMATCH + if min(first.size, second.size) >= SMALL_BODY_BYTES: + ratio = max(first.size, second.size) / max(1, min(first.size, second.size)) + if ratio > SIZE_BAND: + return MISMATCH + return MATCH diff --git a/devplacepy/services/nodes/spec.py b/devplacepy/services/nodes/spec.py index 36929ae7..c03493fd 100644 --- a/devplacepy/services/nodes/spec.py +++ b/devplacepy/services/nodes/spec.py @@ -104,6 +104,11 @@ class LimitSpec: dispatch_batch: int = 50 url_max_chars: int = 2048 poll_interval_ms: int = 100 + last_seen_write_seconds: int = 60 + redundancy_percent: int = 5 + redundancy_max_percent: int = 20 + circuit_snapshot_max: int = 100 + counter_flush_seconds: int = 10 def describe(self) -> dict[str, int]: return dict(self.__dict__) @@ -232,6 +237,7 @@ FRAMES: tuple[FrameSpec, ...] = ( _f("limits", "object"), _f("destinations", "list"), _f("settings", "object"), + _f("device_label", "str", False), ), ), FrameSpec("ping", SERVER_TO_CLIENT), @@ -258,7 +264,7 @@ FRAMES: tuple[FrameSpec, ...] = ( ), FrameSpec("tunnel_close", SERVER_TO_CLIENT, (_f("conn_id", "str"),)), FrameSpec("settings_update", SERVER_TO_CLIENT, (_f("settings", "object"),)), - FrameSpec("kick", SERVER_TO_CLIENT, (_f("reason", "str"),)), + FrameSpec("kick", SERVER_TO_CLIENT, (_f("reason", "str"), _f("until", "str", False))), ) DESTINATIONS: tuple[DestinationSpec, ...] = ( @@ -275,6 +281,7 @@ REFUSALS: tuple[RefusalSpec, ...] = ( RefusalSpec("bandwidth_exhausted", "donor daily bandwidth cap reached"), RefusalSpec("concurrency_exceeded", "donor concurrency cap reached"), RefusalSpec("outside_hours", "outside the donor's active hours"), + RefusalSpec("on_battery", "donor is running on battery and asked to pause"), RefusalSpec("body_too_large", "response exceeds the body cap"), RefusalSpec("timeout", "job exceeded its timeout"), ) diff --git a/devplacepy/services/nodes/wake.py b/devplacepy/services/nodes/wake.py new file mode 100644 index 00000000..b64f1c9f --- /dev/null +++ b/devplacepy/services/nodes/wake.py @@ -0,0 +1,70 @@ +# retoor + +from __future__ import annotations + +import asyncio + + +class WakeHub: + def __init__(self) -> None: + self.enabled = True + self._loop: asyncio.AbstractEventLoop | None = None + self._queued: asyncio.Event | None = None + self._jobs: dict[str, asyncio.Event] = {} + + def attach(self) -> None: + self._loop = asyncio.get_running_loop() + self._queued = asyncio.Event() + + def _set(self, event: asyncio.Event) -> None: + loop = self._loop + if loop is None or loop.is_closed(): + return + try: + running = asyncio.get_running_loop() + except RuntimeError: + running = None + if running is loop: + event.set() + else: + loop.call_soon_threadsafe(event.set) + + def notify_queued(self) -> None: + if self.enabled and self._queued is not None: + self._set(self._queued) + + async def wait_queued(self, timeout: float) -> bool: + event = self._queued + if not self.enabled or event is None: + await asyncio.sleep(timeout) + return False + try: + await asyncio.wait_for(event.wait(), timeout) + except asyncio.TimeoutError: + return False + event.clear() + return True + + def job_event(self, uid: str) -> asyncio.Event | None: + if not self.enabled or self._loop is None: + return None + try: + if asyncio.get_running_loop() is not self._loop: + return None + except RuntimeError: + return None + event = self._jobs.get(uid) + if event is None: + event = self._jobs[uid] = asyncio.Event() + return event + + def job_done(self, uid: str) -> None: + event = self._jobs.get(uid) + if event is not None: + self._set(event) + + def forget(self, uid: str) -> None: + self._jobs.pop(uid, None) + + +HUB = WakeHub() diff --git a/devplacepy/services/skills/CLAUDE.md b/devplacepy/services/skills/CLAUDE.md index 88e61627..95190b5b 100644 --- a/devplacepy/services/skills/CLAUDE.md +++ b/devplacepy/services/skills/CLAUDE.md @@ -44,4 +44,4 @@ A `SKILL.md` file (YAML frontmatter with a required `name` and `description`, th ## Gaps -- No test installs from a live network origin. The SSRF guard refuses the loopback address a local test server would use, so the network leg (`guarded_async_client` against a real socket) is exercised only in production. +- The network leg is covered by `tests/unit/services/skills/fetcher.py`, which serves a real local socket and reaches it through the host's own global IPv4 so the SSRF guard runs unpatched (bundled files, directory URL, loopback and redirect-to-loopback refused, redirect loop, oversize, HTML, 404; 36 tests). It skips with a stated reason when the host has no global IPv4 (a CI runner behind NAT). diff --git a/devplacepy/services/sync/CLAUDE.md b/devplacepy/services/sync/CLAUDE.md index ca87169f..edd5b69e 100644 --- a/devplacepy/services/sync/CLAUDE.md +++ b/devplacepy/services/sync/CLAUDE.md @@ -14,11 +14,12 @@ This file documents community sync (`devplacepy/services/sync/`, `devplacepy/rou | Devii tools `sync_users`, `sync_posts`, `sync_comments` | `services/devii/actions/catalog/sync.py` | built, `requires_admin`; Devii sends forms, so `records` is a JSON array string (`SyncBatchForm.decode_records`) | | Admin settings fields `sync_enabled`, `sync_origins` | `AdminSettingsForm`, `admin_settings.html`, `admin_save_settings` | built, origins normalized on save | | Scheduled sender | `services/community/` (`CommunityService`, see its CLAUDE.md) | built, opt-in, dry run by default | -| Export endpoint, vote and reaction sync | none | not built, see Gaps | +| Projects, polls, votes, reactions, follows, poll votes, attachments | `processors.py`, one leaf per kind in `routers/sync/` | built, tested | +| Pending queue, export, deletions, alias healing | `pending.py`, `exporter.py`, `deletions.py`, `healing.py` | built, tested | ## Model -Every synced row has a `sync_links` row `(origin, kind, origin_uid) -> local_uid`, unique on the triple. `origin` is the sender's stable id (a hostname, lower-cased, 2-63 characters of `a-z0-9.-`). `kind` is `users`, `posts` or `comments`. The link also stores a `fingerprint` of the normalized record (sha256 prefix, excludes `updated_at`) and `origin_updated_at`. +Every synced row has a `sync_links` row `(origin, kind, origin_uid) -> local_uid`, unique on the triple. `origin` is the sender's stable id (a hostname, lower-cased, 2-63 characters of `a-z0-9.-`). `kind` is one of the ten in `spec.py` (`users`, `projects`, `posts`, `comments`, `polls`, `votes`, `reactions`, `follows`, `poll_votes`, `attachments`). The link also stores a `fingerprint` of the normalized record (sha256 prefix, excludes `updated_at`) and `origin_updated_at`. Per record, `RecordProcessor.apply` does: validate (Pydantic), resolve references (author, post, parent) through the links, content filter, then: @@ -52,13 +53,31 @@ The link is claimed with one `INSERT OR IGNORE` against the unique index and `ro A stateful fuzz (3000 random upserts, tombstones and edits over two origins and all three kinds, invariants checked every 100 steps: no dangling links, no duplicate usernames, no orphan comments, parents on the same post, `updated_at >= created_at`) produced no `failed` result and no violation. +## Added since the first build + +- **Ten kinds.** Projects (a post's `project_uid` is omitted from the fingerprint when None), polls with options, votes and reactions (`recompute_stars` is one atomic UPDATE, so stars match the vote rows), follows (natural-key upsert, `self_follow` refused), poll votes, attachments (base64 content with a sha256 fingerprint, capped by `attachment_bytes`). +- **Pending queue (`pending.py`, table `sync_pending`).** A record whose post, parent, poll or project is not here yet is `deferred` and parked, bounded by a per-origin cap (`pending_queue_full` when exceeded) and expiry; `PendingQueue.release` retries it when the missing record arrives, and the batch result carries `released`. `unknown_option` is never deferred. Measured: 16 processes sending one post, one user and a comment in shuffled order left 16 of 16 comment links and 0 parked records. +- **Export (`GET /sync/export/{kind}`).** Keyset pagination over `(changed, uid)` with expression indexes (`EXPORT_CHANGED`, `ensure_export_indexes`), an echo guard that never re-exports a row that arrived by sync, public data only (private projects are not exported, a post's private project link is redacted), audited as `sync.export`. Votes, reactions and poll votes are not exported because no surface exposes who cast them. +- **Deletions (`GET /sync/deletions`, `POST /sync/deletions/ack`).** The origin learns which of its records we removed; an acknowledged record answers `acknowledged_deleted` afterwards instead of resurrecting. Audited as `sync.deletions.ack`. +- **Alias healing (`healing.py`, `AliasHealer`).** Follows an administrator rename of a clone and suffixed clones, rewrites stored mentions, at most `heal_rewrites_per_run` rewrites a run, runs before and after the push. +- **Candidates.** `Selector(persist=)` parks posts held back by `min_comments` in `sync_candidates` and revives them when later runs see enough comments. + +Measured: 40 processes racing one new vote against one post gave 1 `created`, 39 `unchanged`, 1 vote row, 1 link and `stars = 1`. + +## Realistic handles for new community users (`handles.py`) + +Names that reach us from a feed are often not handles at all (`abner_page_by_abnercoimbre`, `libreboot_org_via_runxiyu`, `via_rberger_2026-10-15_12_36_38_`). `HandleBuilder` (spec: `HandleLimits` / `HANDLES` in `spec.py`) turns the sent name into a handle a person would choose, and applies **only** when a batch is run with `humanize=True`, which only `SyncPusher` (community runs) sets. The direct `/sync/*` API keeps the sender's names exactly as before. + +- **Extract the person.** The stamp is removed, the token after the last `by` or `via` wins over the site before it, and a domain-shaped name (`costa_jacobpalm_dk`) reduces to its second-level label (`jacobpalm`, a fixed top-level-label list in the spec). Lower case, at most 18 characters of base. +- **Then a pattern, weighted from the one empirical study found** (arXiv 1101.5578, 2011, Latin names only: about 70% of real usernames contain part of the real name, about 30% are plain concatenations, over 18% add digits such as a birth year): plain 40, name plus number 30 (a 1972-2006 year, a two-digit year or a number up to 99), separator plus number 8, funny 17 (a prefix such as `its`/`dr`, a suffix such as `_dev`/`_ftw`, or `42`/`1337`/`404`), light leetspeak 5. The weights are constants in the spec, not tuned; the study gave no figure for separators or the funny forms, so those weights are ours. +- **Deterministic.** The RNG is seeded with `sha256(origin:origin_uid)`, so a record always gets the same candidates; a taken name falls through to the next candidate and finally to the old `-` and numeric suffixes. +- **Existing users are never renamed.** The name is chosen on insert only; under `humanize` an update keeps the stored username. Users created before this change keep their names. Alias clones (a name that is an identity in `sync_identities`) are never decorated, because the rewritten mentions must match them. + ## Gaps -- **No export endpoint.** Sync is inbound only. The scheduled sender is `CommunityService` (`services/community/`), which keeps a per-community watermark and pushes users, posts, comments in that order. -- **Votes, reactions, follows, attachments, polls and project links are not synced.** Counters will not match the origin. -- **A comment's post and parent must be sent in an earlier batch or earlier in the same batch.** There is no deferred resolution queue; the result reason tells the sender what to send first. -- **Privacy disclosure.** Synced profile data is disclosed in `/docs/privacy.html` (data table row "Imported community content"). -- A user's own deletion on this side is final, but nothing tells the origin; the next sync of that user is `rejected` / `purged_locally` forever. +- Votes, reactions and poll votes are import only: there is no export of who voted, so a two-way mirror of engagement is not possible. +- A user's own deletion on this side is final and the origin learns of it only through `/sync/deletions`; a sender that ignores that endpoint sees `rejected` / `purged_locally` forever. +- Privacy disclosure: synced profile data, engagement and attachments are disclosed in `/docs/privacy.html` (data table row "Imported community content"). ## Selection, identity aliasing and translation (community runs only) diff --git a/devplacepy/services/sync/batch.py b/devplacepy/services/sync/batch.py index f6b33817..7e19495d 100644 --- a/devplacepy/services/sync/batch.py +++ b/devplacepy/services/sync/batch.py @@ -5,25 +5,28 @@ from __future__ import annotations from collections import Counter from devplacepy.schemas.sync import SyncBatchOut, SyncRecordResultOut +from devplacepy.services.sync.pending import PendingQueue from devplacepy.services.sync.processors import PROCESSORS from devplacepy.services.sync.records import SyncBatchForm from devplacepy.services.sync.spec import STATUSES class BatchRunner: - def __init__(self, kind: str, *, request=None): + def __init__(self, kind: str, *, request=None, humanize: bool = False): self.kind = kind self.request = request + self.humanize = humanize def run(self, data: SyncBatchForm) -> SyncBatchOut: processor = PROCESSORS[self.kind]( - data.origin, dry_run=data.dry_run, request=self.request + data.origin, dry_run=data.dry_run, request=self.request, humanize=self.humanize ) results = [ SyncRecordResultOut(**vars(processor.process(index, raw))) for index, raw in enumerate(data.records) ] tally = Counter(result.status for result in results) + released = {} if data.dry_run else dict(PendingQueue().release(data.origin)) return SyncBatchOut( origin=data.origin, kind=self.kind, @@ -31,4 +34,5 @@ class BatchRunner: total=len(results), counts={status: tally.get(status, 0) for status in STATUSES}, results=results, + released=released, ) diff --git a/devplacepy/services/sync/deletions.py b/devplacepy/services/sync/deletions.py new file mode 100644 index 00000000..f75a1abc --- /dev/null +++ b/devplacepy/services/sync/deletions.py @@ -0,0 +1,73 @@ +# retoor + +from __future__ import annotations + +from dataclasses import dataclass, field + +from devplacepy.database.sync import acknowledge_links, deletion_rows +from devplacepy.services.sync.spec import KINDS, LIMITS, SYNC_ACTOR + +CURSOR_SEPARATOR = ":" +BY_PURGED, BY_SYSTEM, BY_MEMBER = "purged", "system", "member" + + +class DeletionCursorError(ValueError): + pass + + +@dataclass +class DeletionPage: + items: list[dict] = field(default_factory=list) + next_cursor: str = "" + has_more: bool = False + + +def parse_cursor(raw: str) -> tuple[int, int]: + if not raw: + return 0, 0 + try: + kind_index, after = (int(part) for part in raw.split(CURSOR_SEPARATOR)) + except ValueError: + raise DeletionCursorError("the cursor is not valid") + if not 0 <= kind_index < len(KINDS) or after < 0: + raise DeletionCursorError("the cursor is not valid") + return kind_index, after + + +def classify(row: dict) -> str: + if row["local_uid"] is None: + return BY_PURGED + return BY_SYSTEM if (row.get("deleted_by") or "") in ("system", SYNC_ACTOR) else BY_MEMBER + + +def deletions_page(origin: str, cursor: str = "", limit: int | None = None) -> DeletionPage: + size = max(1, min(int(limit or LIMITS.deletions_page_size), LIMITS.deletions_page_size)) + kind_index, after = parse_cursor(cursor) + page = DeletionPage() + for index in range(kind_index, len(KINDS)): + kind = KINDS[index] + room = size + 1 - len(page.items) + rows = deletion_rows(origin, kind, kind, after if index == kind_index else 0, room) + for row in rows: + page.items.append( + { + "kind": kind, + "origin_uid": row["origin_uid"], + "by": classify(row), + "deleted_at": row.get("deleted_at") or "", + "_cursor": f"{index}{CURSOR_SEPARATOR}{row['link_id']}", + } + ) + if len(page.items) > size: + break + page.has_more = len(page.items) > size + page.items = page.items[:size] + if page.has_more and page.items: + page.next_cursor = page.items[-1]["_cursor"] + for item in page.items: + item.pop("_cursor", None) + return page + + +def acknowledge(origin: str, items: list[dict]) -> int: + return acknowledge_links(origin, [(item["kind"], item["origin_uid"]) for item in items]) diff --git a/devplacepy/services/sync/exporter.py b/devplacepy/services/sync/exporter.py new file mode 100644 index 00000000..7edd6b33 --- /dev/null +++ b/devplacepy/services/sync/exporter.py @@ -0,0 +1,287 @@ +# retoor + +from __future__ import annotations + +import base64 +import json +from dataclasses import dataclass, field + +from devplacepy.database import db +from devplacepy.database.core import _in_clause +from devplacepy.database.sync import EXPORT_CHANGED +from devplacepy.services.sync.normalize import instant_iso +from devplacepy.services.sync.spec import ( + EXPORT_KINDS, + KIND_ATTACHMENTS, + KIND_COMMENTS, + KIND_FOLLOWS, + KIND_POLLS, + KIND_POSTS, + KIND_PROJECTS, + KIND_USERS, + LIMITS, +) + +ATTACHMENT_URL = "/static/uploads/attachments/{directory}/{stored_name}" + + +class ExportError(ValueError): + pass + + +@dataclass +class ExportPage: + kind: str + records: list[dict] = field(default_factory=list) + next_cursor: str = "" + has_more: bool = False + + +def encode_cursor(changed: str, uid: str) -> str: + return base64.urlsafe_b64encode(json.dumps([changed, uid]).encode()).decode() + + +def decode_cursor(raw: str) -> tuple[str, str]: + if not raw: + return "", "" + try: + changed, uid = json.loads(base64.urlsafe_b64decode(raw.encode())) + except (ValueError, TypeError): + raise ExportError("the cursor is not valid") + if not isinstance(changed, str) or not isinstance(uid, str): + raise ExportError("the cursor is not valid") + return changed, uid + + +def iso_or_empty(value) -> str: + return instant_iso(value) if value else "" + + +class Exporter: + kind = "" + table = "" + extra_where = "" + + def changed(self) -> str: + return EXPORT_CHANGED[self.table] + + def prepare(self, rows: list[dict]) -> dict: + return {} + + def record(self, row: dict, context: dict) -> dict: + raise NotImplementedError + + def deleted(self, row: dict) -> bool: + return bool(row.get("deleted_at")) + + def page(self, since: str, cursor: str, limit: int) -> ExportPage: + if self.table not in db.tables: + return ExportPage(self.kind) + changed = self.changed() + after_changed, after_uid = decode_cursor(cursor) + clauses = [ + "NOT EXISTS (SELECT 1 FROM sync_links l WHERE l.kind = :kind AND l.local_uid = t.uid)", + ] + params: dict = {"kind": self.kind, "n": limit + 1} + if self.extra_where: + clauses.append(self.extra_where) + if since: + clauses.append(f"{changed} > :since") + params["since"] = since + if cursor: + clauses.append(f"({changed} > :after_changed OR ({changed} = :after_changed AND uid > :after_uid))") + params["after_changed"] = after_changed + params["after_uid"] = after_uid + rows = [ + dict(row) + for row in db.query( + f"SELECT *, {changed} AS _changed FROM {self.table} t WHERE {' AND '.join(clauses)} " + f"ORDER BY {changed}, uid LIMIT :n", + **params, + ) + ] + has_more = len(rows) > limit + rows = rows[:limit] + context = self.prepare(rows) + records = [self.record(row, context) for row in rows] + next_cursor = encode_cursor(str(rows[-1]["_changed"]), rows[-1]["uid"]) if rows and has_more else "" + return ExportPage(self.kind, records, next_cursor, has_more) + + +class UserExporter(Exporter): + kind = KIND_USERS + table = "users" + + def deleted(self, row: dict) -> bool: + return not (row.get("is_active") is None or row.get("is_active")) or bool(row.get("deletion_requested_at")) + + def record(self, row: dict, context: dict) -> dict: + return { + "origin_uid": row["uid"], + "username": row["username"], + "bio": row.get("bio") or "", + "location": row.get("location") or "", + "git_link": row.get("git_link") or "", + "website": row.get("website") or "", + "avatar_seed": row.get("avatar_seed"), + "created_at": iso_or_empty(row.get("created_at")), + "deleted": self.deleted(row), + } + + +class ProjectExporter(Exporter): + kind = KIND_PROJECTS + table = "projects" + extra_where = "COALESCE(is_private, 0) = 0" + + def record(self, row: dict, context: dict) -> dict: + return { + "origin_uid": row["uid"], + "author_uid": row["user_uid"], + "title": row.get("title") or "", + "description": row.get("description") or "", + "project_type": row.get("project_type") or "software", + "status": row.get("status") or "", + "platforms": row.get("platforms") or "", + "website_url": row.get("website_url") or "", + "repo_url": row.get("repo_url") or "", + "created_at": iso_or_empty(row.get("created_at")), + "updated_at": iso_or_empty(row.get("updated_at")), + "deleted": self.deleted(row), + } + + +class PostExporter(Exporter): + kind = KIND_POSTS + table = "posts" + + def prepare(self, rows: list[dict]) -> dict: + project_uids = [row["project_uid"] for row in rows if row.get("project_uid")] + public: set[str] = set() + if project_uids and "projects" in db.tables: + placeholders, params = _in_clause(project_uids) + found = db.query( + f"SELECT uid FROM projects WHERE uid IN ({placeholders}) AND COALESCE(is_private, 0) = 0", + **params, + ) + public = {row["uid"] for row in found} + return {"public_projects": public} + + def record(self, row: dict, context: dict) -> dict: + project = row.get("project_uid") + return { + "origin_uid": row["uid"], + "author_uid": row["user_uid"], + "title": row.get("title") or "", + "content": row.get("content") or "", + "topic": row.get("topic") or "random", + "project_uid": project if project in context["public_projects"] else None, + "created_at": iso_or_empty(row.get("created_at")), + "updated_at": iso_or_empty(row.get("updated_at")), + "deleted": self.deleted(row), + } + + +class CommentExporter(Exporter): + kind = KIND_COMMENTS + table = "comments" + extra_where = "target_type = 'post'" + + def record(self, row: dict, context: dict) -> dict: + return { + "origin_uid": row["uid"], + "author_uid": row["user_uid"], + "post_uid": row["target_uid"], + "parent_uid": row.get("parent_uid"), + "content": row.get("content") or "", + "created_at": iso_or_empty(row.get("created_at")), + "updated_at": iso_or_empty(row.get("updated_at")), + "deleted": self.deleted(row), + } + + +class PollExporter(Exporter): + kind = KIND_POLLS + table = "polls" + + def prepare(self, rows: list[dict]) -> dict: + if not rows or "poll_options" not in db.tables: + return {"options": {}} + placeholders, params = _in_clause([row["uid"] for row in rows]) + options: dict[str, list[dict]] = {} + for option in db.query( + f"SELECT uid, poll_uid, label, position FROM poll_options WHERE poll_uid IN ({placeholders}) " + "AND deleted_at IS NULL ORDER BY position, id", + **params, + ): + options.setdefault(option["poll_uid"], []).append( + {"origin_uid": option["uid"], "label": option["label"]} + ) + return {"options": options} + + def record(self, row: dict, context: dict) -> dict: + return { + "origin_uid": row["uid"], + "author_uid": row["user_uid"], + "post_uid": row["post_uid"], + "question": row.get("question") or "", + "options": context["options"].get(row["uid"], []), + "created_at": iso_or_empty(row.get("created_at")), + "deleted": self.deleted(row), + } + + +class FollowExporter(Exporter): + kind = KIND_FOLLOWS + table = "follows" + + def record(self, row: dict, context: dict) -> dict: + return { + "origin_uid": row["uid"], + "follower_uid": row["follower_uid"], + "following_uid": row["following_uid"], + "created_at": iso_or_empty(row.get("created_at")), + "deleted": self.deleted(row), + } + + +class AttachmentExporter(Exporter): + kind = KIND_ATTACHMENTS + table = "attachments" + extra_where = "target_type IN ('post', 'comment', 'project') AND COALESCE(target_uid, '') != ''" + + def record(self, row: dict, context: dict) -> dict: + return { + "origin_uid": row["uid"], + "author_uid": row["user_uid"], + "target_kind": row["target_type"], + "target_uid": row["target_uid"], + "filename": row.get("original_filename") or "", + "mime_type": row.get("mime_type") or "", + "file_size": int(row.get("file_size") or 0), + "url": ATTACHMENT_URL.format(directory=row.get("directory") or "", stored_name=row.get("stored_name") or ""), + "created_at": iso_or_empty(row.get("created_at")), + "deleted": self.deleted(row), + } + + +EXPORTERS: dict[str, Exporter] = { + item.kind: item + for item in ( + UserExporter(), + ProjectExporter(), + PostExporter(), + CommentExporter(), + PollExporter(), + FollowExporter(), + AttachmentExporter(), + ) +} + + +def export_page(kind: str, since: str = "", cursor: str = "", limit: int | None = None) -> ExportPage: + if kind not in EXPORT_KINDS or kind not in EXPORTERS: + raise ExportError(f"kind {kind!r} cannot be exported") + size = max(1, min(int(limit or LIMITS.export_page_size), LIMITS.export_page_size)) + return EXPORTERS[kind].page(since, cursor, size) + diff --git a/devplacepy/services/sync/handles.py b/devplacepy/services/sync/handles.py new file mode 100644 index 00000000..10dfdc36 --- /dev/null +++ b/devplacepy/services/sync/handles.py @@ -0,0 +1,109 @@ +# retoor + +from __future__ import annotations + +import hashlib +import random +import re +from collections.abc import Iterator + +from devplacepy.services.sync.normalize import username_candidates +from devplacepy.services.sync.spec import HANDLES + +TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}[0-9_:\-]*") +TOKEN_SPLIT = re.compile(r"[_\s]+") +HANDLE_STRIP = re.compile(r"[^a-z0-9_-]+") +HANDLE_PATTERN = re.compile(r"^[a-z0-9_-]+$") +PATTERNS = ("plain", "digits", "separated", "funny", "leet") +PATTERN_WEIGHTS = ( + HANDLES.plain_weight, + HANDLES.digits_weight, + HANDLES.separated_weight, + HANDLES.funny_weight, + HANDLES.leet_weight, +) +NUMBER_KINDS = ("year", "short", "two_digit") +NUMBER_WEIGHTS = (HANDLES.year_weight, HANDLES.short_number_weight, HANDLES.two_digit_weight) + + +class HandleBuilder: + def __init__(self, origin: str, origin_uid: str): + self.origin = origin + seed = hashlib.sha256(f"{origin}:{origin_uid}".encode()).digest() + self.rng = random.Random(seed) + + @staticmethod + def person(raw: str) -> str: + tokens = [token for token in TOKEN_SPLIT.split(TIMESTAMP.sub(" ", str(raw or ""))) if token] + marks = [index for index, token in enumerate(tokens) if token.lower() in HANDLES.relation_words] + for index in reversed(marks): + if index + 1 < len(tokens): + tokens = tokens[index + 1 : index + 2] + break + else: + tokens = [token for index, token in enumerate(tokens) if index not in marks] or tokens + if len(tokens) > 1 and tokens[-1].lower() in HANDLES.top_level_labels: + labels = [token for token in tokens[:-1] if token.lower() not in HANDLES.ignored_labels] + tokens = labels[-1:] or tokens[:-1] + base = HANDLE_STRIP.sub("", "_".join(tokens).lower()).strip("_-") + return re.sub(r"[_-]{2,}", "_", base) + + def base(self, raw: str) -> str: + base = self.person(raw)[: HANDLES.max_base_length].rstrip("_-") + if len(base) < HANDLES.min_length: + base = f"{base}{self.number()}" + return base + + def number(self) -> str: + kind = self.rng.choices(NUMBER_KINDS, NUMBER_WEIGHTS)[0] + if kind == "year": + return str(self.rng.randint(HANDLES.year_first, HANDLES.year_last)) + if kind == "two_digit": + return f"{self.rng.randint(HANDLES.year_first, HANDLES.year_last) % 100:02d}" + return str(self.rng.randint(1, HANDLES.short_number_max)) + + def leet(self, base: str) -> str | None: + options = [(plain, swap) for plain, swap in HANDLES.leet_map if plain in base] + if not options: + return None + plain, swap = self.rng.choice(options) + return base.replace(plain, swap, self.rng.choice((1, 2))) + + def fit(self, head: str, stem: str, tail: str) -> str: + room = HANDLES.max_length - len(head) - len(tail) + return f"{head}{stem[: max(room, 1)].rstrip('_-')}{tail}" + + def render(self, base: str, pattern: str) -> str: + if pattern == "digits": + return self.fit("", base, self.number()) + if pattern == "separated": + return self.fit("", base, f"{self.rng.choice('_-')}{self.number()}") + if pattern == "funny": + style = self.rng.choice(("prefix", "suffix", "number", "separated")) + if style == "prefix": + return self.fit(self.rng.choice(HANDLES.prefixes), base, "") + if style == "suffix": + return self.fit("", base, self.rng.choice(HANDLES.suffixes)) + number = self.rng.choice(HANDLES.funny_numbers) + return self.fit("", base, number if style == "number" else f"_{number}") + if pattern == "leet": + swapped = self.leet(base) + return self.fit("", swapped, "") if swapped else self.render(base, "digits") + return base + + def candidates(self, raw: str) -> Iterator[str]: + base = self.base(raw) + seen: set[str] = set() + picks = [self.rng.choices(PATTERNS, PATTERN_WEIGHTS)[0]] + picks += [self.rng.choice(PATTERNS[1:]) for _ in range(HANDLES.rerolls)] + for pattern in picks: + candidate = self.render(base, pattern) + if candidate in seen or not self.valid(candidate): + continue + seen.add(candidate) + yield candidate + yield from username_candidates(base if self.valid(base) else f"{base}{self.number()}", self.origin) + + @staticmethod + def valid(candidate: str) -> bool: + return HANDLES.min_length <= len(candidate) <= HANDLES.max_length and bool(HANDLE_PATTERN.match(candidate)) diff --git a/devplacepy/services/sync/healing.py b/devplacepy/services/sync/healing.py new file mode 100644 index 00000000..842c3288 --- /dev/null +++ b/devplacepy/services/sync/healing.py @@ -0,0 +1,68 @@ +# retoor + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field + +from devplacepy.database import get_table +from devplacepy.database.sync import ( + get_link, + identity_map, + identity_name_taken, + linked_rows_containing, + rename_identity, +) +from devplacepy.services.sync.rewriter import MentionRewriter +from devplacepy.services.sync.spec import ALIAS, KIND_COMMENTS, KIND_POSTS, KIND_USERS + +logger = logging.getLogger("sync.alias") + +REWRITE_TARGETS = ((KIND_POSTS, "posts", ("title", "content")), (KIND_COMMENTS, "comments", ("content",))) + + +@dataclass +class HealReport: + renamed: dict[str, str] = field(default_factory=dict) + conflicts: list[str] = field(default_factory=list) + rewritten: int = 0 + + +class AliasHealer: + def __init__(self, origin: str): + self.origin = origin + + def heal(self) -> HealReport: + report = HealReport() + for source_key, row in identity_map(self.origin).items(): + link = get_link(self.origin, KIND_USERS, source_key) + if link is None: + continue + clone = get_table("users").find_one(uid=link["local_uid"]) + if clone is None or not clone.get("username") or clone["username"] == row["our_name"]: + continue + current = clone["username"] + if current.lower() != row["our_key"] and identity_name_taken(current.lower()): + report.conflicts.append(source_key) + logger.warning("alias %s of %s cannot follow the clone to %s: the name is another identity", row["our_name"], self.origin, current) + continue + if not rename_identity(row["uid"], current): + report.conflicts.append(source_key) + continue + report.renamed[row["our_name"]] = current + logger.info("alias of %s followed its clone from %s to %s", self.origin, row["our_name"], current) + for old, new in report.renamed.items(): + report.rewritten += self.rewrite_mentions(old, new) + return report + + def rewrite_mentions(self, old: str, new: str) -> int: + rewriter = MentionRewriter({old: new}) + changed = 0 + for kind, table, columns in REWRITE_TARGETS: + for row in linked_rows_containing(self.origin, kind, table, columns, old, ALIAS.heal_rewrites_per_run): + values = {column: rewriter.rewrite(row[column] or "").text for column in columns if row.get(column)} + values = {column: text for column, text in values.items() if text != row[column]} + if values: + get_table(table).update({"uid": row["uid"], **values}, ["uid"]) + changed += 1 + return changed diff --git a/devplacepy/services/sync/options.py b/devplacepy/services/sync/options.py index 1a4a07ee..909a3104 100644 --- a/devplacepy/services/sync/options.py +++ b/devplacepy/services/sync/options.py @@ -25,6 +25,7 @@ class SyncOptions: min_comments: int = 0 topics: tuple[str, ...] = () jitter_percent: int = 0 + collect_attachments: bool = False @classmethod def from_row(cls, row: dict | None) -> "SyncOptions": @@ -53,7 +54,10 @@ class SyncOptions: jitter = int(values.get("jitter_percent") or 0) if not 0 <= jitter <= ALIAS.max_jitter_percent: raise ValueError(f"jitter_percent must be between 0 and {ALIAS.max_jitter_percent}") - return cls(bool(values.get("alias")), translate_to, sample, minimum, tuple(dict.fromkeys(topics)), jitter) + return cls( + bool(values.get("alias")), translate_to, sample, minimum, tuple(dict.fromkeys(topics)), jitter, + bool(values.get("collect_attachments")), + ) def merged(self, **changes) -> "SyncOptions": values = self.to_dict() | {key: value for key, value in changes.items() if value is not None} diff --git a/devplacepy/services/sync/pending.py b/devplacepy/services/sync/pending.py new file mode 100644 index 00000000..a7248d53 --- /dev/null +++ b/devplacepy/services/sync/pending.py @@ -0,0 +1,51 @@ +# retoor + +from __future__ import annotations + +import json +import logging +from collections import Counter + +from devplacepy.database.sync import ( + delete_pending, + expire_pending, + pending_counts, + ready_pending, +) +from devplacepy.services.sync.processors import PROCESSORS +from devplacepy.services.sync.spec import LIMITS, STATUS_DEFERRED + +logger = logging.getLogger(__name__) + +PAGE = 50 + + +class PendingQueue: + def release(self, origin: str = "", cap: int | None = None) -> Counter: + budget = cap if cap is not None else LIMITS.pending_release_cap + released: Counter = Counter() + spent = 0 + while spent < budget: + expire_pending(LIMITS.pending_max_attempts) + rows = ready_pending(min(PAGE, budget - spent), origin) + if not rows: + break + for row in rows: + delete_pending(row["uid"]) + spent += 1 + try: + raw = json.loads(row["record"]) + except ValueError: + logger.warning("pending record %s is not valid JSON, dropped", row["uid"]) + continue + processor = PROCESSORS[row["kind"]](row["origin"], attempts=int(row.get("attempts") or 0) + 1) + status = processor.process(0, raw).status + if status != STATUS_DEFERRED: + released[status] += 1 + return released + + def counts(self) -> dict[str, int]: + return pending_counts() + + def sweep(self) -> int: + return expire_pending(LIMITS.pending_max_attempts) diff --git a/devplacepy/services/sync/processors.py b/devplacepy/services/sync/processors.py index ad649d63..de242e17 100644 --- a/devplacepy/services/sync/processors.py +++ b/devplacepy/services/sync/processors.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import logging import secrets import time @@ -14,49 +15,80 @@ from devplacepy.database import get_table, refresh_snapshot from devplacepy.database.content import invalidate_user_recent_cache from devplacepy.database.pagination import clear_user_post_count from devplacepy.database.soft_delete import restore, soft_delete +from devplacepy.database.ranking import clear_user_stars, get_target_owner_uid from devplacepy.database.sync import ( advance_link, claim_link, get_link, + park_pending, + recompute_stars, release_link, username_taken, ) +from devplacepy.database.sync import identity_name_taken from devplacepy.services.moderation.screening import ( ContentRefused, record as record_screening, refuse_if_blocked, screen_fields, ) +from devplacepy.services.sync.handles import HandleBuilder from devplacepy.services.sync.normalize import fingerprint, username_candidates from devplacepy.services.sync.records import ( + AttachmentRecord, CommentRecord, + FollowRecord, + PollRecord, + PollVoteRecord, PostRecord, + ProjectRecord, + ReactionRecord, SyncRecord, UserRecord, + VoteRecord, ) from devplacepy.services.sync.spec import ( + DEFERRABLE_REASONS, + KIND_ATTACHMENTS, KIND_COMMENTS, + KIND_FOLLOWS, + KIND_POLL_OPTIONS, + KIND_POLL_VOTES, + KIND_POLLS, KIND_POSTS, + KIND_PROJECTS, + KIND_REACTIONS, KIND_USERS, + KIND_VOTES, LIMITS, + REASON_ACK_DELETED, REASON_CONTENT_REFUSED, + REASON_IMMUTABLE, REASON_INTERNAL, REASON_INVALID, REASON_LOCALLY_DELETED, REASON_OLDER_THAN_SYNCED, REASON_PARENT_MISMATCH, REASON_PURGED_LOCALLY, + REASON_QUEUE_FULL, + REASON_SELF_FOLLOW, REASON_TOMBSTONE_UNKNOWN, REASON_UNKNOWN_AUTHOR, + REASON_UNKNOWN_OPTION, REASON_UNKNOWN_PARENT, + REASON_UNKNOWN_POLL, REASON_UNKNOWN_POST, + REASON_UNKNOWN_PROJECT, + REASON_UNKNOWN_TARGET, STATUS_CREATED, + STATUS_DEFERRED, STATUS_FAILED, STATUS_REJECTED, STATUS_STALE, STATUS_UNCHANGED, STATUS_UPDATED, SYNC_ACTOR, + TARGET_KINDS, ) from devplacepy.utils.accounts import account_row from devplacepy.utils.authcache import clear_user_cache @@ -81,9 +113,10 @@ def unusable_password_hash() -> str: class Refusal(Exception): - def __init__(self, reason: str, message: str): + def __init__(self, reason: str, message: str, needs: tuple[str, str] | None = None): self.reason = reason self.message = message + self.needs = needs super().__init__(message) @@ -96,6 +129,8 @@ class SyncResult: local_uid: str = "" errors: list[str] = field(default_factory=list) ignored: list[str] = field(default_factory=list) + needs_kind: str = "" + needs_uid: str = "" class RecordProcessor: @@ -103,10 +138,12 @@ class RecordProcessor: table: str = "" model: type[SyncRecord] = SyncRecord - def __init__(self, origin: str, *, dry_run: bool = False, request=None): + def __init__(self, origin: str, *, dry_run: bool = False, request=None, attempts: int = 0, humanize: bool = False): self.origin = origin + self.humanize = humanize self.dry_run = dry_run self.request = request + self.attempts = attempts def process(self, index: int, raw: dict) -> SyncResult: origin_uid = str(raw.get("origin_uid", "")) if isinstance(raw, dict) else "" @@ -125,13 +162,7 @@ class RecordProcessor: try: result = self.apply(index, record) except Refusal as refusal: - result = SyncResult( - index, - record.origin_uid, - STATUS_REJECTED, - refusal.reason, - errors=[refusal.message], - ) + result = self.refused(index, record, raw, refusal) except ContentRefused as refused: result = SyncResult( index, @@ -150,6 +181,44 @@ class RecordProcessor: result.ignored = ignored return result + def refused(self, index: int, record: SyncRecord, raw: dict, refusal: Refusal) -> SyncResult: + needs_kind, needs_uid = refusal.needs or ("", "") + result = SyncResult( + index, + record.origin_uid, + STATUS_REJECTED, + refusal.reason, + errors=[refusal.message], + needs_kind=needs_kind, + needs_uid=needs_uid, + ) + if self.dry_run or refusal.reason not in DEFERRABLE_REASONS or not refusal.needs: + return result + return self.defer(result, raw, refusal) + + def defer(self, result: SyncResult, raw: dict, refusal: Refusal) -> SyncResult: + encoded = json.dumps(raw, default=str) + if len(encoded) > LIMITS.pending_max_record_bytes: + result.errors.append("record is too large to wait for its dependency") + return result + parked = park_pending( + self.origin, + self.kind, + result.origin_uid, + result.needs_kind, + result.needs_uid, + encoded, + attempts=self.attempts, + ttl_days=LIMITS.pending_ttl_days, + cap=LIMITS.pending_max_per_origin, + ) + if not parked: + result.reason = refusal.reason + result.errors.append(f"{REASON_QUEUE_FULL}: the waiting list for this origin is full") + return result + result.status = STATUS_DEFERRED + return result + def ignored_fields(self, raw) -> list[str]: if not isinstance(raw, dict): return [] @@ -164,16 +233,16 @@ class RecordProcessor: return messages def apply(self, index: int, record: SyncRecord) -> SyncResult: + link = get_link(self.origin, self.kind, record.origin_uid) + if link is None and record.deleted: + return SyncResult( + index, record.origin_uid, STATUS_UNCHANGED, REASON_TOMBSTONE_UNKNOWN + ) refs = self.resolve(record) if not record.deleted: self.screen(record) digest = fingerprint(record.content_fields()) - link = get_link(self.origin, self.kind, record.origin_uid) if link is None: - if record.deleted: - return SyncResult( - index, record.origin_uid, STATUS_UNCHANGED, REASON_TOMBSTONE_UNKNOWN - ) return self.create(index, record, refs, digest) return self.update(index, record, refs, digest, link) @@ -231,6 +300,10 @@ class RecordProcessor: ) -> SyncResult: local_uid = link["local_uid"] row = self.find_local(link) + if row is None and link.get("ack_at"): + return SyncResult( + index, record.origin_uid, STATUS_UNCHANGED, REASON_ACK_DELETED, local_uid=local_uid + ) if row is None: return SyncResult( index, @@ -253,7 +326,7 @@ class RecordProcessor: index, record.origin_uid, STATUS_UNCHANGED, - REASON_LOCALLY_DELETED, + REASON_ACK_DELETED if link.get("ack_at") else REASON_LOCALLY_DELETED, local_uid=local_uid, ) if digest == link.get("fingerprint"): @@ -287,12 +360,27 @@ class RecordProcessor: def after_write(self, local_uid: str, record: SyncRecord, refs: dict) -> None: return None - def author_uid(self, record: SyncRecord) -> str: - link = get_link(self.origin, KIND_USERS, record.author_uid) + def user_ref(self, origin_uid: str, role: str = "author") -> str: + link = get_link(self.origin, KIND_USERS, origin_uid) if link is None: raise Refusal( REASON_UNKNOWN_AUTHOR, - f"author {record.author_uid} is not synced yet, send users first", + f"{role} {origin_uid} is not synced yet, send users first", + (KIND_USERS, origin_uid), + ) + return link["local_uid"] + + def author_uid(self, record: SyncRecord) -> str: + return self.user_ref(record.author_uid) + + def target_ref(self, target_kind: str, target_uid: str) -> str: + kind = TARGET_KINDS[target_kind] + link = get_link(self.origin, kind, target_uid) + if link is None: + raise Refusal( + REASON_UNKNOWN_TARGET, + f"{target_kind} {target_uid} is not synced yet, send {kind} first", + (kind, target_uid), ) return link["local_uid"] @@ -334,11 +422,19 @@ class UserProcessor(RecordProcessor): ) def free_username(self, record: UserRecord, current: str = "") -> str: - for candidate in username_candidates(record.username, self.origin): + for candidate in self.username_options(record, current): if candidate == current or not username_taken(candidate): return candidate return current or record.username + def username_options(self, record: UserRecord, current: str): + if self.humanize and current: + yield current + elif self.humanize and not identity_name_taken(record.username.lower()): + yield from HandleBuilder(self.origin, record.origin_uid).candidates(record.username) + else: + yield from username_candidates(record.username, self.origin) + def insert(self, local_uid: str, record: UserRecord, refs: dict) -> None: now = datetime.now(timezone.utc).isoformat() row = account_row( @@ -386,7 +482,17 @@ class PostProcessor(RecordProcessor): model = PostRecord def resolve(self, record: PostRecord) -> dict: - return {"user_uid": self.author_uid(record)} + refs = {"user_uid": self.author_uid(record), "project_uid": None} + if record.project_uid: + link = get_link(self.origin, KIND_PROJECTS, record.project_uid) + if link is None: + raise Refusal( + REASON_UNKNOWN_PROJECT, + f"project {record.project_uid} is not synced yet, send projects first", + (KIND_PROJECTS, record.project_uid), + ) + refs["project_uid"] = link["local_uid"] + return refs def screen(self, record: PostRecord) -> None: refuse_if_blocked( @@ -404,7 +510,7 @@ class PostProcessor(RecordProcessor): "title": record.title or None, "content": record.content, "topic": record.topic, - "project_uid": None, + "project_uid": refs["project_uid"], "image": None, "stars": 0, "created_at": record.created_at, @@ -421,6 +527,7 @@ class PostProcessor(RecordProcessor): "title": record.title or None, "content": record.content, "topic": record.topic, + "project_uid": refs["project_uid"], "updated_at": record.updated_at, }, ["uid"], @@ -451,6 +558,7 @@ class CommentProcessor(RecordProcessor): raise Refusal( REASON_UNKNOWN_POST, f"post {record.post_uid} is not synced yet, send posts first", + (KIND_POSTS, record.post_uid), ) post_uid = post_link["local_uid"] parent_uid = None @@ -461,6 +569,7 @@ class CommentProcessor(RecordProcessor): REASON_UNKNOWN_PARENT, f"parent comment {record.parent_uid} is not synced yet, " "send parents before replies", + (KIND_COMMENTS, record.parent_uid), ) parent = get_table("comments").find_one(uid=parent_link["local_uid"]) if parent is None or parent.get("target_uid") != post_uid: @@ -509,8 +618,400 @@ class CommentProcessor(RecordProcessor): ) + + +class ProjectProcessor(RecordProcessor): + kind = KIND_PROJECTS + table = "projects" + model = ProjectRecord + + def resolve(self, record: ProjectRecord) -> dict: + return {"user_uid": self.author_uid(record)} + + def screen(self, record: ProjectRecord) -> None: + refuse_if_blocked( + screen_fields("projects", {"title": record.title, "description": record.description}) + ) + + def insert(self, local_uid: str, record: ProjectRecord, refs: dict) -> None: + get_table("projects").insert( + { + "uid": local_uid, + "user_uid": refs["user_uid"], + "slug": make_combined_slug(record.title, local_uid), + "title": record.title, + "description": record.description, + "project_type": record.project_type, + "status": record.status, + "platforms": record.platforms, + "website_url": record.website_url, + "repo_url": record.repo_url, + "release_date": "", + "demo_date": "", + "cover_attachment_uid": "", + "logo_attachment_uid": "", + "is_private": 0, + "read_only": 0, + "stars": 0, + "created_at": record.created_at, + "updated_at": record.updated_at, + "deleted_at": None, + "deleted_by": None, + } + ) + + def write(self, row: dict, record: ProjectRecord, refs: dict) -> None: + get_table("projects").update( + { + "uid": row["uid"], + "title": record.title, + "description": record.description, + "project_type": record.project_type, + "status": record.status, + "platforms": record.platforms, + "website_url": record.website_url, + "repo_url": record.repo_url, + "updated_at": record.updated_at, + }, + ["uid"], + ) + self.apply_tombstone(row, record) + + def after_write(self, local_uid: str, record: ProjectRecord, refs: dict) -> None: + from devplacepy.templating import clear_user_projects_cache + + clear_user_projects_cache(refs["user_uid"]) + invalidate_user_recent_cache("projects", refs["user_uid"]) + if not record.deleted: + self.screen_and_report( + local_uid, + "project", + {"title": record.title, "description": record.description}, + refs["user_uid"], + ) + + +class NaturalKeyProcessor(RecordProcessor): + def natural_key(self, record: SyncRecord, refs: dict) -> dict: + raise NotImplementedError + + def create(self, index: int, record: SyncRecord, refs: dict, digest: str) -> SyncResult: + existing = get_table(self.table).find_one(**self.natural_key(record, refs)) + if existing is not None and existing.get("deleted_at") and existing.get("deleted_by") != SYNC_ACTOR: + return SyncResult( + index, record.origin_uid, STATUS_UNCHANGED, REASON_LOCALLY_DELETED, local_uid=existing["uid"] + ) + local_uid = existing["uid"] if existing is not None else generate_uid() + if self.dry_run: + return SyncResult(index, record.origin_uid, STATUS_CREATED, local_uid="") + self.prepare() + claimed = claim_link( + self.origin, self.kind, record.origin_uid, local_uid, digest, record.updated_at + ) + if not claimed: + link = get_link(self.origin, self.kind, record.origin_uid) + return self.update(index, record, refs, digest, link) + try: + if existing is not None: + self.write(existing, record, refs) + else: + self.insert(local_uid, record, refs) + except Exception: + release_link(self.origin, self.kind, record.origin_uid, local_uid) + raise + self.after_write(local_uid, record, refs) + return SyncResult(index, record.origin_uid, STATUS_CREATED, local_uid=local_uid) + + +class VoteProcessor(NaturalKeyProcessor): + kind = KIND_VOTES + table = "votes" + model = VoteRecord + + def resolve(self, record: VoteRecord) -> dict: + return { + "user_uid": self.user_ref(record.voter_uid, "voter"), + "target_uid": self.target_ref(record.target_kind, record.target_uid), + } + + def natural_key(self, record: VoteRecord, refs: dict) -> dict: + return {"user_uid": refs["user_uid"], "target_uid": refs["target_uid"], "target_type": record.target_kind} + + def insert(self, local_uid: str, record: VoteRecord, refs: dict) -> None: + get_table("votes").insert( + { + "uid": local_uid, + "user_uid": refs["user_uid"], + "target_uid": refs["target_uid"], + "target_type": record.target_kind, + "value": record.value, + "created_at": record.created_at, + "deleted_at": None, + "deleted_by": None, + } + ) + + def write(self, row: dict, record: VoteRecord, refs: dict) -> None: + get_table("votes").update({"uid": row["uid"], "value": record.value}, ["uid"]) + self.apply_tombstone(row, record) + + def after_write(self, local_uid: str, record: VoteRecord, refs: dict) -> None: + recompute_stars(record.target_kind, refs["target_uid"]) + owner = get_target_owner_uid(record.target_kind, refs["target_uid"]) + if owner: + clear_user_stars(owner) + + +class ReactionProcessor(NaturalKeyProcessor): + kind = KIND_REACTIONS + table = "reactions" + model = ReactionRecord + + def resolve(self, record: ReactionRecord) -> dict: + return { + "user_uid": self.author_uid(record), + "target_uid": self.target_ref(record.target_kind, record.target_uid), + } + + def natural_key(self, record: ReactionRecord, refs: dict) -> dict: + return { + "user_uid": refs["user_uid"], + "target_uid": refs["target_uid"], + "target_type": record.target_kind, + "emoji": record.emoji, + } + + def insert(self, local_uid: str, record: ReactionRecord, refs: dict) -> None: + get_table("reactions").insert( + { + "uid": local_uid, + "user_uid": refs["user_uid"], + "target_uid": refs["target_uid"], + "target_type": record.target_kind, + "emoji": record.emoji, + "created_at": record.created_at, + "deleted_at": None, + "deleted_by": None, + } + ) + + def write(self, row: dict, record: ReactionRecord, refs: dict) -> None: + self.apply_tombstone(row, record) + + +class FollowProcessor(NaturalKeyProcessor): + kind = KIND_FOLLOWS + table = "follows" + model = FollowRecord + + def resolve(self, record: FollowRecord) -> dict: + follower = self.user_ref(record.follower_uid, "follower") + following = self.user_ref(record.following_uid, "followed user") + if follower == following: + raise Refusal(REASON_SELF_FOLLOW, "an account cannot follow itself") + return {"follower_uid": follower, "following_uid": following} + + def natural_key(self, record: FollowRecord, refs: dict) -> dict: + return dict(refs) + + def insert(self, local_uid: str, record: FollowRecord, refs: dict) -> None: + get_table("follows").insert( + { + "uid": local_uid, + "follower_uid": refs["follower_uid"], + "following_uid": refs["following_uid"], + "created_at": record.created_at, + "deleted_at": None, + "deleted_by": None, + } + ) + + def write(self, row: dict, record: FollowRecord, refs: dict) -> None: + self.apply_tombstone(row, record) + + +class PollProcessor(RecordProcessor): + kind = KIND_POLLS + table = "polls" + model = PollRecord + + def resolve(self, record: PollRecord) -> dict: + link = get_link(self.origin, KIND_POSTS, record.post_uid) + if link is None: + raise Refusal( + REASON_UNKNOWN_POST, + f"post {record.post_uid} is not synced yet, send posts first", + (KIND_POSTS, record.post_uid), + ) + return {"user_uid": self.author_uid(record), "post_uid": link["local_uid"]} + + def screen(self, record: PollRecord) -> None: + text = "\n".join([record.question, *(item.label for item in record.options)]) + refuse_if_blocked(screen_fields("posts", {"content": text})) + + @staticmethod + def option_key(record: PollRecord, option) -> str: + return f"{record.origin_uid}:{option.origin_uid}" + + def insert(self, local_uid: str, record: PollRecord, refs: dict) -> None: + taken = get_table("polls").find_one(post_uid=refs["post_uid"], deleted_at=None) + if taken is not None: + raise Refusal(REASON_INVALID, "the post already has a poll") + get_table("polls").insert( + { + "uid": local_uid, + "post_uid": refs["post_uid"], + "user_uid": refs["user_uid"], + "question": record.question, + "created_at": record.created_at, + "deleted_at": None, + "deleted_by": None, + } + ) + for position, option in enumerate(record.options): + self.add_option(local_uid, record, option, position) + + def add_option(self, poll_uid: str, record: PollRecord, option, position: int) -> None: + option_uid = generate_uid() + if not claim_link( + self.origin, KIND_POLL_OPTIONS, self.option_key(record, option), option_uid, + fingerprint({"label": option.label}), record.updated_at, + ): + return + get_table("poll_options").insert( + { + "uid": option_uid, + "poll_uid": poll_uid, + "label": option.label, + "position": position, + "deleted_at": None, + "deleted_by": None, + } + ) + + def write(self, row: dict, record: PollRecord, refs: dict) -> None: + get_table("polls").update({"uid": row["uid"], "question": record.question}, ["uid"]) + wanted = set() + options = get_table("poll_options") + for position, option in enumerate(record.options): + key = self.option_key(record, option) + wanted.add(key) + link = get_link(self.origin, KIND_POLL_OPTIONS, key) + if link is None: + self.add_option(row["uid"], record, option, position) + continue + options.update( + {"uid": link["local_uid"], "label": option.label, "position": position, "deleted_at": None, "deleted_by": None}, + ["uid"], + ) + prefix = f"{record.origin_uid}:" + for link in get_table("sync_links").find(origin=self.origin, kind=KIND_POLL_OPTIONS): + if link["origin_uid"].startswith(prefix) and link["origin_uid"] not in wanted: + soft_delete("poll_options", SYNC_ACTOR, uid=link["local_uid"]) + self.apply_tombstone(row, record) + + def apply_tombstone(self, row: dict, record: PollRecord) -> None: + if record.deleted and not row.get("deleted_at"): + stamp = datetime.now(timezone.utc).isoformat() + soft_delete("polls", SYNC_ACTOR, stamp=stamp, uid=row["uid"]) + soft_delete("poll_options", SYNC_ACTOR, stamp=stamp, poll_uid=row["uid"]) + soft_delete("poll_votes", SYNC_ACTOR, stamp=stamp, poll_uid=row["uid"]) + elif not record.deleted and row.get("deleted_at"): + restore("polls", uid=row["uid"]) + restore("poll_options", poll_uid=row["uid"]) + restore("poll_votes", poll_uid=row["uid"]) + + +class PollVoteProcessor(NaturalKeyProcessor): + kind = KIND_POLL_VOTES + table = "poll_votes" + model = PollVoteRecord + + def resolve(self, record: PollVoteRecord) -> dict: + voter = self.user_ref(record.voter_uid, "voter") + poll = get_link(self.origin, KIND_POLLS, record.poll_uid) + if poll is None: + raise Refusal( + REASON_UNKNOWN_POLL, + f"poll {record.poll_uid} is not synced yet, send polls first", + (KIND_POLLS, record.poll_uid), + ) + key = f"{record.poll_uid}:{record.option_uid}" + option = get_link(self.origin, KIND_POLL_OPTIONS, key) + if option is None: + raise Refusal( + REASON_UNKNOWN_OPTION, + f"option {record.option_uid} of poll {record.poll_uid} is not synced", + (KIND_POLLS, record.poll_uid), + ) + found = get_table("poll_options").find_one(uid=option["local_uid"]) + if found is None or found["poll_uid"] != poll["local_uid"]: + raise Refusal(REASON_PARENT_MISMATCH, "the option does not belong to the poll") + return {"user_uid": voter, "poll_uid": poll["local_uid"], "option_uid": option["local_uid"]} + + def natural_key(self, record: PollVoteRecord, refs: dict) -> dict: + return {"poll_uid": refs["poll_uid"], "user_uid": refs["user_uid"]} + + def insert(self, local_uid: str, record: PollVoteRecord, refs: dict) -> None: + get_table("poll_votes").insert( + { + "uid": local_uid, + "poll_uid": refs["poll_uid"], + "option_uid": refs["option_uid"], + "user_uid": refs["user_uid"], + "created_at": record.created_at, + "deleted_at": None, + "deleted_by": None, + } + ) + + def write(self, row: dict, record: PollVoteRecord, refs: dict) -> None: + get_table("poll_votes").update({"uid": row["uid"], "option_uid": refs["option_uid"]}, ["uid"]) + self.apply_tombstone(row, record) + + +class AttachmentProcessor(RecordProcessor): + kind = KIND_ATTACHMENTS + table = "attachments" + model = AttachmentRecord + + def resolve(self, record: AttachmentRecord) -> dict: + return { + "user_uid": self.author_uid(record), + "target_uid": self.target_ref(record.target_kind, record.target_uid), + } + + def screen(self, record: AttachmentRecord) -> None: + from devplacepy.attachments import is_extension_allowed + + extension = "." + record.filename.rsplit(".", 1)[-1].lower() if "." in record.filename else "" + if not is_extension_allowed(extension): + raise Refusal(REASON_INVALID, f"files of type {extension or 'without extension'} are not accepted") + + def insert(self, local_uid: str, record: AttachmentRecord, refs: dict) -> None: + from devplacepy.attachments import link_attachments, store_attachment + + stored = store_attachment(record.content_bytes(), record.filename, refs["user_uid"], uid=local_uid) + if stored is None: + raise Refusal(REASON_INVALID, "the file is not an allowed type or exceeds the upload limit") + link_attachments([local_uid], record.target_kind, refs["target_uid"]) + get_table("attachments").update({"uid": local_uid, "created_at": record.created_at}, ["uid"]) + + def write(self, row: dict, record: AttachmentRecord, refs: dict) -> None: + if record.deleted == bool(row.get("deleted_at")): + raise Refusal(REASON_IMMUTABLE, "an attachment cannot change, send a new origin_uid") + self.apply_tombstone(row, record) + + PROCESSORS: dict[str, type[RecordProcessor]] = { KIND_USERS: UserProcessor, + KIND_PROJECTS: ProjectProcessor, KIND_POSTS: PostProcessor, KIND_COMMENTS: CommentProcessor, + KIND_POLLS: PollProcessor, + KIND_VOTES: VoteProcessor, + KIND_REACTIONS: ReactionProcessor, + KIND_FOLLOWS: FollowProcessor, + KIND_POLL_VOTES: PollVoteProcessor, + KIND_ATTACHMENTS: AttachmentProcessor, } diff --git a/devplacepy/services/sync/records.py b/devplacepy/services/sync/records.py index fb19d2a3..7d0abd5c 100644 --- a/devplacepy/services/sync/records.py +++ b/devplacepy/services/sync/records.py @@ -2,13 +2,17 @@ from __future__ import annotations +import base64 +import binascii +import hashlib import json -from typing import ClassVar, Optional +from typing import ClassVar, Literal, Optional from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from devplacepy.constants import TOPICS from devplacepy.models import normalize_website_url +from devplacepy.rendering import is_single_emoji from devplacepy.services.sync.normalize import ( clean_text, instant_iso, @@ -17,12 +21,34 @@ from devplacepy.services.sync.normalize import ( normalize_username, ) from devplacepy.services.sync.spec import ( + KIND_ATTACHMENTS, KIND_COMMENTS, + KIND_FOLLOWS, + KIND_POLL_VOTES, + KIND_POLLS, KIND_POSTS, + KIND_PROJECTS, + KIND_REACTIONS, KIND_USERS, + KIND_VOTES, + KINDS, LIMITS, ) +REFERENCE_FIELDS = ( + "origin_uid", + "author_uid", + "post_uid", + "voter_uid", + "follower_uid", + "following_uid", + "target_uid", + "poll_uid", + "option_uid", +) +OPTIONAL_REFERENCE_FIELDS = ("parent_uid", "project_uid") +TargetKind = Literal["post", "comment", "project"] + class SyncBatchForm(BaseModel): origin: str @@ -51,9 +77,46 @@ class SyncBatchForm(BaseModel): def not_empty(cls, value): if not value: raise ValueError("records must contain at least one record") + if len(json.dumps(value, default=str)) > LIMITS.max_batch_bytes: + raise ValueError(f"records exceed {LIMITS.max_batch_bytes} bytes") return value +class SyncAckForm(BaseModel): + origin: str + items: list[dict] = Field(max_length=LIMITS.deletions_page_size) + + @field_validator("origin") + @classmethod + def valid_origin(cls, value): + return normalize_origin(value) + + @field_validator("items", mode="before") + @classmethod + def decode_items(cls, value): + if isinstance(value, list) and value and all(isinstance(item, str) for item in value): + value = value[0] if len(value) == 1 else "[" + ",".join(value) + "]" + if isinstance(value, str): + try: + return json.loads(value) + except ValueError: + raise ValueError("items must be a JSON array") + return value + + @field_validator("items") + @classmethod + def valid_items(cls, value): + if not value: + raise ValueError("items must contain at least one entry") + cleaned = [] + for item in value: + kind = str(item.get("kind", "")) + if kind not in KINDS: + raise ValueError(f"unknown kind {kind!r}") + cleaned.append({"kind": kind, "origin_uid": normalize_origin_uid(item.get("origin_uid"))}) + return cleaned + + class SyncRecord(BaseModel): model_config = ConfigDict(extra="ignore") @@ -65,14 +128,12 @@ class SyncRecord(BaseModel): updated_at: Optional[str] = None deleted: bool = False - @field_validator( - "origin_uid", "author_uid", "post_uid", mode="before", check_fields=False - ) + @field_validator(*REFERENCE_FIELDS, mode="before", check_fields=False) @classmethod def valid_reference(cls, value): return normalize_origin_uid(value) - @field_validator("parent_uid", mode="before", check_fields=False) + @field_validator(*OPTIONAL_REFERENCE_FIELDS, mode="before", check_fields=False) @classmethod def valid_optional_reference(cls, value): if value is None or value == "": @@ -154,14 +215,82 @@ class UserRecord(SyncRecord): return text +class ProjectRecord(SyncRecord): + kind: ClassVar[str] = KIND_PROJECTS + reference_fields: ClassVar[tuple[str, ...]] = ("author_uid",) + + author_uid: str + title: str + description: str + project_type: Literal["game", "game_asset", "software", "mobile_app", "website"] = "software" + status: str = "In Development" + platforms: str = "" + website_url: str = "" + repo_url: str = "" + + @field_validator("title", mode="before") + @classmethod + def valid_title(cls, value): + text = clean_text(value, multiline=False) + if not text or len(text) > LIMITS.max_project_title_length: + raise ValueError(f"title must be 1-{LIMITS.max_project_title_length} characters") + return text + + @field_validator("description", mode="before") + @classmethod + def valid_description(cls, value): + text = clean_text(value, multiline=True) + if not text or len(text) > LIMITS.max_project_description_length: + raise ValueError( + f"description must be 1-{LIMITS.max_project_description_length} characters" + ) + return text + + @field_validator("status", "platforms", mode="before") + @classmethod + def valid_short_text(cls, value): + return clean_text(value, multiline=False)[: LIMITS.max_project_field_length] + + @field_validator("website_url", "repo_url", mode="before") + @classmethod + def valid_link(cls, value): + text = clean_text(value, multiline=False) + if not text or len(text) > LIMITS.max_url_length: + return "" + try: + return normalize_website_url(text) + except ValueError: + return "" + + @field_validator("project_type", mode="before") + @classmethod + def known_type(cls, value): + text = clean_text(value, multiline=False).lower() + return text if text in ("game", "game_asset", "software", "mobile_app", "website") else "software" + + class PostRecord(SyncRecord): kind: ClassVar[str] = KIND_POSTS - reference_fields: ClassVar[tuple[str, ...]] = ("author_uid",) + reference_fields: ClassVar[tuple[str, ...]] = ("author_uid", "project_uid") author_uid: str title: str = "" content: str topic: str = "random" + project_uid: Optional[str] = None + + @field_validator("project_uid", mode="before") + @classmethod + def valid_project(cls, value): + if value is None or value == "": + return None + return normalize_origin_uid(value) + + def content_fields(self) -> dict: + fields = super().content_fields() + if fields.get("project_uid") is None: + fields.pop("project_uid", None) + return fields @field_validator("title", mode="before") @classmethod @@ -208,8 +337,151 @@ class CommentRecord(SyncRecord): return text +class PollOptionRecord(BaseModel): + model_config = ConfigDict(extra="ignore") + + origin_uid: str + label: str + + @field_validator("origin_uid", mode="before") + @classmethod + def valid_reference(cls, value): + return normalize_origin_uid(value) + + @field_validator("label", mode="before") + @classmethod + def valid_label(cls, value): + text = clean_text(value, multiline=False) + if not text or len(text) > LIMITS.max_poll_option_length: + raise ValueError(f"option label must be 1-{LIMITS.max_poll_option_length} characters") + return text + + +class PollRecord(SyncRecord): + kind: ClassVar[str] = KIND_POLLS + reference_fields: ClassVar[tuple[str, ...]] = ("author_uid", "post_uid") + + author_uid: str + post_uid: str + question: str + options: list[PollOptionRecord] + + @field_validator("question", mode="before") + @classmethod + def valid_question(cls, value): + text = clean_text(value, multiline=False) + if not text or len(text) > LIMITS.max_poll_question_length: + raise ValueError(f"question must be 1-{LIMITS.max_poll_question_length} characters") + return text + + @model_validator(mode="after") + def valid_options(self): + if not LIMITS.min_poll_options <= len(self.options) <= LIMITS.max_poll_options: + raise ValueError( + f"a poll needs {LIMITS.min_poll_options}-{LIMITS.max_poll_options} options" + ) + if len({item.origin_uid for item in self.options}) != len(self.options): + raise ValueError("option ids must be unique within a poll") + return self + + +class VoteRecord(SyncRecord): + kind: ClassVar[str] = KIND_VOTES + reference_fields: ClassVar[tuple[str, ...]] = ("voter_uid", "target_uid") + + voter_uid: str + target_kind: TargetKind + target_uid: str + value: Literal[1, -1] + + +class ReactionRecord(SyncRecord): + kind: ClassVar[str] = KIND_REACTIONS + reference_fields: ClassVar[tuple[str, ...]] = ("author_uid", "target_uid") + + author_uid: str + target_kind: TargetKind + target_uid: str + emoji: str + + @field_validator("emoji", mode="before") + @classmethod + def valid_emoji(cls, value): + text = clean_text(value, multiline=False) + if not text or len(text) > LIMITS.max_emoji_length or not is_single_emoji(text): + raise ValueError("emoji must be a single emoji") + return text + + +class FollowRecord(SyncRecord): + kind: ClassVar[str] = KIND_FOLLOWS + reference_fields: ClassVar[tuple[str, ...]] = ("follower_uid", "following_uid") + + follower_uid: str + following_uid: str + + +class PollVoteRecord(SyncRecord): + kind: ClassVar[str] = KIND_POLL_VOTES + reference_fields: ClassVar[tuple[str, ...]] = ("voter_uid", "poll_uid", "option_uid") + + voter_uid: str + poll_uid: str + option_uid: str + + +class AttachmentRecord(SyncRecord): + kind: ClassVar[str] = KIND_ATTACHMENTS + reference_fields: ClassVar[tuple[str, ...]] = ("author_uid", "target_uid") + + author_uid: str + target_kind: TargetKind + target_uid: str + filename: str + content_b64: str = "" + + @field_validator("filename", mode="before") + @classmethod + def valid_filename(cls, value): + text = clean_text(value, multiline=False).replace("\\", "/").rsplit("/", 1)[-1] + if not text or len(text) > LIMITS.max_filename_length: + raise ValueError(f"filename must be 1-{LIMITS.max_filename_length} characters") + return text + + @model_validator(mode="after") + def valid_content(self): + if self.deleted and not self.content_b64: + return self + try: + raw = base64.b64decode(self.content_b64.encode(), validate=True) + except (binascii.Error, ValueError): + raise ValueError("content_b64 must be strict base64") + if not raw: + raise ValueError("content_b64 is empty") + if len(raw) > LIMITS.max_attachment_bytes: + raise ValueError(f"attachment exceeds {LIMITS.max_attachment_bytes} bytes") + self._digest = hashlib.sha256(raw).hexdigest() + return self + + def content_bytes(self) -> bytes: + return base64.b64decode(self.content_b64.encode()) + + def content_fields(self) -> dict: + fields = super().content_fields() + fields.pop("content_b64", None) + fields["sha256"] = getattr(self, "_digest", "") + return fields + + RECORD_MODELS = { KIND_USERS: UserRecord, + KIND_PROJECTS: ProjectRecord, KIND_POSTS: PostRecord, KIND_COMMENTS: CommentRecord, + KIND_POLLS: PollRecord, + KIND_VOTES: VoteRecord, + KIND_REACTIONS: ReactionRecord, + KIND_FOLLOWS: FollowRecord, + KIND_POLL_VOTES: PollVoteRecord, + KIND_ATTACHMENTS: AttachmentRecord, } diff --git a/devplacepy/services/sync/selection.py b/devplacepy/services/sync/selection.py index 8bef2e35..1e3a1444 100644 --- a/devplacepy/services/sync/selection.py +++ b/devplacepy/services/sync/selection.py @@ -3,13 +3,20 @@ from __future__ import annotations import hashlib +import json import random from collections import Counter -from devplacepy.database.sync import get_links_by_origin_uids +from devplacepy.database.sync import ( + delete_candidate, + get_links_by_origin_uids, + load_candidates, + save_candidate, +) from devplacepy.services.sync.options import SyncOptions from devplacepy.services.sync.spec import ( KIND_POSTS, + LIMITS, SKIP_MIN_COMMENTS, SKIP_PARENT, SKIP_SAMPLED_OUT, @@ -32,28 +39,81 @@ def jittered(seconds: int, percent: int, rng: random.Random | None = None) -> in class Selector: - def __init__(self, origin: str, options: SyncOptions): + def __init__(self, origin: str, options: SyncOptions, *, persist: bool = False): self.origin = origin self.options = options + self.persist = persist self.dropped: list[tuple[str, dict, str]] = [] + self.revived = 0 + self.parked = 0 + + def revive(self, records: dict[str, list[dict]]) -> dict[str, list[dict]]: + if not self.persist or not self.options.min_comments: + return records + posts = {str(item.get("origin_uid")): item for item in records.get("posts", [])} + comments: dict[str, dict] = {} + users: dict[str, dict] = {} + for candidate in load_candidates(self.origin): + try: + data = json.loads(candidate["record"]) + except ValueError: + delete_candidate(self.origin, candidate["post_uid"]) + continue + posts.setdefault(str(data["post"].get("origin_uid")), data["post"]) + comments.update({str(item.get("origin_uid")): item for item in data.get("comments", [])}) + users.update({str(item.get("origin_uid")): item for item in data.get("users", [])}) + self.revived += 1 + if not self.revived: + return records + comments.update({str(item.get("origin_uid")): item for item in records.get("comments", [])}) + users.update({str(item.get("origin_uid")): item for item in records.get("users", [])}) + return {**records, "users": list(users.values()), "posts": list(posts.values()), "comments": list(comments.values())} + + def park(self, post: dict, comments: list[dict], users: dict[str, dict]) -> None: + if not self.persist: + return + wanted = {str(post.get("author_uid"))} | {str(item.get("author_uid")) for item in comments} + data = {"post": post, "comments": comments, "users": [users[uid] for uid in wanted if uid in users]} + encoded = json.dumps(data, default=str) + if len(encoded) > LIMITS.candidate_max_bytes: + return + if save_candidate( + self.origin, + str(post.get("origin_uid")), + encoded, + len(comments), + ttl_days=LIMITS.candidate_ttl_days, + cap=LIMITS.candidate_max_per_origin, + ): + self.parked += 1 def select(self, records: dict[str, list[dict]]) -> tuple[dict[str, list[dict]], Counter]: skipped: Counter = Counter() if not self.options.selective(): return records, skipped + records = self.revive(records) posts = records.get("posts", []) known = get_links_by_origin_uids(self.origin, KIND_POSTS, [str(post.get("origin_uid")) for post in posts]) - counts = Counter(str(comment.get("post_uid")) for comment in records.get("comments", [])) + users = {str(item.get("origin_uid")): item for item in records.get("users", [])} + by_post: dict[str, list[dict]] = {} + for comment in records.get("comments", []): + by_post.setdefault(str(comment.get("post_uid")), []).append(comment) kept, dropped = [], set() for post in posts: uid = str(post.get("origin_uid")) - reason = None if uid in known else self.refusal(post, counts[uid]) + reason = None if uid in known else self.refusal(post, len(by_post.get(uid, []))) if reason: skipped[reason] += 1 dropped.add(uid) self.dropped.append(("posts", post, reason)) + if reason == SKIP_MIN_COMMENTS: + self.park(post, by_post.get(uid, []), users) + elif self.persist: + delete_candidate(self.origin, uid) else: kept.append(post) + if self.persist: + delete_candidate(self.origin, uid) comments = [] for comment in records.get("comments", []): if str(comment.get("post_uid")) in dropped: diff --git a/devplacepy/services/sync/spec.py b/devplacepy/services/sync/spec.py index 80081eac..6dafd5dd 100644 --- a/devplacepy/services/sync/spec.py +++ b/devplacepy/services/sync/spec.py @@ -5,9 +5,38 @@ from __future__ import annotations from dataclasses import dataclass KIND_USERS = "users" +KIND_PROJECTS = "projects" KIND_POSTS = "posts" KIND_COMMENTS = "comments" -KINDS = (KIND_USERS, KIND_POSTS, KIND_COMMENTS) +KIND_POLLS = "polls" +KIND_VOTES = "votes" +KIND_REACTIONS = "reactions" +KIND_FOLLOWS = "follows" +KIND_POLL_VOTES = "poll_votes" +KIND_ATTACHMENTS = "attachments" +KIND_POLL_OPTIONS = "poll_options" +KINDS = ( + KIND_USERS, + KIND_PROJECTS, + KIND_POSTS, + KIND_COMMENTS, + KIND_POLLS, + KIND_VOTES, + KIND_REACTIONS, + KIND_FOLLOWS, + KIND_POLL_VOTES, + KIND_ATTACHMENTS, +) +TARGET_KINDS = {"post": KIND_POSTS, "comment": KIND_COMMENTS, "project": KIND_PROJECTS} +EXPORT_KINDS = ( + KIND_USERS, + KIND_PROJECTS, + KIND_POSTS, + KIND_COMMENTS, + KIND_POLLS, + KIND_FOLLOWS, + KIND_ATTACHMENTS, +) STATUS_CREATED = "created" STATUS_UPDATED = "updated" @@ -15,6 +44,7 @@ STATUS_UNCHANGED = "unchanged" STATUS_STALE = "stale" STATUS_REJECTED = "rejected" STATUS_FAILED = "failed" +STATUS_DEFERRED = "deferred" STATUSES = ( STATUS_CREATED, STATUS_UPDATED, @@ -22,6 +52,7 @@ STATUSES = ( STATUS_STALE, STATUS_REJECTED, STATUS_FAILED, + STATUS_DEFERRED, ) REASON_INVALID = "invalid" @@ -35,6 +66,14 @@ REASON_LOCALLY_DELETED = "locally_deleted" REASON_OLDER_THAN_SYNCED = "older_than_synced" REASON_TOMBSTONE_UNKNOWN = "tombstone_for_unknown" REASON_INTERNAL = "internal_error" +REASON_UNKNOWN_TARGET = "unknown_target" +REASON_UNKNOWN_POLL = "unknown_poll" +REASON_UNKNOWN_OPTION = "unknown_option" +REASON_UNKNOWN_PROJECT = "unknown_project" +REASON_SELF_FOLLOW = "self_follow" +REASON_IMMUTABLE = "immutable" +REASON_ACK_DELETED = "acknowledged_deleted" +REASON_QUEUE_FULL = "pending_queue_full" REASONS = ( REASON_INVALID, REASON_UNKNOWN_AUTHOR, @@ -47,6 +86,22 @@ REASONS = ( REASON_OLDER_THAN_SYNCED, REASON_TOMBSTONE_UNKNOWN, REASON_INTERNAL, + REASON_UNKNOWN_TARGET, + REASON_UNKNOWN_POLL, + REASON_UNKNOWN_OPTION, + REASON_UNKNOWN_PROJECT, + REASON_SELF_FOLLOW, + REASON_IMMUTABLE, + REASON_ACK_DELETED, + REASON_QUEUE_FULL, +) +DEFERRABLE_REASONS = ( + REASON_UNKNOWN_AUTHOR, + REASON_UNKNOWN_POST, + REASON_UNKNOWN_PARENT, + REASON_UNKNOWN_TARGET, + REASON_UNKNOWN_POLL, + REASON_UNKNOWN_PROJECT, ) SYNC_ACTOR = "sync" @@ -60,6 +115,7 @@ SKIP_TRANSLATION = "translation_failed" SKIP_REWRITE = "rewrite_failed" SKIP_PARENT = "parent_skipped" SKIP_IDENTITY_CAP = "identity_cap" +SKIP_ALIAS_ENGAGEMENT = "alias_engagement" TRANSIENT_SKIPS = (SKIP_IDENTITY, SKIP_TRANSLATION, SKIP_REWRITE, SKIP_IDENTITY_CAP) SKIP_REASONS = ( SKIP_SAMPLED_OUT, @@ -71,6 +127,7 @@ SKIP_REASONS = ( SKIP_REWRITE, SKIP_PARENT, SKIP_IDENTITY_CAP, + SKIP_ALIAS_ENGAGEMENT, ) @@ -94,6 +151,27 @@ class SyncLimits: future_skew_seconds: int = 300 earliest_year: int = 2000 max_errors_reported: int = 5 + max_batch_bytes: int = 32 * 1024 * 1024 + max_attachment_bytes: int = 5 * 1024 * 1024 + max_filename_length: int = 200 + max_project_title_length: int = 200 + max_project_description_length: int = 5000 + max_project_field_length: int = 500 + max_poll_question_length: int = 500 + max_poll_option_length: int = 200 + min_poll_options: int = 2 + max_poll_options: int = 6 + max_emoji_length: int = 16 + pending_max_per_origin: int = 5000 + pending_ttl_days: int = 7 + pending_max_attempts: int = 5 + pending_release_cap: int = 200 + pending_max_record_bytes: int = 256 * 1024 + export_page_size: int = 50 + deletions_page_size: int = 50 + candidate_ttl_days: int = 7 + candidate_max_per_origin: int = 1000 + candidate_max_bytes: int = 512 * 1024 @dataclass(frozen=True) @@ -109,6 +187,7 @@ class AliasLimits: standalone_min_length: int = 4 mention_min_length: int = 3 max_new_identities_per_run: int = 100 + heal_rewrites_per_run: int = 5000 max_translations_per_run: int = 40 translate_timeout_seconds: int = 90 name_timeout_seconds: int = 60 @@ -118,6 +197,40 @@ class AliasLimits: banned_fragments: tuple[str, ...] = ("user", "bot", "gpt", "claude", "assistant", "anon") +@dataclass(frozen=True) +class HandleLimits: + min_length: int = 3 + max_length: int = 32 + max_base_length: int = 18 + plain_weight: int = 40 + digits_weight: int = 30 + separated_weight: int = 8 + funny_weight: int = 17 + leet_weight: int = 5 + year_weight: int = 35 + short_number_weight: int = 40 + two_digit_weight: int = 25 + year_first: int = 1972 + year_last: int = 2006 + short_number_max: int = 99 + rerolls: int = 24 + top_level_labels: tuple[str, ...] = ( + "com", "net", "org", "io", "dev", "me", "sh", "co", "app", "ai", "be", "nl", "de", "fr", "uk", "us", + "ca", "au", "dk", "se", "no", "fi", "ru", "jp", "cn", "info", "blog", "page", "xyz", "tech", "site", + "online", "cc", "tv", "fm", "ly", "gg", "so", "im", "is", "it", "es", "pl", "cz", "ch", "at", "eu", + "pt", "br", "in", "kr", "tw", "hk", "sg", "nz", "za", "mx", "ar", "cl", "gl", "pub", "wiki", "social", + ) + ignored_labels: tuple[str, ...] = ("www", "blog", "mail", "m", "en") + relation_words: tuple[str, ...] = ("by", "via") + prefixes: tuple[str, ...] = ("its", "real", "mr", "just", "not", "dr", "captain", "sir", "the") + suffixes: tuple[str, ...] = ( + "_dev", "codes", "hq", "_ftw", "_sh", "_irl", "_the_great", "_wasnt_here", "_has_bugs", + "_on_fire", "_not_found", "_sudo", + ) + funny_numbers: tuple[str, ...] = ("42", "1337", "404", "007", "9000", "8080", "31337", "101", "64", "256", "500", "418") + leet_map: tuple[tuple[str, str], ...] = (("a", "4"), ("e", "3"), ("o", "0"), ("i", "1"), ("s", "5")) + + @dataclass(frozen=True) class SyncSpec: limits: SyncLimits = SyncLimits() @@ -137,3 +250,4 @@ class SyncSpec: SPEC = SyncSpec() LIMITS = SPEC.limits ALIAS = AliasLimits() +HANDLES = HandleLimits() diff --git a/devplacepy/static/css/admin.css b/devplacepy/static/css/admin.css index 176741e4..bce42c27 100644 --- a/devplacepy/static/css/admin.css +++ b/devplacepy/static/css/admin.css @@ -621,11 +621,13 @@ } .nodes-state.off, -.nodes-status.revoked { +.nodes-status.revoked, +.nodes-status.open { color: var(--danger); } -.nodes-status.suspended { +.nodes-status.suspended, +.nodes-status.half_open { color: var(--warning); } diff --git a/devplacepy/static/css/communities.css b/devplacepy/static/css/communities.css index 41906fbd..6c5dd175 100644 --- a/devplacepy/static/css/communities.css +++ b/devplacepy/static/css/communities.css @@ -442,3 +442,23 @@ .ledger-runs td[data-label="Mode"] { white-space: nowrap; } + +.community-schema-form { + display: grid; + gap: var(--space-md); + margin-bottom: var(--space-lg); +} + +.community-schema-text { + width: 100%; + min-height: 24rem; + padding: var(--space-md); + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--bg-card); + color: var(--text-primary); + font-family: var(--font-mono); + font-size: 0.8125rem; + line-height: 1.5; + resize: vertical; +} diff --git a/devplacepy/static/js/AdminCommunity.js b/devplacepy/static/js/AdminCommunity.js new file mode 100644 index 00000000..587be6c9 --- /dev/null +++ b/devplacepy/static/js/AdminCommunity.js @@ -0,0 +1,39 @@ +// retoor + +import { Poller } from "./Poller.js"; + +const REFRESH_MS = 3000; +const BUSY = "true"; + +class AdminCommunity { + constructor() { + this.root = document.querySelector("[data-community-root]"); + if (!this.root || this.root.dataset.busy !== BUSY) return; + this.poller = new Poller(() => this.refresh(), REFRESH_MS, { pauseHidden: true, immediate: false }); + } + + async refresh() { + const response = await fetch(window.location.href, { headers: { "Accept": "text/html" }, credentials: "same-origin" }); + if (!response.ok) return; + const fresh = new DOMParser().parseFromString(await response.text(), "text/html"); + const next = fresh.querySelector("[data-community-root]"); + if (!next) return; + this.root.querySelectorAll("[data-live]").forEach((section) => { + const replacement = next.querySelector(`[data-live="${section.dataset.live}"]`); + if (replacement) this.swap(section, replacement); + }); + this.root.dataset.busy = next.dataset.busy; + if (next.dataset.busy !== BUSY) this.poller.stop(); + } + + swap(section, replacement) { + const open = [...section.querySelectorAll("details[data-keep-open]")].map((item) => item.open); + section.innerHTML = replacement.innerHTML; + section.querySelectorAll("details[data-keep-open]").forEach((item, index) => { + item.open = Boolean(open[index]); + }); + } +} + +window.AdminCommunity = AdminCommunity; +export { AdminCommunity }; diff --git a/devplacepy/static/js/AdminNodes.js b/devplacepy/static/js/AdminNodes.js index 6508e86b..eefdbfaf 100644 --- a/devplacepy/static/js/AdminNodes.js +++ b/devplacepy/static/js/AdminNodes.js @@ -44,6 +44,10 @@ class AdminNodes { this.openSuspend(uid, button.dataset.label); return; } + if (action === "override") { + await this.override(uid, button.dataset.value, button.dataset.label); + return; + } const urls = { "unsuspend": `/admin/nodes/devices/${uid}/unsuspend`, "resolve-flag": `/admin/nodes/flags/${uid}/resolve`, @@ -52,6 +56,16 @@ class AdminNodes { }); } + async override(uid, value, label) { + const allowing = value === "allow"; + const message = allowing + ? `Let ${label} serve jobs although its address is not residential or mobile?` + : `Remove the address class override of ${label}?`; + const ok = await window.app.dialog.confirm({ message, danger: allowing }); + if (!ok) return; + await this.post(`/admin/nodes/devices/${uid}/class`, { value }, allowing ? "Override set" : "Override cleared"); + } + openSuspend(uid, label) { this.suspendForm.action = `/admin/nodes/devices/${uid}/suspend`; this.suspendTitle.textContent = `Suspend device: ${label}`; diff --git a/devplacepy/static/js/Application.js b/devplacepy/static/js/Application.js index af66fc6f..8e5b5651 100644 --- a/devplacepy/static/js/Application.js +++ b/devplacepy/static/js/Application.js @@ -28,6 +28,7 @@ import { TelegramPairing } from "./TelegramPairing.js"; import { NodeDonation } from "./NodeDonation.js"; import { AdminNotificationDefaults } from "./AdminNotificationDefaults.js"; import { AdminNodes } from "./AdminNodes.js"; +import { AdminCommunity } from "./AdminCommunity.js"; import { UserAiUsage } from "./UserAiUsage.js"; import { Tabs } from "./Tabs.js"; import { DeviiTerminal } from "./DeviiTerminal.js"; @@ -105,6 +106,7 @@ class Application { this.nodeDonation = new NodeDonation(); this.adminNotificationDefaults = new AdminNotificationDefaults(); this.adminNodes = new AdminNodes(); + this.adminCommunity = new AdminCommunity(); this.userAiUsage = new UserAiUsage(); this.tabs = new Tabs(); this.windows = new WindowManager(); diff --git a/devplacepy/static/js/GameFarm.js b/devplacepy/static/js/GameFarm.js index b6d54856..3c4f4f06 100644 --- a/devplacepy/static/js/GameFarm.js +++ b/devplacepy/static/js/GameFarm.js @@ -278,6 +278,13 @@ export class GameFarm { return `
${cells}${buy}
`; } + _plotStopHtml(plot) { + if (!plot.can_stop_replant) return ""; + const next = plot.stop_after_harvest ? "false" : "true"; + const label = plot.stop_after_harvest ? "▶ Resume auto-replant" : "⏹ Stop after harvest"; + return `
`; + } + _plotHtml(plot, farm) { const golden = plot.is_golden ? " game-plot-golden" : ""; const head = `
`; @@ -316,6 +323,7 @@ export class GameFarm { if (farm.is_owner && plot.fertilize_cost) { body += `
`; } + body += this._plotStopHtml(plot); } else { const goldenMark = plot.is_golden ? " ✨" : ""; const raided = plot.raided_pct ? `Raided ${plot.raided_pct}%` : ""; @@ -323,6 +331,7 @@ export class GameFarm { if (farm.is_owner) { const label = plot.is_golden ? "Harvest golden" : "Harvest"; body += `
`; + body += this._plotStopHtml(plot); } else if (plot.can_steal) { body += `
`; } else if (plot.steal_reason === "stripped") { diff --git a/devplacepy/templates/_game_grid.html b/devplacepy/templates/_game_grid.html index 6bbf5430..34205ebf 100644 --- a/devplacepy/templates/_game_grid.html +++ b/devplacepy/templates/_game_grid.html @@ -32,6 +32,9 @@ {% endif %} + {% if plot.can_stop_replant %} + {% include "_game_plot_stop.html" %} + {% endif %} {% else %} {{ plot.crop_name }}{% if plot.is_golden %} ✨{% endif %} @@ -41,6 +44,9 @@ + {% if plot.can_stop_replant %} + {% include "_game_plot_stop.html" %} + {% endif %} {% elif plot.can_steal %}
diff --git a/devplacepy/templates/_game_plot_stop.html b/devplacepy/templates/_game_plot_stop.html new file mode 100644 index 00000000..157d8a2c --- /dev/null +++ b/devplacepy/templates/_game_plot_stop.html @@ -0,0 +1,5 @@ + + + + +
diff --git a/devplacepy/templates/admin_communities.html b/devplacepy/templates/admin_communities.html index ab8f2743..3cad3719 100644 --- a/devplacepy/templates/admin_communities.html +++ b/devplacepy/templates/admin_communities.html @@ -33,6 +33,9 @@ + diff --git a/devplacepy/templates/admin_community.html b/devplacepy/templates/admin_community.html index 4f1c3889..24561e65 100644 --- a/devplacepy/templates/admin_community.html +++ b/devplacepy/templates/admin_community.html @@ -9,8 +9,10 @@ All communities
+
+
{% if busy %} -

A run or a discovery is in progress. Reload this page in a moment to see its result.

+

A run or a discovery is in progress. This page updates by itself.

{% endif %} {% if not sync_enabled %}

Community sync is switched off: live runs are refused, dry runs still work.

@@ -18,6 +20,7 @@ {% if community['last_error'] %} {% endif %} +
@@ -61,6 +64,12 @@ + @@ -76,6 +85,9 @@ + @@ -87,7 +99,7 @@
-
+

Actions

@@ -97,6 +109,7 @@
+ Edit schema
@@ -140,7 +153,7 @@

Recent runs

Everything the sync did
-
+
@@ -174,4 +187,5 @@
The most recent runs of this community, newest first
+
{% endblock %} diff --git a/devplacepy/templates/admin_community_ledger.html b/devplacepy/templates/admin_community_ledger.html index 5468136d..e5841b4b 100644 --- a/devplacepy/templates/admin_community_ledger.html +++ b/devplacepy/templates/admin_community_ledger.html @@ -12,6 +12,8 @@ Back to community
+
+
{% if run %}
{{ verdict['title'] }} @@ -27,8 +29,9 @@ {% else %}

The most recent records across every run, problems first. Open a run from the community page to see exactly what that run did.

{% endif %} +
-
+
{{ tiles|sum(attribute='listed') }} All listed @@ -78,7 +81,7 @@
-
+
@@ -122,10 +125,13 @@ {% include "_pagination.html" %} +
{% if run_log %} -
+
Full run log 
{{ run_log }}
{% endif %} +
+ {% endblock %} diff --git a/devplacepy/templates/admin_community_schema.html b/devplacepy/templates/admin_community_schema.html new file mode 100644 index 00000000..c24c6d68 --- /dev/null +++ b/devplacepy/templates/admin_community_schema.html @@ -0,0 +1,33 @@ +{% extends "admin_base.html" %} +{% block extra_head %} +{{ super() }} + +{% endblock %} +{% block admin_content %} +{% set slug = community['slug'] %} +
+

{{ community['name'] or slug }} learned schema

+ Back to community +
+

The schema is the structure learned once for this community: which feeds exist and which paths or selectors hold each field. Edit it by hand when the site markup defeated the learner. A save is checked first: the structure must be valid and a small live sample must collect at least one post. Nothing is stored otherwise.

+{% if notice %}

{{ notice }}

{% endif %} +{% if errors %} + +{% endif %} + + +
+ +
+ +{% if has_previous %} + + + +{% endif %} +{% endblock %} diff --git a/devplacepy/templates/admin_nodes.html b/devplacepy/templates/admin_nodes.html index fad38eee..08147e45 100644 --- a/devplacepy/templates/admin_nodes.html +++ b/devplacepy/templates/admin_nodes.html @@ -109,7 +109,7 @@ {% if device.status == 'suspended' and device.suspend_reason %}{{ device.suspend_reason }}{% endif %} - + {% else %} @@ -125,6 +130,78 @@
Records the sync handled, problems first
{{ device.kind }}{{ device.country or '-' }} {{ device.asn_class }}{{ device.country or '-' }} {{ device.asn_class }}{% if device.asn %} AS{{ device.asn }}{% endif %}{% if device.class_override == 'allow' %}, overridden{% endif %} {% if device.last_seen_at %}{{ local_dt(device.last_seen_at) }}{% else %}-{% endif %} {% if device.status == 'suspended' %} @@ -117,6 +117,11 @@ {% elif device.status == 'active' %} {% endif %} + {% if device.class_override == 'allow' %} + + {% elif device.asn_class not in ('residential', 'mobile') %} + + {% endif %}
+ +

Circuit breakers

+
+ + + + + + + + + + + + {% for circuit in circuits %} + + + + + + + {% else %} + + {% endfor %} + +
Destination and node circuit breakers that are not closed
SubjectStateFailuresSeconds left
{{ circuit.key }}{{ circuit.state }}{{ circuit.failures }}{{ circuit.seconds_left }}
No failing circuits.
+
+ +

Success by address class

+
+ + + + + + + + + + + + + + {% for row in outcomes %} + + + + + + + + + {% else %} + + {% endfor %} + +
Lifetime job outcomes per destination adapter and donor address class
DestinationAddress classAnsweredBlockedFailedTotal
{{ row.adapter }}{{ row.asn_class }}{{ row.ok }}{{ row.blocked }}{{ row.failed }}{{ row.total }}
No job has been served yet, so no rate is known (n = 0).
+
+ +
+ {{ breakdown("Lifetime counters", lifetime) }} + {{ breakdown("Shared cache", cache) }} +
+

Address database

+ {% if geo.available %} +

Month {{ geo.month }}, {{ geo.asn_rows }} ASN ranges, {{ geo.country_rows }} country ranges.

+

{{ geo.attribution }}

+ {% else %} +

Not installed. Every donor is classed unknown. Run devplace nodes geo-sync.

+ {% endif %} +
+

Settings

diff --git a/devplacepy/templates/docs/code-farm.html b/devplacepy/templates/docs/code-farm.html index 718b75e8..45bfe9b2 100644 --- a/devplacepy/templates/docs/code-farm.html +++ b/devplacepy/templates/docs/code-farm.html @@ -191,6 +191,7 @@ Each plot on a farm you view reports: raided for its full value), or `immune` (Security Fortress). Empty when stealable or not applicable. - `raided_fraction` / `raided_pct` - how much of this build has already been taken by raiders. +- `can_stop_replant` / `stop_after_harvest` - your own plots only, with Continuous Delivery: whether the pause can be set, and whether the next harvest will leave the plot empty. - `steal_cooldown_seconds` - seconds until your per-owner cooldown ends (only set with `steal_reason` of `cooldown`; also mirrored farm-wide as `steal_cooldown_seconds` on the farm). - `ready_at` - add the protection window to compute when a raid becomes possible. @@ -294,7 +295,7 @@ rounded down). Points never expire, and crop unlocks check points **ever earned* | Mastery upgrade | Key | Effect | Cost (points) | |------------------|-----|--------|:-------------:| -| 🔁 Continuous Delivery | `autoreplant` | Auto-replant the same crop right after every harvest, if still affordable and unlocked | 3 | +| 🔁 Continuous Delivery | `autoreplant` | Auto-replant the same crop right after every harvest, if still affordable and unlocked. Pause it for one plot with `POST /game/plot/stop` to choose a different crop | 3 | | 📊 Farm Analytics | `analytics` | Unlocks lifetime stats: `lifetime_coins_earned` and `lifetime_harvests` | 2 | | 📜 Legacy Contracts | `contracts` | Unlocks a weekly contract slot paying Stars and a temporary coin boost | 4 | @@ -432,6 +433,7 @@ your **API key** in an `X-API-KEY` header (or `Authorization: Bearer`). Your key | `GET` | `/game/farm/{username}` | Another member's farm. | | `POST` | `/game/plant` | Plant `crop` in `slot`. | | `POST` | `/game/harvest` | Harvest the build in `slot`. | +| `POST` | `/game/plot/stop` | With Continuous Delivery: `stop` true skips the replant after the next harvest of `slot` (cleared when the plot is planted again), false cancels. | | `POST` | `/game/fertilize` | Halve the remaining time of `slot`. | | `POST` | `/game/buy-plot` | Unlock one more plot. | | `POST` | `/game/upgrade` | Raise the CI tier. | @@ -525,7 +527,9 @@ lists you act on. A plot looks like this: "is_golden": false, "fertilize_cost": 22, "raided_fraction": 0.0, - "raided_pct": 0 + "raided_pct": 0, + "can_stop_replant": false, + "stop_after_harvest": false } ``` diff --git a/devplacepy/templates/docs/privacy.html b/devplacepy/templates/docs/privacy.html index c1987393..f3ac7586 100644 --- a/devplacepy/templates/docs/privacy.html +++ b/devplacepy/templates/docs/privacy.html @@ -16,8 +16,8 @@ used, and how it is retained, deleted and shared. | Technical | IP address and user agent on audit-relevant actions, request timing | Security, abuse prevention and rate limiting | | Assistant | Prompts and results of assistant, search and analysis features you invoke | Deliver the feature and attribute its cost to you | | Moderation | Reports you file, reports filed about you, and decisions taken | Operate the safety controls the platform is required to have | -| Donated bandwidth | Only if you donate: your device label and secret hash, a keyed hash of your IP address (never the address itself), an approximate country and network class, request and byte counters, and node events | Run the community node network and show you what you donated. See section 10. | -| Imported community content | Where an administrator synchronises an external community: public usernames, bios, locations, links, posts and comments published there, as the origin made them public. No email address, password, vote count, XP or role is imported, and imported accounts cannot sign in. Imported rows may be reported and removed like any other content | Mirror a public community inside DevPlace. If you are the author of imported content, ask for its removal through a report or by contacting an administrator. | +| Donated bandwidth | Only if you donate: your device label and secret hash, a keyed hash of your IP address (never the address itself), an approximate country and network class (looked up locally from the DB-IP lite database, licensed CC BY 4.0, https://db-ip.com; the address is never sent to a third party), request and byte counters, and node events | Run the community node network and show you what you donated. See section 10. | +| Imported community content | Where an administrator synchronises an external community: public usernames, bios, locations, links, posts, comments, polls, projects, follows, votes, reactions and attachments published there, as the origin made them public. No email address, password, XP or role is imported, and imported accounts cannot sign in. Imported rows may be reported and removed like any other content | Mirror a public community inside DevPlace. If you are the author of imported content, ask for its removal through a report or by contacting an administrator. | DevPlace runs **no third-party analytics, no advertising and no cross-site or cross-app tracking**. There is no tracking identifier shared with any other company. diff --git a/events.md b/events.md index 01d11a93..56c3ba75 100644 --- a/events.md +++ b/events.md @@ -19,12 +19,12 @@ Every state-changing action in DevPlace records one append-only row through `dev | Category | Keys | |---|---| | `account` | 16 | -| `admin` | 26 | +| `admin` | 28 | | `ai` | 11 | | `attachment` | 5 | | `auth` | 8 | | `backup` | 2 | -| `cli` | 49 | +| `cli` | 50 | | `container` | 41 | | `content` | 37 | | `database` | 4 | @@ -36,7 +36,7 @@ Every state-changing action in DevPlace records one append-only row through `dev | `message` | 2 | | `moderation` | 12 | | `news` | 9 | -| `nodes` | 9 | +| `nodes` | 10 | | `notification` | 4 | | `project` | 12 | | `project_files` | 14 | @@ -46,11 +46,11 @@ Every state-changing action in DevPlace records one append-only row through `dev | `security` | 3 | | `service` | 5 | | `social` | 10 | -| `sync` | 1 | +| `sync` | 3 | | `telegram` | 5 | | `tools` | 12 | -**Total: 380 keys.** +**Total: 386 keys.** ## Account and profile (`account`) @@ -89,6 +89,8 @@ Every state-changing action in DevPlace records one append-only row through `dev | `admin.community.delete` | `routers/admin/communities.py` | | `admin.community.relearn` | `routers/admin/communities.py` | | `admin.community.run` | `routers/admin/communities.py` | +| `admin.community.schema` | `routers/admin/communities.py` | +| `admin.community.schema_restore` | `routers/admin/communities.py` | | `admin.community.update` | `routers/admin/communities.py` | | `admin.devii_task.delete` | `routers/admin/devii_tasks.py` | | `admin.devii_task.disable` | `routers/admin/devii_tasks.py` | @@ -171,6 +173,7 @@ Every state-changing action in DevPlace records one append-only row through `dev | `cli.community.run` | `cli/community.py` | | `cli.containers.reconcile` | `cli/containers.py` | | `cli.nodes.clear` | `cli/nodes.py` | +| `cli.nodes.geo_sync` | `cli/nodes.py` | | `cli.nodes.suspend` | `cli/nodes.py` | | `cli.nodes.unsuspend` | `cli/nodes.py` | | `cli.deepsearch.clear` | `cli/jobs.py` | @@ -393,6 +396,7 @@ Every state-changing action in DevPlace records one append-only row through `dev | `game.perk.upgrade` | `routers/game/index.py` | | `game.plant` | `routers/game/index.py` | | `game.plot.buy` | `routers/game/index.py` | +| `game.plot.stop` | `routers/game/index.py` | | `game.prestige` | `routers/game/index.py` | | `game.quest.claim` | `routers/game/index.py` | | `game.steal` | `routers/game/farm.py` | @@ -563,6 +567,7 @@ Every state-changing action in DevPlace records one append-only row through `dev | `node.admin.settings` | `routers/admin/nodes.py` | | `node.device.create` | `routers/nodes/devices.py` | | `node.device.delete` | `routers/nodes/devices.py` | +| `node.device.override` | `routers/admin/nodes.py` | | `node.device.rotate` | `routers/nodes/devices.py` | | `node.flag.resolve` | `routers/admin/nodes.py` | | `node.killswitch` | `routers/admin/nodes.py` | @@ -575,6 +580,8 @@ Every state-changing action in DevPlace records one append-only row through `dev | Event key | Recorded in | |---|---| | `sync.batch` | `routers/sync/_shared.py` | +| `sync.deletions.ack` | `routers/sync/deletions.py` | +| `sync.export` | `routers/sync/export.py` | ## Developer tools (`tools`) diff --git a/gaplord.md b/gaplord.md new file mode 100644 index 00000000..49f52db3 --- /dev/null +++ b/gaplord.md @@ -0,0 +1,405 @@ +# gaplord: close every documented gap + +Plan and result in one file. The plan was written first, from a read of the code behind each gap. Every item is then worked in order and its **Status** and **Result** lines are filled in when it is done, with the measurement or test that proves it. Nothing is checked off without evidence. + +Written 2026-10-16. Scope: every entry under a `Gaps` heading in `devplacepy/services/{nodes,sync,community,skills}/CLAUDE.md` and `nodes/extension/README.md`, except the plain-text community passwords (explicitly excluded by the owner). + +Legend: `[ ]` open, `[~]` in progress, `[x]` done and proven, `[!]` cannot be closed by code (reason stated, nothing faked). + +## Ground rules for this work + +- No fiction. A gap is only closed when code exists, a test or measurement proves it, and the nested `CLAUDE.md` says what the system does today. Where a gap needs something this machine does not have, it is marked `[!]` with the exact missing input and what was built so the measurement happens by itself later. +- Specification first. Every new protocol field, limit, refusal, reason and kind is declared in `spec.py` (nodes) or `services/sync/spec.py` and read from there everywhere else. +- Caps on everything new (rows, bytes, intervals, retries). Machines hold, humans decide: nothing here bans a member or deletes a ledger row on its own. +- One fan-out per feature: data layer, schema (`*Out`), route, view, Devii tool, `docs_api`, docs prose, privacy page where data categories change, README, nested `CLAUDE.md`, tests in the matching tier. +- Em dash is forbidden in every touched file. No comments or docstrings in Python source. +- The dev server on port 10500 runs `--reload` against the live database. Files are therefore written whole and import-checked right after each edit, and no command in this work reads or writes `data/`. All scripts use scratch databases (`DEVPLACE_DATABASE_URL`, `DEVPLACE_DATA_DIR`). +- Final gate: `make test` (unit, api, e2e), plus `ruff`/`pyflakes` on touched files, `go vet` + `go test`, `node --test`. + +## Gap inventory (source of truth for this plan) + +| Id | Gap (as written in the docs) | Source | +|----|------------------------------|--------| +| G-N1 | Reddit viability unmeasured from a residential line | nodes | +| G-N2 | `country`, `asn`, `asn_class` never filled, datacenter exclusion (D6) not enforced | nodes | +| G-N3a | `last_seen_at` written on connect only | nodes | +| G-N3b | Socket counters per process | nodes | +| G-N3c | Fabric cache per process | nodes | +| G-N3d | Circuit breaker state in memory (and no per-destination admin view) | nodes | +| G-N3e | Same-process fast path not built, callers poll | nodes | +| G-N4 | Fetch-mode results not cross-checked (sampled redundancy) | nodes | +| G-N5 | DeepSearch skips hostile domains with a static list, no routing through `/nodes/internal/fetch` | nodes | +| G-N6a | Go client has no battery-aware pause (`pause_on_battery` ignored) | nodes | +| G-N6b | Go client treats a timed suspension as permanent until restarted | nodes | +| G-N6c | Extension enforces its own local hours in the browser clock, not the server timezone | nodes | +| G-N7 | Not measured: Reddit/X success per address class, WebSocket idle timeouts through the public hostnames, socket capacity of one worker, 24 hour soak | nodes | +| G-N8 | Launch blocked on a recorded legal decision | nodes | +| G-E1 | Extension has no icons | extension | +| G-E2 | `welcome` frame carries no device label | extension | +| G-E3 | Extension not tested in a real Chrome or Firefox session | extension | +| G-S1 | No export endpoint, sync is inbound only | sync | +| G-S2 | Votes, reactions, follows, attachments, polls and project links not synced | sync | +| G-S3 | A comment's post and parent must arrive earlier, no deferred resolution queue | sync | +| G-S4 | A local deletion is not communicated to the origin; the origin is rejected forever with `purged_locally` | sync | +| G-S5 | `min_comments` only sees comments collected in the same run | sync | +| G-S6 | A clone renamed by an administrator no longer matches its alias | sync | +| G-S7 | A username taken between alias generation and clone creation diverges from the rewritten mentions | sync | +| G-K1 | Community page does not live-refresh during a run | community | +| G-K2 | Run log and ledger are written only when the run finishes; a killed run leaves neither | community | +| G-K3 | Text search across a whole community scans up to 40000 rows | community | +| G-K4 | HTML comments follow `rel=next` links only | community | +| G-K5 | No two-factor, captcha or JSON-API login | community | +| G-K6 | HTML learning is heuristic plus AI; a hostile markup needs a hand-edited `SCHEMA` | community | +| G-K7 | Votes, reactions and attachments are not collected | community | +| G-K8 | Cross-feed duplicate detection compares sampled addresses at discovery time only | community | +| G-SK1 | No test installs a skill from a live network origin | skills | +| G-X | Excluded by the owner: community passwords stored in plain text | community | + +## Honest classification before any code + +Closable by code and proven on this machine: G-N2, N3a-e, N4, N5, N6a-c, E1, E2, S1-S7, K1-K8 (K5 partly, see its item), SK1, and the capacity part of G-N7. + +Closable only partly, with the remainder stated: + +- **G-N1 / G-N7 (success per address class).** A residential donor is a physical machine this session does not have. What is built: every job records the donor's address class, the admin page and `GET /admin/nodes` aggregate success per destination and class, and the docs print `n = 0` until real donors exist. The viability number itself stays unmeasured and the docs say so. +- **G-N7 (24 hour soak).** A soak can only be as long as this session. It is run in the background for as long as the work lasts and reported with its true duration. +- **G-N7 (idle timeout through the public hostname).** Measuring it requires an authenticated WebSocket on `devplace.net`, which is a write to the production database. It is measured against the local stack and the exact proxy settings are read from `nginx/nginx.conf.template`; the public leg is listed as needing the owner's explicit go-ahead. +- **G-E3 (Firefox).** Chromium is driven for real through Playwright. Firefox is attempted; if the available build cannot load a temporary add-on, the item stays open for Firefox only. +- **G-K5 captcha.** A captcha exists to stop automation. It is detected and reported loudly (`captcha_required`), never solved. + +Not closable by code: **G-N8** (a legal decision is a human act). It stays `[!]`; the plan records exactly what the decision must cover. + +## Protocol and specification changes (done first, shared by server and both clients) + +All additive and optional, `PROTOCOL_VERSION` stays 1 (an old client ignores unknown fields on server frames; the server rejects unknown fields from clients, so only server-to-client additions are made). + +| Change | Where | Why | +|--------|-------|-----| +| `welcome.device_label` (str, optional) | `spec.py` FRAMES, `session.handshake`, Go `Frame`, extension `WsClient` | G-E2 | +| `kick.until` (str, optional ISO instant) | `spec.py`, `registry.kick`, clients | G-N6b | +| Ticket `403` body carries `until` for a timed suspension | `routers/nodes/ticket.py` | G-N6b | +| New refusal `on_battery` ("donor is running on battery") | `spec.py` REFUSALS, `dispatcher.DONOR_LIMIT_CODES` (pause 300 s), Go `RefuseBattery`, extension `REFUSALS` | G-N6a | +| `LimitSpec.last_seen_write_seconds` (60), `redundancy_percent` (5), `redundancy_max_percent` (20), `geo_lookup_cache` (4096) | `spec.py` | G-N3a, N4, N2 | + +The extension spec test and the Go policy tests are updated in the same change so drift fails loudly. + +--- + +## Track N: node network, server (`devplacepy/services/nodes/`, `database/nodes.py`, `routers/nodes/`, `routers/admin/nodes.py`) + +### N-01 Geo and ASN database (G-N2) + +- Root cause: `record_device_connection` is called with defaults, so `country`, `asn`, `asn_class` stay empty (`routers/nodes/ws.py`). The doc says the fix is an offline ASN database. +- Source: DB-IP "IP to ASN Lite" and "IP to Country Lite" monthly CSVs, free, no key, CC BY 4.0 (attribution printed on the docs page and in `devplace nodes geo-status`). Verified on 2026-10-16: `https://download.db-ip.com/free/dbip-asn-lite-2026-10.csv.gz` answers 200, 7.0 MB, 477362 ranges. +- Build `services/nodes/geo.py`: `GeoDatabase` backed by one SQLite file at `config.NODE_GEO_DB` (declared in `DATA_PATHS`, created by `ensure_data_dirs`). Tables `geo_asn` and `geo_country`: `(family INTEGER, first BLOB, last BLOB, ...)`, primary key `(family, first)`. Addresses are packed with `ipaddress` (4 or 16 bytes, compared bytewise by SQLite). Lookup is `WHERE family=? AND first<=? ORDER BY first DESC LIMIT 1` then `last>=ip`. Memory stays flat, no 477k-entry Python list. +- Build `services/nodes/addressclass.py`: `classify(asn, org) -> residential | mobile | datacenter | unknown`. A static frozenset of well-known hosting and cloud ASNs (AWS, GCP, Azure, Hetzner, OVH, DigitalOcean, Linode, Vultr, Oracle, Alibaba, Cloudflare, Contabo, Scaleway, Leaseweb, M247, Choopa, Tencent, IBM, Rackspace) plus word-boundary org keywords (`hosting`, `cloud`, `datacenter`, `data center`, `vps`, `server`, `colo`, `dedicated`, `cdn`, `vpn`, `proxy`); mobile by keywords (`mobile`, `wireless`, `cellular`, `lte`, `gsm`); no data means `unknown`; everything else is `residential`. Heuristic by nature, so the docs call it that and the admin override (N-02) exists. +- Loader `geo_sync()` downloads through `stealth_async_client` (the project rule), verifies gzip, parses row by row with a hard cap on rows, loads into a temporary file and swaps it in atomically (verify before write). `--dry-run` reports counts only. CLI: `devplace nodes geo-sync [--dry-run] [--month YYYY-MM]`, `devplace nodes geo-status`. +- Connect path: `ws.py` resolves the address transiently (the address is never stored, only `hash_ip`), calls `GeoDatabase.lookup`, passes `country`, `asn`, `asn_class` to `record_device_connection`, and puts `asn_class` on `session`. A missing or unreadable database yields `unknown` plus the counter `geo_unavailable` and one log line per process. +- Tests: unit for packing/lookup edge cases (first/last address of a range, gap between ranges, IPv6, v4-mapped), classifier table (including keyword false-positive guards such as `Hostingly`), loader against a local fixture file, atomic swap; api for `devplace nodes geo-sync --dry-run` is CLI-unit. Static check that no raw address column exists on `node_devices`. +- Docs: nodes CLAUDE.md, `templates/docs/nodes.html` (donor-facing: what is looked up and that the address is not stored), `privacy.html` (the lookup), attribution line. +- Status: [~] Result: geo.py + addressclass.py + CLI geo-sync/geo-status; real DB-IP month built in 10 s (477362 ASN, 710834 country ranges, 48 MB), lookup 0.17 ms, installed through the real CLI against a scratch data dir; 30 unit tests + 2 api tests (header-driven class, missing database counter). Docs pending. + +### N-02 Enforce the datacenter exclusion, with an admin override (G-N2, D6) + +- Scheduler eligibility gains one rule: `session.asn_class in {residential, mobile}` or the device carries `class_override = "allow"`. `unknown` is refused by default, because "residential and mobile only" is the decision and an unverifiable address must not be assumed good (loud skip, counter `jobs_no_eligible_class`). Site setting `nodes_allow_unknown_class` (default `0`, apply live) lets an administrator run without a geo database during bring-up. +- New column `node_devices.class_override` (ensured in `NODES_COLUMNS`). Admin route `POST /admin/nodes/devices/{uid}/class` (`allow` or `clear`), `require_admin`, `is_senior_admin` against the device owner, audit key `node.device.override`, catalogued in `audit/events.md` and `category_for`. Devii `node_admin_class_override` (admin only, in `CONFIRM_REQUIRED`, declares `confirm`). +- Admin page: class column and an override button in the recent-devices table; `AdminNodesOut` carries both fields. +- Tests: unit scheduler (each class, override, unknown with and without the setting), api admin route (authz, senior-admin denial, audit row), Devii action gating. +- Status: [~] Result: scheduler rule (residential/mobile, unknown only with nodes_allow_unknown_class, per-device override), POST /admin/nodes/devices/{uid}/class, Devii node_admin_class_override (confirm-gated), audit node.device.override, admin page button; 5 scheduler + 6 api tests. Docs pending. + +### N-03 `last_seen_at` while connected (G-N3a) + +- One batched `UPDATE node_devices SET last_seen_at=:now WHERE uid IN (...)` per `last_seen_write_seconds` from `NodesService.reconcile`, one statement for all sessions (no per-session query, the measured lesson in the doc), plus a final write in the disconnect path. +- Tests: unit with fake sessions (statement count is 1 for N sessions, interval respected), api that a connected fake device advances `last_seen_at` while idle. +- Status: [~] Result: NodesService.persist_state touches all connected devices in one batched UPDATE per last_seen_write_seconds; unit tests (interval, force, failure). + +### N-04 Durable counters (G-N3b) + +- Table `node_counters(name PRIMARY KEY, value)` (derived, hard-GC only, not soft-deletable). `NodeCounters` keeps a delta per name; `reconcile` flushes all deltas in one `INSERT ... ON CONFLICT(name) DO UPDATE SET value = value + excluded.value` transaction and resets them only after commit. `snapshot()` stays the live process view; `lifetime()` reads the table. Admin page shows both. +- Tests: unit (flush adds, a failed flush keeps the delta, two processes flushing concurrently sum exactly: real separate processes, as the project procedure requires), api admin payload. +- Status: [~] Result: node_counters table, NodeCounters pending deltas flushed in one transaction every counter_flush_seconds, failed flush restores deltas; unit + api tests. Multi-process flush race still to run in validation. + +### N-05 Persisted circuit snapshots and a per-destination admin view (G-N3d) + +- `CircuitBreaker.snapshot()` returns `(key, failures, state, opened_at_wall)`. Table `node_circuits` is rewritten (upsert, delete when closed) from `reconcile` only when the breaker marked itself dirty. `NodesService` loads open circuits on start, converting wall time back to the monotonic clock, so a restart does not forget an open destination circuit. +- Admin page gets a "Circuits" table (destination or node, state, failures, seconds until half-open); `AdminNodesOut.circuits` capped at 100 rows. +- Tests: unit round trip across a simulated restart, expiry on load, cap, admin payload. +- Status: [~] Result: CircuitBreaker snapshot/restore + node_circuits + admin Circuit breakers table (read from the table so any worker answers); round-trip restart test. + +### N-06 Cross-process fabric cache (G-N3c) + +- Table `node_cache(key PRIMARY KEY, method, url, status, headers, body BLOB, stored_at, expires_at, size)` as an L2 under the existing in-process `ResultCache` (L1). Reads: L1, then L2 (then promote). Writes: both. Bounded by `LimitSpec.cache_max_bytes` and `cache_entry_max_bytes` and the TTL; `NodesService` prunes expired rows and, when over the byte cap, the oldest rows, in `run_once`. `cache.clear()` clears both. Responses with `status >= 500` or truncated are never stored (existing rule kept). +- Tests: unit (L1 miss L2 hit promotes, TTL, cap eviction order, oversize skipped), a real two-process test (worker A stores, worker B hits), kill-switch off does not serve from cache? (decision: cache is served regardless, as today, and the test pins that). +- Status: [~] Result: SharedResultCache (L1 + node_cache table), prune in service, cross-instance hit test; real two-process proof still to run in validation. + +### N-07 Same-process fast path (G-N3e) + +- `services/nodes/wake.py`: `WakeHub` holding an `asyncio.Event` for the dispatcher and one per awaited job. `JobBroker.enqueue` signals the dispatcher when it runs in the lock-owner process; `dispatcher._account`/`_failed`/`broker.complete` resolve the job event; `JobBroker.wait` awaits the event first and keeps the DB poll only as the cross-process fallback and a safety net. No behavior change in other workers. +- Measure before and after on this machine with a fake session: enqueue-to-assigned latency (today up to `dispatcher_tick_ms` = 200 ms) and result-to-caller latency (today up to `poll_interval_ms` = 100 ms). The numbers go into the doc. +- Tests: unit (event wake, no lost wake-up when set before wait, timeout path unchanged), existing broker tests stay green. +- Status: [~] Result: WakeHub + JobBroker events; unit tests prove immediate wake. Before/after latency numbers still to measure. + +### N-08 Sampled redundancy for fetch-mode results (G-N4) + +- Spec: `redundancy_percent` (default 5, ceiling `redundancy_max_percent` 20, site setting `nodes_redundancy_percent` clamped to it, 0 turns it off). +- After a `fetch`-transport result is accepted, a deterministic sample (`sha256(job uid) % 100 < percent`) enqueues a shadow job (`caller_kind = "shadow"`, `shadow_of = `, `exclude_device = `), scheduled to a different donor owner. The shadow never spends the caller's quota and never enters the cache. +- Comparison is on a **shape**, not bytes (live pages change): status class, JSON top-level key set or the HTML ``/first heading, plus a body size ratio inside a band. A mismatch raises a `node_flags` row `redundancy_mismatch` against both devices and counts `redundancy_mismatches`; no auto-ban. Match and mismatch counts are persisted (N-04) and shown on the admin page so the false-positive rate is itself measured. +- Tests: unit shape extractor (JSON, HTML, binary), sampling determinism and rate over 10000 uids, scheduler exclusion, dispatcher comparison with two fake nodes (match, forged body, one node errors), cap. +- Status: [~] Result: shape.py + dispatcher shadow jobs (scheduler exclusion, flags, counters, clamped setting); 9 shape + 8 dispatcher tests. + +### N-09 DeepSearch routes node-servable hostile URLs through the fabric (G-N5) + +- `services/jobs/deepsearch/crawl.py`: `_resolve_candidate` asks a new `NodeCrawl` helper first when the URL is allowlisted (`SPEC.destination(host)`) and `fabric.can_serve(url)` is true; the fetch uses `shared_fabric().fetch(..., caller_kind="user", caller_id=<session owner>)`, so it spends the owner's quota, honours the kill switch and falls back to the snippet on any `FabricError`. The static `HOSTILE_DOMAINS` tuple stays only for hosts the allowlist cannot serve (X, YouTube, Instagram and so on); Reddit is removed from it and the owner uid is threaded from the DeepSearch worker through `crawl()`. +- Same extraction path as `fetch_page` (shared function), so the text is parsed identically. Guests are not routed (no owner quota to charge). +- Tests: unit with a fake fabric (served, refused, unavailable, quota exceeded, guest), existing crawl tests stay green. +- Status: [~] Result: crawl.node_page via POST /nodes/internal/fetch with the owner key for allowlisted hostile hosts; 6 unit tests. + +### N-10 Success per address class, measured automatically (G-N1, G-N7) + +- `node_requests` records `device_asn_class` at claim time. `database.nodes.success_by_class(days)` aggregates `(adapter, asn_class, outcome)` over the retained window. `GET /admin/nodes` (`AdminNodesOut.success`) and the admin page show it; Devii `node_admin_status` includes it. +- Docs print the live table. Until a residential donor exists the cell reads `n = 0`, and the Reddit-from-residential gap is documented as unmeasured with the exact query that will answer it. +- Tests: unit aggregation (including empty), api payload. +- Status: [~] Result: claim records device_asn_class; durable outcome counters (outcome:adapter:class:kind) + admin table with n = 0 state; unit + api tests. + +### N-11 Measurements (G-N7) + +- **Socket capacity of one worker:** scratch server on a free port, up to `max_connected_nodes` (2000) synthetic devices and tickets created in a scratch database, asyncio WebSocket clients answering pings; record RSS, event-loop lag (a 10 ms ticker), reconcile tick duration and connect rate at 250, 500, 1000, 2000 sockets. Harness lives in the scratchpad (disposable), method and numbers go into the doc. +- **Soak:** same harness held open with jobs flowing through fake nodes; started in the background as early as possible and sampled every minute (RSS, loop lag, socket count, error counters). Reported with its real duration. Not claimed as 24 hours unless it ran 24 hours. +- **Idle timeout:** local stack measured (idle WebSocket for 5 minutes with no application frames, ping interval 20 s, observing whether anything closes it); `nginx/nginx.conf.template` read for `proxy_read_timeout` on the nodes location and quoted. The public hostname leg needs a production write and is left for the owner's go-ahead. +- Status: [~] Result: measured. Fast path p50 209 ms to 25 ms; capacity 2000 sockets in 341 MB; soak 81 minutes (NOT 24 h) clean; counter and cache races exact. Residential Reddit rate and public-hostname idle remain unmeasured (see Cannot be closed by code). + +--- + +## Track C: clients (`nodes/client/` Go, `nodes/extension/`) + +### C-01 Go: battery-aware pause (G-N6a) + +- `ServerSettings` gains `PauseOnBattery`. New files with build tags: `battery_linux.go` (parse `/sys/class/power_supply/*`: on battery when a `Battery` supply exists, no `Mains`/`USB` supply is online, and the battery status is `Discharging`), `battery_darwin.go` (`pmset -g batt`), `battery_windows.go` (`GetSystemPowerStatus` through `syscall.NewLazyDLL`, no cgo), `battery_other.go` (unknown). `Limiter.Acquire` refuses with `on_battery` when the server setting is on and the probe says battery; the probe result is cached for 15 seconds. The server pauses the donor for 300 s on that code (N-spec change). Unknown means "not on battery" and logs once at debug (a probe failure must not silence a donor forever). +- Tests: table tests for the Linux parser with fixture trees, limiter test with a fake probe, the refusal code against `spec.py`. +- Status: [x] Result: Go battery probes (linux/darwin/windows/other), limiter refuses on_battery; go test green (agent report, re-verified: go vet clean, go test ok). + +### C-02 Go and extension: a timed suspension is not permanent (G-N6b) + +- Server: ticket `403 suspended` and the `kick` frame carry `until` for a timed suspension (read from `suspended_until`). A suspension without an end date stays a plain hold. +- Go: `suspended` leaves `FatalReasons`. New `outcome.holdUntil`; `Session.Run` sleeps until `until` (clamped to at least 60 s and at most 24 h, plus jitter) and then retries the ticket. `revoked` and the other reasons stay fatal. Extension `WsClient` gets the same state (`SUSPENDED` with a countdown in the popup). +- Tests: Go session test with a fake clock (suspended with `until`, without, expired), extension unit test, api test that the ticket body carries `until`. +- Status: [x] Result: Go holdUntil + extension SUSPENDED state; server sends until in ticket 403 and kick; tests in Go, node and api tiers. + +### C-03 Extension: server-timezone hours (G-N6c) + +- `Limits.applyServerSettings` also reads `active_from`, `active_to`, `timezone`; `inActiveHours()` requires both the donor's local window (browser clock, as configured in the popup) and the server window evaluated in the server-stored timezone with `Intl.DateTimeFormat(..., {timeZone})`. An invalid zone falls back to UTC, as `scheduler._zone` does. +- Tests: node tests with a stubbed clock across zones, windows that wrap midnight, an invalid zone. Parity test against the Go `InWindow` and the Python `in_active_hours` for the same 200 sampled instants. +- Status: [x] Result: extension server-timezone hours with Python-generated parity table shared by JS and Go tests; node --test 68/68 re-verified. + +### C-04 Extension icons (G-E1) + +- Four PNGs (16, 32, 48, 128) generated reproducibly by a small script using Pillow (already a dependency), a node-network glyph on the project accent colour. `manifest.json` gets `icons` and `action.default_icon`; `build.sh` includes them; the packaging test checks the files exist and the sizes match. +- Status: [x] Result: four PNG icons generated by nodes/extension/tools/make_icons.py, manifest + build.sh updated, package test. + +### C-05 Device label in `welcome` (G-E2) + +- Server sends `device_label` (from `node_devices.label`); extension `State` stores it and the popup shows the server label as the authority, falling back to the local one before the first connect. Go prints it in `status`/`run` logs. +- Tests: frame spec test, extension popup state test, Go decode test. +- Status: [x] Result: welcome.device_label server + Go + extension; api test asserts it. + +### C-06 Real browser test of the extension (G-E3) + +- Playwright persistent Chromium context with `--disable-extensions-except` and `--load-extension` against a scratch DevPlace server on a free port (not 10500/10501): register a device, drive the disclosure and connect flow through the real popup page, assert the socket connects (server registry shows the node), a job for an allowlisted host is answered (`via: fetch`; a 403 from this datacenter address is an acceptable status, as in the Go run), refusals work, and disabling disconnects. Firefox: attempt with the installed Playwright Firefox; if a temporary add-on cannot be loaded, record that and leave E3 open for Firefox. +- The test is kept as `tests/e2e/nodes/extension.py` (skips with a stated reason when Chromium extensions cannot load, never silently passes). +- Status: [x] Result: Chromium e2e test tests/e2e/nodes/extension.py passes (~10 s); Firefox 158 verified by scratch geckodriver script; CSP fix found and applied by that test; method in nodes/extension/README.md. + +### C-07 Client build and verification + +- `go vet ./...`, `go test ./...`, `node --test`, `nodes/client/build.sh` into a scratch directory (not `data/`). The distributable build into `data/node_clients/` is a deploy step left to the owner. +- Status: [x] Result: go vet, go test, node --test 68/68, cross-compile linux/windows/darwin amd64+arm64 and freebsd (agent report; I re-ran vet, test and node tests). + +--- + +## Track S: community sync (`devplacepy/services/sync/`, `database/sync.py`, `routers/sync/`) + +### S-00 Design decisions for new kinds (G-S2) + +- Kinds, in dependency order: `users`, `projects`, `posts`, `comments`, `polls`, `votes`, `reactions`, `follows`, `poll_votes`, `attachments`. `KINDS` is the single list; `SyncPusher`, `BatchRunner`, docs and Devii tools read it. +- Counters stay un-importable as numbers. A synced vote exists only with a synced voter, and the target `stars` is recomputed from the local `votes` rows by the same function `apply_vote` uses (`update_target_stars` + `clear_user_stars`), once per touched target per batch. This replaces "synced posts have stars = 0" with "stars are derived from synced votes only"; the doc rule is rewritten, not silently broken. +- No side effects, as before: no notifications, XP, badges, mention fan-out, AI correction/modifier. Content filter runs on every authored text (`projects`, `polls`). +- Records: + - `ProjectRecord`: `author_uid`, `title`, `description`, `status`, `platforms`, `website_url`, `repo_url`, `project_type`; always public and writable (`is_private = 0`, `read_only = 0`); no files, no containers. + - `PostRecord.project_uid` (optional reference to a synced project) closes "project links". + - `PollRecord`: `post_uid`, `question`, `options: [{origin_uid, label}]` (2 to 6, same caps as `create_poll`); options are rows of the poll, fingerprinted together. + - `VoteRecord`: `voter_uid`, `target_kind` in `post|comment|project`, `target_uid`, `value` in `{1, -1}`; a cleared vote is `deleted: true`. + - `ReactionRecord`: `author_uid`, `target_kind` in `post|comment|project`, `target_uid`, `emoji` (validated against the existing reaction emoji rule); tombstone removes. + - `FollowRecord`: `follower_uid`, `following_uid`; no self follow; tombstone removes. + - `PollVoteRecord`: `voter_uid`, `poll_uid`, `option_uid`; tombstone removes. + - `AttachmentRecord`: `author_uid`, `target_kind` in `post|comment|project`, `target_uid`, `filename`, `mime_type`, `content_b64`. **No network fetch inside a processor** (it runs on the request path and SQLite/HTTP stay out of each other's way): the sender downloads and sends the bytes; the processor decodes, applies the existing upload validators (type, size, image dimensions), and writes through `attachments.store_attachment` + `link_attachments`. `max_attachment_bytes` is a `SyncLimits` field and the envelope gets a total byte cap (`max_batch_bytes`). +- Voter-visibility: `poll_votes` are private ballots in this product (`UNREPORTABLE_TABLES` says so). They are accepted inbound (the sender owns that decision) and never exported (S-02). +- Each new local row is soft-deletable and created with `deleted_at: None, deleted_by: None`; tombstones soft-delete with `deleted_by = "sync"`; toggles revive instead of duplicating (the project's hard rules). +- Reportability: `projects` and `polls` are already reportable targets; synced rows reuse them. `votes`, `reactions`, `follows`, `poll_votes`, `attachments` are not new tables, so `REPORTABLE_TARGETS`/`UNREPORTABLE_TABLES` stay exact (the registry test is the check). +- Status: [~] Result: decisions implemented; correction recorded: votes, reactions and poll ballots are accepted inbound but NOT exported because no UI exposes who voted. + +### S-01 Implement the new kinds (G-S2) + +- `spec.py` (kinds, limits), `records.py` (models + `RECORD_MODELS`), `normalize.py` (emoji, options, platform lists), `processors.py` (one processor per kind on `RecordProcessor`, `PROCESSORS`), `database/sync.py` (link helpers for composite natural keys: a vote/reaction/follow link is keyed by the sender's own `origin_uid`; a second record for the same `(voter, target)` with a different `origin_uid` revives the same local row instead of duplicating, the concurrency lesson of the toggles), `routers/sync/` one leaf per kind (`projects.py`, `polls.py`, `votes.py`, `reactions.py`, `follows.py`, `poll_votes.py`, `attachments.py`), `GET /sync/spec.json` gains the schemas, Devii `sync_<kind>` tools (`requires_admin`, form-encoded `records`), `docs_api` endpoints, README, `docs/privacy.html` row update, sync CLAUDE.md rewritten for the new rules. +- Tests: unit per normaliser and processor (create, update, stale, tombstone, locally deleted, purged, wrong references), api per route (authz, gate, dry run, cap), stateful fuzz extended to all kinds with invariants (no dangling links, no orphan votes/reactions/follows, `stars` equals net local votes for every synced target, one vote per voter and target, no self follow), and a real multi-process race on the vote/follow toggle rows started from a fresh unseeded database (40 racers, exactly one row). +- Status: [~] Result: 10 kinds, records, processors, 7 new routes, Devii tools, docs_api; 69 api tests incl. new kinds. Fuzz + multi-process race and docs/privacy pending. + +### S-02 Export endpoint (G-S1) + +- `GET /sync/export/{kind}?since=<iso>&cursor=<opaque>&limit=<=50>` (administrator only, behind `sync_enabled`, audited as `sync.export`). Keyset pagination on `(updated_at, uid)`; the cursor is a signed-free, validated base64 of those two values. Output items are exactly the `*Record` models, so a DevPlace instance can feed another with the sender it already has. Soft-deleted rows are emitted as tombstones. +- Exported: `users` (public fields only: never email, hash, role, api key), `projects` (public only), `posts`, `comments`, `polls`, `votes` and `reactions` (author uid and target only), `follows`, `attachments` (as `url`, not bytes). **Not exported:** `poll_votes` (private ballots), anything private or deleted-by-account-deletion beyond the tombstone, and rows that arrived by sync themselves (`sync_links.local_uid`), so two instances cannot echo each other's data in a loop. +- Tests: api per kind (shape equals the inbound model, no private field, pagination is gapless and repeat-free over 120 rows, tombstones, the echo guard), a round trip test (export from one scratch database, import into another through the real routes, compare counts). +- Status: [~] Result: GET /sync/export/{kind} with expression indexes (EXPLAIN-tested), echo guard, keyset pagination; 27 api tests with deletions. Cross-database round trip pending. + +### S-03 Deferred resolution queue (G-S3) + +- Statuses gain `deferred` (an applied-later state, distinct from `rejected`). Table `sync_pending` (`origin`, `kind`, `origin_uid`, `missing_kind`, `missing_uid`, `record` JSON, `attempts`, `created_at`, `expires_at`) with a unique index on `(origin, kind, origin_uid)` so a resend replaces instead of stacking. +- A record rejected for `unknown_author`, `unknown_post`, `unknown_parent`, `unknown_target`, `unknown_poll`, `unknown_option`, `unknown_project` is queued (not on dry run) and reported `deferred` with the same reason. When any record of kind K with origin uid U is `created`, `BatchRunner` looks up pending rows waiting on `(K, U)` and re-applies them with the same processors, iteratively (a released comment can release its replies) under `pending_release_cap_per_batch`. Caps: `pending_max_per_origin` 5000 (the 5001st is rejected with the old reason, loudly counted), `pending_ttl_days` 7, `pending_max_attempts` 5; expired rows are garbage collected hard by the service already responsible for sync housekeeping (Community sync service `run_once`). +- The community ledger gains the outcome `deferred` (warn tone) with a plain-language reason; the pusher's projection logic for dry runs is unchanged. +- Tests: unit (queue, release in any order, chain of three, cap, ttl, replace-on-resend, dry run writes nothing), the 16-process race (post and its 15 comments arriving in random order from separate processes: exactly the right final counts), ledger reason text test. +- Status: [~] Result: sync_pending + PendingQueue; 12 api tests (chain, shuffle of 18 records, cap, ttl, dry run). 16-process race pending. + +### S-04 Deletion feedback to the origin (G-S4) + +- `GET /sync/deletions?origin=<id>&since=<iso>&cursor=` (administrator only) lists synced rows deleted or purged on this side: `{kind, origin_uid, deleted_at, deleted_by_class: "owner|moderator|system|purged"}`; `POST /sync/deletions/ack` (`origin`, up to 50 `{kind, origin_uid}`) stamps `sync_links.ack_at`. A record that the origin re-sends after acknowledging answers `unchanged / acknowledged_deleted` instead of the loud `rejected / purged_locally`, so the noise stops once the origin has been told. New columns `sync_links.deleted_seen_at`, `ack_at` (ensured). +- The scheduled community sender skips records whose link is locally deleted (cheap lookup) and records the skip in the ledger, so a re-collection of a deleted post stops being pushed every run. +- Devii tools `sync_deletions`, `sync_deletions_ack`; docs; audit `sync.deletions.ack`. +- Tests: api (listing by class, pagination, ack, reappearance answers `acknowledged_deleted`, authz), unit. +- Status: [~] Result: GET /sync/deletions and POST /sync/deletions/ack, acknowledged_deleted quiet answer; tests in deletions.py. + +### S-05 `min_comments` counts what is already synced (G-S5) + +- `Selector` counts, per post, comments collected in this run plus the live local comment count of the post's existing link (one batched `GROUP BY target_uid` query for all linked posts in the run, not per post). A post that was kept once is never dropped (existing rule), so this only changes first-time decisions. +- Tests: unit (collected 0 + 3 already synced passes a threshold of 3; none anywhere is dropped; the batched query is one statement). +- Status: [~] Result: Selector candidates (sync_candidates) park and revive a post with its comments and authors across runs; 10 unit tests. + +### S-06 and S-07 Alias heals when the clone name changes or collides (G-S6, G-S7) + +- `Aliaser.reconcile(origin)` runs at the start of every aliased run and after the push: for every identity whose clone user exists, compare `users.username` with `our_name`. On a difference: update the identity (`our_name`, `our_key`, skipping and logging when the new key collides with another identity), then clear the fingerprint of the origin's posts and comments whose stored text contains the old alias (bounded to 5000 links per run, oldest first), so the next push rewrites them with the live name. The rewriter always resolves names through the live clone username first. +- This covers both gaps: an administrator rename, and the case where `free_username` gave the clone a suffix because a registration took the name between alias generation and clone creation (the post-push reconcile sees the suffix and heals the mentions on the next run). The race window itself is also narrowed: the clone user is created before dependent content is rewritten. +- Tests: unit (rename, suffix collision, key collision skip, bounded sweep, nothing to do), an integration run through `CommunityRunner` with a fake collector: rename between runs, mentions follow. +- Status: [~] Result: AliasHealer rewrites stored mentions and follows the clone name; wired before and after the push in CommunityRunner; 6 unit tests. + +--- + +## Track K: community discovery (`devplacepy/services/community/`, `routers/admin/communities.py`, `static/js/`) + +Note: `runtime.py` and `extract.py` are inlined into generated collectors by `generator.py`. Every change there is covered by a generator test that executes the generated file, and existing communities regenerate their files (the database row holds the schema; files are regenerated when missing). + +### K-01 Live refresh while a run is in progress (G-K1) + +- `/admin/communities/{slug}` and the ledger page swap their `[data-live]` sections on an interval while the community is `busy` or a run row is `running` (the `AdminNodes.js` pattern, shared `Poller`, stops when idle). New `static/js/AdminCommunity.js` (one class, `app.adminCommunity`), no inline script; responsive at phone width. +- Tests: e2e (start a slow fake run, the progress counters change without reload, polling stops at the end), api (the page carries the live hooks and the run row state). +- Status: [x] Result: AdminCommunity.js live refresh on data-busy/data-live sections; busy field on AdminCommunityLedgerOut; tests/e2e/admin/communities/live.py. + +### K-02 Incremental run log and ledger, interrupted runs are visible (G-K2) + +- `CommunityRunner.run` registers a checkpoint callback: every `checkpoint_seconds` (15) and after every pushed batch it writes the current `LogBook` text, tally and verdict `running` into the run row, and flushes new ledger drafts with `LedgerStore.append` (continuing `seq`, respecting the same caps and rank rules). The final `finish_run` replaces the totals; ledger rows are never written twice (an index on `(run_uid, seq)` makes a duplicate a no-op). +- At service start, `CommunityStore.reap_interrupted()` marks any run left `running` for longer than `run_timeout_seconds` as `interrupted` (verdict `bad`, headline "interrupted: the process ended before the run finished") and keeps whatever was checkpointed. Retention (`prune_runs`) is unchanged. +- Tests: unit (a run killed after the second batch leaves log, tally and ledger; resume never duplicates), a real process kill test (`SIGKILL` a subprocess mid-run, then reap), caps still hold. +- Status: [x] Result: RunCheckpoint every 15 s, unique (run_uid, seq), reap_interrupted marks RUN_INTERRUPTED; unit tests in checkpoint.py. + +### K-03 Indexed text search (G-K3) + +- SQLite FTS5 external-content table `community_run_items_fts` over `label`, `origin_uid`, `detail` with `tokenize='trigram'` (substring semantics like the current `LIKE`, query minimum 3 characters), kept in step by triggers on `community_run_items`. Queries of 3+ characters use `MATCH` joined back to the scoped rows; shorter queries keep the bounded `LIKE` on the scoped index. If FTS5 or the trigram tokenizer is unavailable, `init_db` logs it once and the old path is used (the same results, the old cost). Prune deletes cascade through the triggers. +- Measure on the same scratch ceiling used before (10 runs of 4000 rows plus the community scan) and update the numbers: the no-match search was 37 ms, the old unbounded case 97 ms. `EXPLAIN QUERY PLAN` test extended so the FTS path is asserted, not assumed. +- Tests: unit (substring parity with `LIKE` over a generated corpus including `%`, `_`, quotes, unicode), query-plan test, trigger consistency after insert, delete and prune. +- Status: [x] Result: FTS5 trigram table with triggers for community-wide search of 3+ chars; measured 34 ms to 0.4 ms (no match), 34.5 to 0.8 ms (rare), common term 9.4 to 21 ms (slower, accepted). + +### K-04 Comment pagination beyond `rel=next` (G-K4) + +- `comments.pagination` becomes a learned and overridable spec: `next_link` (selector list, default adds text-matched anchors in a pagination container: `next`, `older`, `more`, `volgende`, `›`, `»`, `→`), `page_param` (`?page=N`, `?p=N`, `/page/N`, detected from the current URL or the numbered page links), `json_next` (a JSON `next`/`next_page`/`links.next` key). The collector stops on a page that adds no new comment ids or repeats a URL, always under `max_comment_pages`. +- The learner (`PageProbe`) detects the style from the sampled post pages and stores it; `NEXT_PAGE_FIELDS` remains the default. +- Tests: extraction fixtures for each style (rel next, text next, numbered page links, `?page=`, `/page/N`, JSON next), loop and duplicate guards, generated-collector parity. +- Status: [x] Result: next_link, page_param, json_next, none hints learned and enforced; stops on a page with no new comments; unit tests pagination.py. + +### K-05 Login: JSON-API and two-factor, captcha detected (G-K5, passwords excluded) + +- JSON-API login: `schema.login.kind = "json"` with `url`, `fields` (username/email/password key names), `token_path` (JSON path to a bearer token, optional); success is a 2xx plus either a token or a session cookie. `LoginLearner` probes a short fixed list of common endpoints (`/api/login`, `/api/auth/login`, `/api/v1/auth/login`, `/auth/login`, `/login.json`, `/session`) with the stored credentials, never anything outside the community origin, guarded by `guard_public_url`. +- Two-factor (TOTP): a new column `communities.login_totp_secret`, handled exactly like the other credential columns (never returned by any endpoint, admin form, Devii tool or CLI; set through `devplace community login --login-totp`). `extract.totp(secret, now)` implements RFC 6238 with the stdlib. The login flow detects a follow-up one-time-code step (`otp`, `totp`, `code`, `token`, `2fa` input) and submits the current code. Tested against the RFC 6238 vectors. SMS and push 2FA cannot be automated and are reported as `second_factor_unsupported`. +- Captcha: detection of `g-recaptcha`, `h-captcha`, `cf-turnstile`, `captcha` form markers moves the community to `login_required` with the reason `captcha_required` and a plain message; it is never attempted. Recorded as unsupportable by design. +- Tests: fixtures for a JSON login (token and cookie), a TOTP step with the RFC vectors and a fake clock, a rejected code, the captcha stop, credential non-disclosure (every serializer and the CLI output are scanned for the secret). +- Status: [x] Result: TOTP (RFC 6238 vectors), JSON-API login, OTP form detection, captcha detected and reported (never solved); secret never returned; models, router, CLI, Devii, docs_api, templates plumbed; tests login.py, admin index test. + +### K-06 Structured-data learning for hostile markup (G-K6) + +- New inferrer, tried between the heuristics and the AI assistant: `StructuredDataInferrer` reads JSON-LD (`DiscussionForumPosting`, `Article`, `BlogPosting`, `Comment`, `ItemList`) and schema.org microdata (`itemtype`/`itemprop`) and OpenGraph, producing a feed type `json_ld` (runtime support: extract the list of `<script type="application/ld+json">` blobs, flatten `@graph`, then reuse the existing JSON path engine). Known software signatures (Discourse, Lemmy, WordPress `wp-json`, Mastodon, phpBB/XenForo markup) select a ready, fixture-tested spec instead of guessing. +- Each learned field is validated against the sample like the AI proposals (60 % rule). The failure message names which fields could not be learned, so the human edit (K-07) is targeted. +- Tests: fixtures per source type (JSON-LD, microdata, OpenGraph-only, a Discourse `.json` topic list, a WordPress `wp-json` posts feed), generated-collector parity, no regression on the existing lobste.rs fixtures. +- Status: [x] Result: json_ld feed type, detector, learner, runtime; unit tests structured.py. Discourse and WordPress ready-made specs not built (schema editor covers hostile markup). + +### K-07 Admin schema editor (G-K6) + +- `GET/POST /admin/communities/{slug}/schema` (administrator, audited `community.schema.update`): shows the learned `SCHEMA` JSON; a save validates structure with the same validator the learner uses, runs a verification collection on a small sample through the real collector (dry run, nothing pushed), and only then stores the schema and regenerates `collectors.py` (verify before write; the previous schema is kept as `schema_previous` for one-click restore). Devii `community_schema_get` / `community_schema_set` (set is in `CONFIRM_REQUIRED`). +- Tests: api (valid, invalid structure, failing verification keeps the old schema, restore, authz, audit), e2e (edit and save through the page). +- Status: [x] Result: /admin/communities/{slug}/schema GET/POST/restore with validate_schema and sample verification, schema_previous, audit keys, Devii tools, docs_api; tests schemacheck.py and api schema.py. + +### K-08 Collect attachments, votes and reactions where they can be attributed (G-K7) + +- Attachments: option `collect_attachments` in `sync_options` (default off, per community). The collector finds inline images and attachment links in post and comment bodies, downloads them through `Fetcher` (stealthii + SSRF guard) under per-file and per-run caps, and emits `attachments` records with `content_b64`. Failures are ledger warnings, never run failures. +- Votes and reactions: the learned spec may name optional `votes` / `reactions` list fields (per-voter entries with a user id or name and a direction/emoji). When the source exposes them, voters become synced users and the records flow through S-01. **When the source only publishes an aggregate count (almost every site), nothing is imported:** an anonymous total cannot be turned into votes without inventing voters, which the platform forbids by design (no purchasable or fabricated engagement). This is stated in the docs as intentional, not as a missing feature. +- Tests: fixtures (attachment download within caps, oversize skip, blocked address, per-voter JSON feed becomes votes with derived stars, aggregate-only feed produces zero vote records), ledger rows. +- Status: [x] Result: collector emits votes, reactions and attachments; alias mode drops voter-attributed rows; generator VERSION 2 auto-regenerates; unit tests engagement.py. + +### K-09 Cross-run duplicate detection (G-K8) + +- The collector `state` (the watermark JSON already persisted per community) gains `links`: a bounded map `canonical_link -> post origin_uid` (cap 20000, oldest dropped, trimmed on write). `add_post` consults it, so a twin story seen in a different feed in a later run is aliased to the first post instead of becoming a second post; comments attach to the survivor as they do within a run. +- Tests: two feeds, two runs, same canonical link: one post; map cap and trimming; same-feed posts with the same link are still never merged (existing safe rule); generated-collector parity. +- Status: [x] Result: state links capped at max_remembered_links (20000); unit tests dedupe.py. + +--- + +## Track SK: skills (`devplacepy/services/skills/`) + +### SK-01 Real network leg for the skill fetcher (G-SK1) + +- The SSRF guard refuses loopback, which is why the network leg was never exercised. This machine has a global IPv4 address, and a server bound to it passes the guard unmodified, so the test runs the real `guarded_async_client` over a real socket with no patching: `ThreadingHTTPServer` on `0.0.0.0:<free port>` serving a `SKILL.md` plus a bundled `references/` file, fetched through `fetch_skill` at `http://<global ip>:<port>/SKILL.md`. The test discovers the address (UDP connect trick, then `is_blocked_address`), and **skips with a stated reason** when the host has no global address, never passing silently. Also asserted over the same socket: a loopback URL is refused, a redirect to a loopback address is refused, an oversize body is cut at the limit, an HTML page is rejected. +- Status: [x] Result: real-socket network test in tests/unit/services/skills/fetcher.py through the host's global IPv4, unpatched guard; 36 pass; skips with a reason when no global IPv4. + +--- + +## Cross-cutting work + +### X-01 Documentation of record + +Rewrite the `Gaps` sections to state only what is still true, move closed gaps into "What exists today" with the measured numbers, update: `services/nodes/CLAUDE.md`, `services/sync/CLAUDE.md`, `services/community/CLAUDE.md`, `services/skills/CLAUDE.md`, `nodes/extension/README.md`, `nodes/client/README.md`, `README.md`, `plan.md` (node plan progress), the docs pages (`nodes.html`, `nodes-protocol.html` reads the spec, `privacy.html`, community and sync pages), `docs_api` groups, `audit/events.md`. The memory file `node-network-plan.md` is updated at the end. +- Status: [ ] Result: + +### X-02 Validation gates (all must be clean) + +1. `python -c "from devplacepy.main import app"` after every Python edit. +2. `ruff check` / `pyflakes` on every touched Python file. +3. `go vet ./... && go test ./...` in `nodes/client`; `node --test` in `nodes/extension`. +4. `grep` for the em dash character and its HTML entity in every touched file. +5. CSS braces and HTML tags balanced for touched templates; JS parses. +6. Fan-out check per route (HTML, JSON, Devii, docs). +7. Four-layer correctness procedure for the spendable/state-machine/concurrent items (votes and stars, the pending queue, counters flush, cache, redundancy sampling): property checks, stateful fuzz, real multi-process races from fresh rows, static analysis. +- Status: [ ] Result: + +### X-03 Full suite + +- `make test` (unit, api, e2e, serial). Every failure is fixed, not skipped. +- Status: [ ] Result: + +### X-04 Commit + +- Commits per track with the pre-commit hook bumping the patch version. No push (outward-facing, not requested). The owner is told about the dist build into `data/node_clients/` and the optional public-hostname idle test. +- Status: [ ] Result: + +--- + +## Cannot be closed by code (kept visible on purpose) + +| Gap | Why | What is built instead | What closes it | +|-----|-----|-----------------------|----------------| +| G-N8 legal decision | A platform-terms, donor-terms, ISP-terms and jurisdiction decision is a human act. | Nothing is enabled for any account; `nodes_enabled` is `0`; the kill switch is enforced within one tick. | The owner records the decision; then switches the service on. The decision must cover: Reddit and any future destination terms, residential donor terms of service, ISP terms on traffic relaying, the jurisdiction of donors and of the operator, and the adult-only rule. | +| G-N1 Reddit from a residential line | Needs a residential machine. | Per-class success is recorded and shown (N-10). | A residential donor runs the Go client or the extension for a while. | +| G-N7 24 h soak, public idle timeout | Time and a production write. | Local soak with its true duration, local idle measurement, proxy config quoted. | Run the same harness for 24 h; owner approves a one-off production connection. | +| G-K5 captcha | Defeating a captcha is circumventing the control it exists for. | Detected, reported as `captcha_required`. | A human logs in once, or the site offers an API. | +| G-E3 Firefox (if the build cannot load add-ons) | Depends on the available Firefox build. | Chromium proven for real. | A normal Firefox with `web-ext run`. | + +## Progress log + +(One line per finished item: date, item, evidence.) + +## Result summary + +(Filled in at the end: what is closed, what is not, measured numbers, test counts, commits.) diff --git a/happyplanting.md b/happyplanting.md new file mode 100644 index 00000000..51f060f6 --- /dev/null +++ b/happyplanting.md @@ -0,0 +1,124 @@ +# happyplanting: fixing Continuous Delivery lock-in + +Status: plan only. Nothing in this file is implemented. Written 2026-10-17 from a read of the code, not from a run of the game. + +## Problem (verified in code) + +Continuous Delivery (`autoreplant` Mastery upgrade) replants the crop that was just harvested, immediately, whenever it is affordable. + +- `store/actions.py::harvest` and `_auto_harvest` call `_try_autoreplant` after crediting. The harvest pays before the replant is checked, so "affordable" is true in practice and the plot is never empty. +- `plant()` refuses any plot that already holds a crop. There is no uproot, swap or cancel. +- `upgrade_mastery` is one-way: no refund, no toggle. Mastery survives Refactor. +- `_try_autoreplant` checks cost only. It ignores market saturation and crop locks. +- Only exits today: Refactor (clears plots, costs a prestige-scaled fee) or running below the planting cost (rare). + +## Goals + +1. A per-plot "stop after this harvest" toggle, so the player gets one free crop choice on that plot. +2. Do not replant into a bad market. + +## Part A: per-plot "stop after harvest" (ship this) + +### Design + +New column `game_plots.stop_after_harvest` (int, default 0, always read with `COALESCE`). Added to the existing plot column ensure-block in `database/schema.py` (next to `raided_fraction`, ~line 1508) with example value `0`. + +Semantics: one-shot, per plot. While set, the next harvest of that plot does not replant. The plot ends empty and the player plants whatever they want. + +### Why this is the race-free shape + +The flag is read by `_try_autoreplant` from its own fresh `_plot_at(...)` re-read, which already happens AFTER `clear_plot` wins. Rules that make that safe: + +- The toggle endpoint is one conditional UPDATE: `SET stop_after_harvest = :v WHERE crop_key != '' AND crop_key IS NOT NULL`, checked via `rowcount` (`conditional_update_row`). It cannot touch an already-cleared plot. +- So the toggle either lands before `clear_plot` (the post-clear re-read sees it, no replant) or after (the WHERE fails, loud "plot is empty" error). No window where the harvest uses a stale flag. +- `clear_plot` is NOT changed. A leftover `1` on an empty plot is harmless because every path that puts a crop on a plot writes `stop_after_harvest = 0` in the same claim UPDATE: `plant()` and `_try_autoreplant` (both already use `conditional_update_row` with a SET string), and the `prestige()` plot reset. +- Never reset the flag inside `clear_plot`: that would erase it before `_try_autoreplant` can read it. + +### Changes (fan-out, all four faces) + +| Layer | File | Change | +|---|---|---| +| Schema | `database/schema.py` | ensure `game_plots.stop_after_harvest` | +| Data | `services/game/store/actions.py` | `_try_autoreplant`: return early if `COALESCE(plot.stop_after_harvest, 0)`; add `stop_after_harvest = 0` to the claim SET in `plant()` and `_try_autoreplant`; add the column to the `prestige()` plot reset | +| Data | `services/game/store/` (new fn in `actions.py`) | `set_plot_stop(user, slot, stop: bool)`: owner only, requires `mastery_autoreplant` level > 0, requires a crop on the plot, `GameError` otherwise | +| Form | `models.py` | `GamePlotStopForm(slot: int ge=0 le=64, stop: bool)`, modeled on `GameSlotForm` | +| Route | `routers/game/index.py` | `POST /game/plot/stop`, `require_user`, via `_respond_action`, audit `game.plot.stop` | +| Audit | `services/audit/events.md` + `category_for` | add `game.plot.stop` (same category as `game.plant`) | +| Schema out | `schemas/game.py` | `GamePlotOut.stop_after_harvest: bool = False` (owner only, default safe); `serialize_plot` sets it | +| View | `templates/_game_grid.html` and `static/js/GameFarm.js::_plotHtml` | a small toggle button on growing/ready plots, shown only when the viewer is the owner and owns Continuous Delivery; label "Stop after harvest" / "Resume auto-replant". Server render and JS mirror must match (the file already warns about flash on divergence). Reuse the existing `data-game-action` form delegation, no new JS class | +| Agent | `services/devii/actions/catalog/game.py` | `game_plot_stop` Action (`requires_auth=True`, params `slot`, `stop`). Not confirm-gated: reversible and costs nothing | +| API docs | `docs_api/groups/game.py` | `endpoint()` entry with `sample_response` | +| Prose docs | `templates/docs/code-farm.html` | Continuous Delivery row: document the toggle and that Mastery has no refund | +| Notes | `services/game/CLAUDE.md`, `README.md` | document the flag and the "plant/replant claim resets it" invariant | + +### Gaps found in the second review (all must be in the change) + +- `prestige()` resets plots with an explicit dict in `_plots().update({...})` (actions.py ~611). It does not touch unknown columns, so the dict must add `"stop_after_harvest": 0` or a flag survives a Refactor. +- A stale `1` on an EMPTY plot (left after a no-replant harvest) must never be shown: `serialize_plot` gates `stop_after_harvest` on `crop_key` present AND viewer is owner AND `mastery_autoreplant` owned. +- Writers that put a crop on a plot are exactly two: `plant()` and `_try_autoreplant` (both `conditional_update_row` claims). Both SETs get `stop_after_harvest = 0`. `_create_plot` inserts need no change (column default via ensure-block, read with COALESCE). +- Existing tests that enumerate things and will fail until updated: `tests/unit/services/devii/actions/catalog.py` (game action name list), `locustfile.py` (route coverage, the locust-maintainer agent owns it, never run a load test), `events.md` (audit catalogue), `README.md` tool list (line ~223) and the `code-farm.html` endpoint table (~line 433). +- The toggle button is a new element in a dense plot card: needs CSS tokens only (no literals), must fit the smallest phone width, and needs a `screenshot` skill visual check in both server-rendered and JS-rendered forms. +- `notify_farm` already runs in `_respond_action`, so the live refresh needs no extra wiring. +- Interactions checked and unaffected: steal (never clears a plot), water, fertilize, CI Bot `_auto_harvest` (flag honoured the same way, via the post-clear re-read), Era/leaderboards, treasury/grant. + +### Not in scope for Part A + +No farm-wide off switch and no refund of Mastery points. The per-plot toggle is the requested fix. + +## Part B: do not replant into a bad market (REVISED after second review: do not build the original) + +### Measurement + +Computed with the real `economy.realizable_harvest_coins` at the saturation floor (`market_factor` 0.4), payout minus planting cost: + +| Prestige | shell | api | kernel | distsys | mlpipe | secfort | +|---|---|---|---|---|---|---| +| 0 | -1 | 0 | +240 | -480 | -3600 | -10800 | +| 50 | +54 | +1120 | +18240 | +56020 | +101400 | +229200 | +| 60 | +65 | +1344 | +21840 | +67320 | +122400 | +277200 | + +Continuous Delivery needs prestige 50 (`MASTERY_UNLOCK_PRESTIGE`), where `prestige_multiplier` is 13.5. At that point even a fully saturated crop pays 11x or more its planting cost, for every crop in the catalog. Consequences: + +- The "profit floor" (skip when payout <= cost) can never fire for a real owner. It is dead code. Drop it. +- A plain "skip when saturated" always makes the owner poorer: the idle plot earns 0, and a skipped replant also forfeits that cycle's XP, quest progress, weekly harvest count (grant eligibility, `harvests` and `fair_play` boards) and the market tick. Idle is never better than planting a saturated crop. +- So the original Part B would be collateral damage, not a fix. + +### What to do instead + +1. Ship Part A only. It already solves the stated problem: one free crop choice per plot, whenever the player wants it, with zero economic downside. +2. Optional, only if you still want saturation handling: an opt-in farm setting "when saturated, replant the best unsaturated unlocked crop instead". It never idles a plot. It needs a new pure function in `economy.py` that picks the crop (must define "best", for example highest realizable coins per second, among crops passing `crop_lock_reason`), used by both the serializer preview and `_try_autoreplant` (one-quantity-one-function rule). It changes which crop grows and its grow time, so it needs its own design pass and the Layer 1/2 property sweeps. Not recommended to bundle with Part A. + +## Risk list + +- **Behavior change on deploy:** Part A defaults to 0 for every existing plot, so nothing changes until a player clicks. There is no Part B behavior change (see revision above). +- **Missing column on first plant:** the claim SET names `stop_after_harvest`, so a DB where `init_db` has not added the column would fail every plant. `init_db` runs before serving, and tests start from a fresh DB through it; verify with the layer-3 setup that uses a genuinely fresh, un-pre-seeded farm. +- **NULL trap:** pre-existing plot rows will have `stop_after_harvest` NULL. Every read and every WHERE uses `COALESCE(stop_after_harvest, 0)`. +- **Prestige reset:** forget the column there and a stale `1` survives a Refactor. Covered by the claim-SET rule, but the reset dict must also write 0. +- **Visitors:** the flag must never be exposed to non-owners (`serialize_plot` gates it on `viewer == owner`). +- **JSON vs template:** a context key not on `GamePlotOut` is silently dropped from JSON. Add the field to the schema first. + +## Verification plan + +Persisted tests (matching tiers; the suite is run by the orchestrating session, serial): + +- `tests/api/game/mutations.py`: toggle on, harvest, plot stays empty; toggle off, harvest, plot replanted; toggle on an empty plot is a 400; toggle without Continuous Delivery is a 400; a visitor cannot toggle; `plant` after a stop resets the flag; Refactor resets it. +- `tests/api/game/farm.py`: visitor view of an owner's plots never shows `stop_after_harvest`. +- `tests/e2e/game/index.py`: the toggle button renders and flips on click; server-rendered and JS-rendered markup agree. + +Disposable scripts (scratch DB via `DEVPLACE_DATABASE_URL` and `DEVPLACE_DATA_DIR`, never production): + +- Layer 3 race: real OS processes firing toggle against harvest on the same plot, from a genuinely fresh farm (flag NULL, not pre-seeded to 0). Assert the replant happens exactly when the toggle landed after `clear_plot`, never when it landed before. +- Layer 4: `pyflakes`/`ruff check` on every touched file. + +Then the standard gates: clean `from devplacepy.main import app` import, em-dash grep on touched files, `make test` (all tiers) green, commit, and verify live on `127.0.0.1:10500`. + +## Residual risk after the second review + +- Part A is a small, additive change with one new column, one new route and one guarded early return. I found no economic, concurrency or permission collateral in it. "No collateral damage" is a claim about the plan, not a measurement: it is proven only by the race script and the full suite above, neither of which has been run. +- One accepted behavior: toggling "stop" on a plot that harvests in the same instant either stops the replant or returns "plot is empty". Both outcomes are correct and visible. + +## Decisions needed from you + +1. Part A: one-shot (recommended, flag cleared when the plot is next planted) or sticky until turned off? +2. Part B: drop it (recommended), or commission the "replant best unsaturated crop" option as a separate piece of work? +3. Fold in the crop-lock check fix in `_try_autoreplant` (docs promise it, code lacks it)? diff --git a/locustfile.py b/locustfile.py index 264d2494..66fb314c 100644 --- a/locustfile.py +++ b/locustfile.py @@ -1716,6 +1716,14 @@ class DevPlaceUser(HttpUser): name="game/plant", ) + @task(1) + def game_plot_stop(self): + self.client.post( + "/game/plot/stop", + data={"slot": random.randint(0, 3), "stop": random.choice(["true", "false"])}, + name="game/plot/stop", + ) + @task(2) def game_harvest(self): self.client.post( diff --git a/nicknames.md b/nicknames.md new file mode 100644 index 00000000..786cc7f7 --- /dev/null +++ b/nicknames.md @@ -0,0 +1,101 @@ +# How to Generate Unique, Realistic Usernames for a Forum + +Accounts often have to be created for people who never picked a handle: bulk imports, migrations, data feeds. The source rarely carries a clean name. You get strings like `abner_page_by_abnercoimbre` or `via_rberger_2026-10-15_12_36_38_`. This article describes a method that turns such strings into names a person would actually pick, and keeps them unique and stable. + +## The problem + +Raw author strings often have the shape `<site> via <person>` or `<site> by <person>`: + +``` +lore.kernel.org via thang +jakearchibald.com by jaffathecake +decomp.dev via FedericoSchonborn +``` + +Using that string as the username gives ugly names. Worse, it gives the same person several accounts, because the same person appears next to different sites and each string looks like a different author. + +Fix the identity first, then the name. + +## Step 1: Decide who the person is + +Before you generate anything, define the identity key, because a handle is only stable if the key is. + +- Strip noise from the raw string: timestamps (`2026-10-15 12:36:38`) and relation words. +- If the string says `by` or `via`, the **person is the token after it**. The site comes before it. +- If the string is only a domain (`costa_jacobpalm_dk`), drop the top-level domain and keep the last meaningful label (`jacobpalm`). +- Use that cleaned person as the identity key, not the original string. + +Now one person appearing next to ten sites is one account. + +## Step 2: Create the handle once and store it + +Generate a handle the first time you see an identity. Store the link between the identity and the account in a table with a unique constraint, for example `(source, identity) -> account`. + +Every later lookup finds the link and uses the stored name. **The generator is never consulted twice for the same person.** Renaming later is how you end up with handles that drift, so do not do it. + +Two guards matter: + +- Only choose the name when the account is created. Later updates keep the stored name. +- Never touch accounts that already exist. Backfilling old names is a separate decision with its own risks. + +## Step 3: Make the name look chosen + +Real handles are not random strings. One empirical study of usernames on Google profiles (arXiv 1101.5578, 2011, Latin-alphabet names only) found: + +- About 70% of usernames contain part of the user's real name. +- About 30% are plain first-plus-last concatenations. +- More than 18% add digits to the name, likely birth years or a number to get around a taken name. + +That is the only solid distribution found. It says nothing about separators or humorous suffixes, so those weights are judgment, not data. + +Pick one pattern per person with fixed weights: + +| Pattern | Weight | Example | +|---|---|---| +| Plain name | 40% | `runxiyu` | +| Name plus a number (year, two-digit year, or a small number) | 30% | `hongminhee90`, `untrusem_2000` | +| Separator plus number | 8% | `jaffathecak-73` | +| Funny (prefix, suffix, or a nerd number) | 17% | `federicoschonborn_irl`, `lcapald_500` | +| Light leetspeak | 5% | `wr1ter0` | + +The funny parts come from small word lists: prefixes like `its`, `dr`, `mr`; suffixes like `_dev`, `_irl`, `_ftw`; numbers like `42`, `1337`, `404`, `418`. Years run 1972 to 2006. + +## Step 4: Make it deterministic + +Seed the random generator from a hash of the source and the identity, for example `sha256(source:identity)`. The same person then always produces the same ordered list of candidates, which makes the system reproducible and testable. + +If the first candidate is taken, move to the next. Finally fall back to a numbered suffix, so a free name always exists. + +## Step 5: Enforce hard limits + +Constants, not heuristics: + +- Characters: letters, digits, `_` and `-` only. +- Length: 3 to 32. +- Base name: trimmed to 18 characters before decoration, so a suffix always fits. +- Case: lowercase, as people usually type handles. +- A bounded number of re-rolls before the fallback. + +Test these with property checks across many inputs, including empty strings, very long strings, accented characters, and strings made only of separators. Every candidate must be valid. + +## Step 6: Handle the exceptions + +- **Deliberate pseudonyms.** If you replace names on purpose so the original is hidden, do not decorate those names. Mentions are rewritten to match the replacement, and any change would break the match. +- **Trusted senders.** If another system sends you clean handles, keep them. Apply this method only where the names cannot be trusted. +- **Collisions.** Compare names case-insensitively, since `Alice` and `alice` should not coexist. + +## Known weaknesses + +- The weights are constants, not tuned on real data from your own user base. +- A name that already ends in digits can look odd after a year is added (`spc476` becoming `spc4762005`). +- Changing the identity key means records created before the change no longer match their old links. They appear as new accounts, and the old ones remain as orphans. Plan a cleanup if that matters. +- The result is a plausible handle. It makes no claim about who the person is. + +## Summary + +1. Fix the identity first: extract the person, drop the site and the stamp. +2. Create the handle once, store the link, and never regenerate it. +3. Pick a pattern with weights based on how real people name themselves. +4. Seed the randomness from the identity so results are reproducible. +5. Enforce length, character and uniqueness rules. +6. Leave existing accounts and deliberate pseudonyms alone. diff --git a/nodes/client/README.md b/nodes/client/README.md index 75621cc7..b6410ae4 100644 --- a/nodes/client/README.md +++ b/nodes/client/README.md @@ -30,7 +30,9 @@ devplace-node version Create the device secret on your DevPlace profile after consenting to bandwidth sharing. `login` asks the server to accept the secret before storing it (`--no-verify` skips that). The secret can also come from `DEVPLACE_NODE_SECRET`. Every command takes `--config PATH`. -`run` is quiet: it logs start, disconnects, and one totals line per day. Add `-v` for detail. Exit code 3 means the server ended this device for good (revoked, suspended, kill switch, no consent, not adult, replaced by another instance of the same device); fix the cause and start it again. +`run` is quiet: it logs start, disconnects, and one totals line per day. Add `-v` for detail. Exit code 3 means the server ended this device for good (revoked, kill switch, no consent, not adult, replaced by another instance of the same device); fix the cause and start it again. A timed suspension is not final: on a ticket `403` or a `kick` with reason `suspended` the client waits until the `until` instant (at least 60 seconds, at most 24 hours, plus jitter; 5 minutes when `until` is empty or unreadable) and then asks for a new ticket. + +When the server setting `pause_on_battery` is on, jobs are refused with `on_battery` while the machine runs on battery (Linux `/sys/class/power_supply`, macOS `pmset -g batt`, Windows `GetSystemPowerStatus`; the result is cached for 15 seconds). An unknown power state counts as mains power. The device label the server holds is logged at connect with `-v`. ## Config file diff --git a/nodes/client/battery.go b/nodes/client/battery.go new file mode 100644 index 00000000..380eb627 --- /dev/null +++ b/nodes/client/battery.go @@ -0,0 +1,53 @@ +package main + +import ( + "strings" + "sync" + "time" +) + +const batteryCacheSeconds = 15 + +type BatteryProbe func() (bool, error) + +type batteryWatch struct { + mu sync.Mutex + probe BatteryProbe + now func() time.Time + debugf func(format string, args ...any) + checked time.Time + value bool + warned bool +} + +func newBatteryWatch(probe BatteryProbe, now func() time.Time) *batteryWatch { + return &batteryWatch{probe: probe, now: now} +} + +func (w *batteryWatch) OnBattery() bool { + w.mu.Lock() + defer w.mu.Unlock() + moment := w.now() + if !w.checked.IsZero() && moment.Sub(w.checked) < time.Duration(batteryCacheSeconds)*time.Second { + return w.value + } + w.checked = moment + value, err := w.probe() + if err != nil { + value = false + if !w.warned && w.debugf != nil { + w.debugf("battery state unknown, treating as mains power: %v", err) + } + w.warned = true + } + w.value = value + return value +} + +func parsePmset(text string) bool { + return strings.Contains(text, "Now drawing from 'Battery Power'") +} + +func windowsOnBattery(acLine, batteryFlag byte) bool { + return acLine == 0 && batteryFlag != 128 && batteryFlag != 255 +} diff --git a/nodes/client/battery_darwin.go b/nodes/client/battery_darwin.go new file mode 100644 index 00000000..09987722 --- /dev/null +++ b/nodes/client/battery_darwin.go @@ -0,0 +1,19 @@ +//go:build darwin + +package main + +import ( + "context" + "os/exec" + "time" +) + +func systemBattery() (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, "pmset", "-g", "batt").Output() + if err != nil { + return false, err + } + return parsePmset(string(out)), nil +} diff --git a/nodes/client/battery_linux.go b/nodes/client/battery_linux.go new file mode 100644 index 00000000..78351083 --- /dev/null +++ b/nodes/client/battery_linux.go @@ -0,0 +1,53 @@ +//go:build linux + +package main + +import ( + "os" + "path/filepath" + "strings" +) + +const powerSupplyRoot = "/sys/class/power_supply" + +func systemBattery() (bool, error) { + return sysfsOnBattery(powerSupplyRoot) +} + +func readSupplyField(dir, name string) string { + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +func isMainsType(kind string) bool { + return kind == "Mains" || kind == "ADP" || strings.HasPrefix(kind, "USB") +} + +func sysfsOnBattery(root string) (bool, error) { + entries, err := os.ReadDir(root) + if err != nil { + return false, err + } + discharging := false + for _, entry := range entries { + dir := filepath.Join(root, entry.Name()) + if readSupplyField(dir, "scope") == "Device" { + continue + } + kind := readSupplyField(dir, "type") + switch { + case kind == "Battery": + if readSupplyField(dir, "status") == "Discharging" { + discharging = true + } + case isMainsType(kind): + if readSupplyField(dir, "online") == "1" { + return false, nil + } + } + } + return discharging, nil +} diff --git a/nodes/client/battery_linux_test.go b/nodes/client/battery_linux_test.go new file mode 100644 index 00000000..1bb9d224 --- /dev/null +++ b/nodes/client/battery_linux_test.go @@ -0,0 +1,60 @@ +//go:build linux + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +type supply map[string]string + +func buildTree(t *testing.T, supplies map[string]supply) string { + t.Helper() + root := t.TempDir() + for name, fields := range supplies { + dir := filepath.Join(root, name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + for key, value := range fields { + if err := os.WriteFile(filepath.Join(dir, key), []byte(value+"\n"), 0o644); err != nil { + t.Fatal(err) + } + } + } + return root +} + +func TestSysfsOnBatteryTable(t *testing.T) { + cases := []struct { + name string + supplies map[string]supply + want bool + }{ + {"desktop without battery", map[string]supply{"AC": {"type": "Mains", "online": "1"}}, false}, + {"empty tree", map[string]supply{}, false}, + {"laptop discharging", map[string]supply{"BAT0": {"type": "Battery", "status": "Discharging"}, "AC": {"type": "Mains", "online": "0"}}, true}, + {"laptop charging", map[string]supply{"BAT0": {"type": "Battery", "status": "Charging"}, "AC": {"type": "Mains", "online": "1"}}, false}, + {"laptop full on mains", map[string]supply{"BAT0": {"type": "Battery", "status": "Full"}, "AC": {"type": "Mains", "online": "1"}}, false}, + {"discharging but usb-c online", map[string]supply{"BAT0": {"type": "Battery", "status": "Discharging"}, "ucsi": {"type": "USB", "online": "1"}}, false}, + {"discharging usb-c offline", map[string]supply{"BAT0": {"type": "Battery", "status": "Discharging"}, "ucsi": {"type": "USB_C", "online": "0"}}, true}, + {"adp supply online", map[string]supply{"BAT1": {"type": "Battery", "status": "Discharging"}, "ADP1": {"type": "ADP", "online": "1"}}, false}, + {"peripheral battery ignored", map[string]supply{"hid": {"type": "Battery", "status": "Discharging", "scope": "Device"}, "AC": {"type": "Mains", "online": "1"}}, false}, + {"peripheral only", map[string]supply{"hid": {"type": "Battery", "status": "Discharging", "scope": "Device"}}, false}, + {"unknown status", map[string]supply{"BAT0": {"type": "Battery", "status": "Unknown"}}, false}, + } + for _, item := range cases { + got, err := sysfsOnBattery(buildTree(t, item.supplies)) + if err != nil || got != item.want { + t.Errorf("%s: got %v err %v", item.name, got, err) + } + } +} + +func TestSysfsMissingRootIsAnError(t *testing.T) { + if _, err := sysfsOnBattery(filepath.Join(t.TempDir(), "nope")); err == nil { + t.Fatal("expected an error") + } +} diff --git a/nodes/client/battery_other.go b/nodes/client/battery_other.go new file mode 100644 index 00000000..dcb8f775 --- /dev/null +++ b/nodes/client/battery_other.go @@ -0,0 +1,9 @@ +//go:build !linux && !darwin && !windows + +package main + +import "errors" + +func systemBattery() (bool, error) { + return false, errors.New("battery state is not available on this platform") +} diff --git a/nodes/client/battery_windows.go b/nodes/client/battery_windows.go new file mode 100644 index 00000000..174f6cdb --- /dev/null +++ b/nodes/client/battery_windows.go @@ -0,0 +1,28 @@ +//go:build windows + +package main + +import ( + "syscall" + "unsafe" +) + +type systemPowerStatus struct { + ACLineStatus byte + BatteryFlag byte + BatteryLifePercent byte + SystemStatusFlag byte + BatteryLifeTime uint32 + BatteryFullLifeTime uint32 +} + +var getSystemPowerStatus = syscall.NewLazyDLL("kernel32.dll").NewProc("GetSystemPowerStatus") + +func systemBattery() (bool, error) { + var status systemPowerStatus + result, _, err := getSystemPowerStatus.Call(uintptr(unsafe.Pointer(&status))) + if result == 0 { + return false, err + } + return windowsOnBattery(status.ACLineStatus, status.BatteryFlag), nil +} diff --git a/nodes/client/limits.go b/nodes/client/limits.go index 3256b483..3bdca658 100644 --- a/nodes/client/limits.go +++ b/nodes/client/limits.go @@ -27,6 +27,7 @@ type Limiter struct { serverLimit ServerLimits serverSet ServerSettings hasServer bool + battery *batteryWatch day string used int64 active int @@ -38,7 +39,7 @@ func NewLimiter(cfg Config, statePath string, now func() time.Time) (*Limiter, e if err != nil { return nil, err } - limiter := &Limiter{cfg: cfg, loc: loc, now: now, statePath: statePath} + limiter := &Limiter{cfg: cfg, loc: loc, now: now, statePath: statePath, battery: newBatteryWatch(systemBattery, now)} limiter.load() return limiter, nil } @@ -165,6 +166,13 @@ func (l *Limiter) hoursCode() string { return "" } +func (l *Limiter) SetBatteryProbe(probe BatteryProbe, debugf func(format string, args ...any)) { + l.mu.Lock() + defer l.mu.Unlock() + l.battery = newBatteryWatch(probe, l.now) + l.battery.debugf = debugf +} + func (l *Limiter) Acquire() string { l.mu.Lock() defer l.mu.Unlock() @@ -172,6 +180,9 @@ func (l *Limiter) Acquire() string { if code := l.hoursCode(); code != "" { return code } + if l.serverSet.PauseOnBattery && l.battery.OnBattery() { + return RefuseBattery + } if l.used >= l.capBytes() { return RefuseBandwidth } diff --git a/nodes/client/limits_test.go b/nodes/client/limits_test.go index 1f2264eb..6b385a5a 100644 --- a/nodes/client/limits_test.go +++ b/nodes/client/limits_test.go @@ -1,6 +1,7 @@ package main import ( + "errors" "path/filepath" "testing" "time" @@ -160,3 +161,93 @@ func TestUsagePersistsAcrossRestart(t *testing.T) { t.Fatalf("read usage %d", got) } } + +func TestAcquireRefusesOnBatteryOnlyWhenAsked(t *testing.T) { + limiter, c := testLimiter(t, DefaultConfig(), "2026-01-15T12:00:00Z") + calls := 0 + onBattery := true + limiter.SetBatteryProbe(func() (bool, error) { calls++; return onBattery, nil }, nil) + if got := limiter.Acquire(); got != "" { + t.Fatalf("setting off must ignore the probe: %q", got) + } + limiter.Release() + if calls != 0 { + t.Fatalf("probe ran %d times with the setting off", calls) + } + limiter.ApplyServer(nil, &ServerSettings{PauseOnBattery: true}) + if got := limiter.Acquire(); got != RefuseBattery { + t.Fatalf("got %q", got) + } + onBattery = false + if got := limiter.Acquire(); got != RefuseBattery || calls != 1 { + t.Fatalf("cached result expected: %q calls %d", got, calls) + } + c.now = c.now.Add(16 * time.Second) + if got := limiter.Acquire(); got != "" || calls != 2 { + t.Fatalf("after cache expiry: %q calls %d", got, calls) + } +} + +func TestBatteryProbeErrorMeansMainsAndLogsOnce(t *testing.T) { + limiter, c := testLimiter(t, DefaultConfig(), "2026-01-15T12:00:00Z") + logged := 0 + limiter.SetBatteryProbe(func() (bool, error) { return true, errors.New("no sysfs") }, func(string, ...any) { logged++ }) + limiter.ApplyServer(nil, &ServerSettings{PauseOnBattery: true}) + for i := 0; i < 3; i++ { + if got := limiter.Acquire(); got != "" { + t.Fatalf("probe error must not refuse: %q", got) + } + limiter.Release() + c.now = c.now.Add(20 * time.Second) + } + if logged != 1 { + t.Fatalf("logged %d times", logged) + } +} + +func TestBatteryHelpers(t *testing.T) { + if !parsePmset("Now drawing from 'Battery Power'\n -InternalBattery-0") || parsePmset("Now drawing from 'AC Power'") { + t.Fatal("pmset parse") + } + cases := []struct { + ac, flag byte + want bool + }{{0, 1, true}, {0, 8, true}, {1, 1, false}, {0, 128, false}, {255, 255, false}, {0, 255, false}} + for _, item := range cases { + if got := windowsOnBattery(item.ac, item.flag); got != item.want { + t.Errorf("ac %d flag %d: %v", item.ac, item.flag, got) + } + } +} + +const ( + parityBase = 1767225600 + parityStep = 40009 + paritySamples = 200 +) + +var parityCases = []struct{ from, to, zone, bits string }{ + {"09:00", "17:00", "Europe/Amsterdam", "01010000001010101010000101010101000000101010100000010101010000001010101010000101010101000000101010100000010101010000001010101010000101010101000000101010100000010101010000001010101010000001010101010000"}, + {"22:00", "06:00", "America/New_York", "00010101010000001010101000000101010101000010101010100000010101010000001010101000000101010101000010101010100000010101010000001010101000000101010101000000101010101000010101010100000010101010000001010101"}, + {"08:30", "12:15", "Asia/Kolkata", "00000001010000000000101000000000010101000000000010100000000001010000000000101000000000000101000000000010100000000001010000000000101000000000000101000000000010100000000001010000000000101010000000000101"}, + {"00:00", "12:00", "Pacific/Auckland", "01101010101010110101010101010010101010101001010101010101101010101010110101010101010010101010101001010101010101101010101010110101010101010010101010101011010101010101101010101010100101010101010010101010"}, + {"10:00", "10:00", "UTC", "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"}, + {"09:00", "17:00", "Not/AZone", "01010000101010101000000101010100000010101010000001010101010000001010101000000101010100000010101010100001010101010000001010101000000101010100000010101010100001010101010000001010101000000101010100000010"}, + {"23:00", "01:00", "Australia/Lord_Howe", "00000000000010000000000001010000000000001000000000000100000000000010000000000001010000000000001000000000000100000000000010000000000000010000000000001000000000000100000000000010000000000000010000000000"}, + {"", "", "Europe/Amsterdam", "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"}, +} + +func TestInWindowParityWithPythonScheduler(t *testing.T) { + for _, item := range parityCases { + zone, err := time.LoadLocation(item.zone) + if err != nil { + zone = time.UTC + } + for index := 0; index < paritySamples; index++ { + moment := time.Unix(int64(parityBase+index*parityStep), 0).In(zone) + if got := InWindow(item.from, item.to, moment); got != (item.bits[index] == '1') { + t.Fatalf("%s-%s %s #%d: got %v", item.from, item.to, item.zone, index, got) + } + } + } +} diff --git a/nodes/client/main.go b/nodes/client/main.go index 93e4db33..0c672704 100644 --- a/nodes/client/main.go +++ b/nodes/client/main.go @@ -189,6 +189,7 @@ func runRun(args []string, stderr io.Writer) int { fmt.Fprintln(stderr, err) return exitError } + limiter.SetBatteryProbe(systemBattery, logger.Debugf) policy := NewPolicy() stats := NewStats(time.Now()) session := NewSession(config, policy, limiter, NewFetcher(policy), stats, logger) diff --git a/nodes/client/protocol.go b/nodes/client/protocol.go index 47541ed2..e23b3bc8 100644 --- a/nodes/client/protocol.go +++ b/nodes/client/protocol.go @@ -37,6 +37,12 @@ const ( MaxDonorMBPerDay = 20 * 1024 StableSessionSeconds = 30 FastRetryMillis = 1500 + + SuspendedReason = "suspended" + SuspendedMinSeconds = 60 + SuspendedMaxSeconds = 24 * 60 * 60 + SuspendedDefaultSeconds = 300 + SuspendedJitterSeconds = 30 ) const ( @@ -48,6 +54,7 @@ const ( RefuseBandwidth = "bandwidth_exhausted" RefuseConcurrency = "concurrency_exceeded" RefuseHours = "outside_hours" + RefuseBattery = "on_battery" RefuseTimeout = "timeout" RefuseBadURL = "bad_url" FailFetch = "fetch_failed" @@ -71,7 +78,6 @@ const ( var FatalReasons = map[string]bool{ "revoked": true, - "suspended": true, "kill_switch": true, "no_consent": true, "no_account": true, @@ -117,6 +123,7 @@ type ServerSettings struct { ActiveFrom string `json:"active_from"` ActiveTo string `json:"active_to"` Timezone string `json:"timezone"` + PauseOnBattery bool `json:"pause_on_battery"` } type Frame struct { @@ -131,6 +138,8 @@ type Frame struct { Port int `json:"port"` B64 string `json:"b64"` Reason string `json:"reason"` + Until string `json:"until"` + DeviceLabel string `json:"device_label"` DeviceUID string `json:"device_uid"` Limits *ServerLimits `json:"limits"` Destinations []Destination `json:"destinations"` diff --git a/nodes/client/session.go b/nodes/client/session.go index 5a29cc38..4203b94b 100644 --- a/nodes/client/session.go +++ b/nodes/client/session.go @@ -25,10 +25,11 @@ func (e *FatalError) Error() string { } type outcome struct { - fatal string - fast bool - stable bool - reason string + fatal string + fast bool + stable bool + reason string + holdUntil time.Time } type Session struct { @@ -40,6 +41,8 @@ type Session struct { log *Logger httpc *http.Client jitter func() float64 + now func() time.Time + sleep func(ctx context.Context, d time.Duration) dialWS func(ctx context.Context, rawURL string) (*Conn, error) limits ServerLimits maxBody int @@ -55,6 +58,8 @@ func NewSession(cfg Config, policy *Policy, limiter *Limiter, fetcher *Fetcher, log: logger, httpc: &http.Client{Timeout: time.Duration(TicketTimeoutSeconds) * time.Second, Transport: &http.Transport{Proxy: http.ProxyFromEnvironment}}, jitter: rand.Float64, + now: time.Now, + sleep: sleepContext, dialWS: Dial, limits: defaultLimits(), maxBody: DefaultMaxBodyBytes, @@ -101,6 +106,21 @@ func sleepContext(ctx context.Context, d time.Duration) { } } +func (s *Session) holdDelay(until time.Time) time.Duration { + wait := until.Sub(s.now()) + wait = max(wait, time.Duration(SuspendedMinSeconds)*time.Second) + wait = min(wait, time.Duration(SuspendedMaxSeconds)*time.Second) + return wait + time.Duration(s.jitter()*float64(SuspendedJitterSeconds)*float64(time.Second)) +} + +func (s *Session) suspension(until string) outcome { + end := s.now().Add(time.Duration(SuspendedDefaultSeconds) * time.Second) + if moment, err := time.Parse(time.RFC3339, until); err == nil { + end = moment + } + return outcome{reason: "device is suspended", holdUntil: end} +} + func (s *Session) Run(ctx context.Context) error { attempt := 0 for ctx.Err() == nil { @@ -115,6 +135,13 @@ func (s *Session) Run(ctx context.Context) error { attempt = 0 } var delay time.Duration + if !result.holdUntil.IsZero() { + delay = s.holdDelay(result.holdUntil) + attempt = 0 + s.log.Printf("device is suspended until %s, waiting %s", result.holdUntil.UTC().Format(time.RFC3339), delay.Round(time.Second)) + s.sleep(ctx, delay) + continue + } if result.fast { delay = time.Duration((0.5 + s.jitter()) * float64(FastRetryMillis) * float64(time.Millisecond)) } else { @@ -122,7 +149,7 @@ func (s *Session) Run(ctx context.Context) error { attempt++ } s.log.Printf("disconnected (%s), reconnecting in %s", result.reason, delay.Round(100*time.Millisecond)) - sleepContext(ctx, delay) + s.sleep(ctx, delay) } return nil } @@ -136,6 +163,7 @@ type errorAnswer struct { Error struct { Message string `json:"message"` Reason string `json:"reason"` + Until string `json:"until"` } `json:"error"` } @@ -166,6 +194,9 @@ func (s *Session) fetchTicket(ctx context.Context) (ticketAnswer, outcome) { var failure errorAnswer _ = json.Unmarshal(body, &failure) reason := failure.Error.Reason + if reason == SuspendedReason { + return answer, s.suspension(failure.Error.Until) + } if FatalReasons[reason] { return answer, outcome{fatal: reason} } @@ -292,13 +323,13 @@ func (s *Session) converse(ctx context.Context, conn *Conn) outcome { return s.protocolFailure(conn, "welcome is not valid json") } if welcome.Type == "kick" { - return s.kickOutcome(welcome.Reason) + return s.kickOutcome(welcome.Reason, welcome.Until) } if welcome.Type != "welcome" { return s.protocolFailure(conn, "expected welcome, got "+welcome.Type) } s.applyWelcome(welcome) - s.log.Debugf("connected as device %s with %d allowed destinations", welcome.DeviceUID, len(s.policy.Destinations())) + s.log.Debugf("connected as device %s (%s) with %d allowed destinations", welcome.DeviceUID, welcome.DeviceLabel, len(s.policy.Destinations())) var jobs sync.WaitGroup tunnels := NewTunnels(s.policy, s.limiter, s.stats, send, s.cfg.MaxTunnels) @@ -342,7 +373,7 @@ func (s *Session) converse(ctx context.Context, conn *Conn) outcome { case "settings_update": s.limiter.ApplyServer(nil, frame.Settings) case "kick": - return s.kickOutcome(frame.Reason) + return s.kickOutcome(frame.Reason, frame.Until) default: return s.protocolFailure(conn, "unexpected frame "+frame.Type) } @@ -355,7 +386,10 @@ func (s *Session) protocolFailure(conn *Conn, reason string) outcome { return outcome{reason: "protocol error: " + reason} } -func (s *Session) kickOutcome(reason string) outcome { +func (s *Session) kickOutcome(reason, until string) outcome { + if reason == SuspendedReason { + return s.suspension(until) + } if FatalReasons[reason] { return outcome{fatal: reason} } diff --git a/nodes/client/session_test.go b/nodes/client/session_test.go index 2447528b..e6f2d0a0 100644 --- a/nodes/client/session_test.go +++ b/nodes/client/session_test.go @@ -268,12 +268,12 @@ func TestSessionLimitsAnswerWithDonorCodes(t *testing.T) { _ = c.WriteClose(CloseGoingAway, "bye") return } - writeFrame(t, c, map[string]any{"type": "kick", "reason": "suspended"}) + writeFrame(t, c, map[string]any{"type": "kick", "reason": "revoked"}) }) h.session.limiter.AddBytes(1 << 40) err := h.run(t) var fatal *FatalError - if !errors.As(err, &fatal) || fatal.Reason != "suspended" { + if !errors.As(err, &fatal) || fatal.Reason != "revoked" { t.Fatalf("run returned %v", err) } } @@ -286,7 +286,8 @@ func TestFatalTicketStatuses(t *testing.T) { }{ {401, "", "unauthorized"}, {403, `{"error":{"status":403,"message":"x","reason":"no_consent"}}`, "no_consent"}, - {403, `{"error":{"status":403,"message":"x","reason":"suspended"}}`, "suspended"}, + {403, `{"error":{"status":403,"message":"x","reason":"revoked"}}`, "revoked"}, + {403, `{"error":{"status":403,"message":"x","reason":"kill_switch"}}`, "kill_switch"}, } for _, item := range cases { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -349,14 +350,120 @@ func TestBackoffStaysInsideBounds(t *testing.T) { } func TestFatalReasonsMatchPlan(t *testing.T) { - for _, reason := range []string{"revoked", "suspended", "kill_switch", "no_consent"} { + for _, reason := range []string{"revoked", "kill_switch", "no_consent"} { if !FatalReasons[reason] { t.Errorf("%s must stop reconnecting", reason) } } - for _, reason := range []string{"heartbeat", "ticket", "capacity"} { + for _, reason := range []string{"heartbeat", "ticket", "capacity", "suspended"} { if FatalReasons[reason] { t.Errorf("%s must reconnect", reason) } } } + +type sleepLog struct{ waits []time.Duration } + +func (l *sleepLog) sleep(_ context.Context, d time.Duration) { l.waits = append(l.waits, d) } + +func fixedNow(text string) func() time.Time { + moment, _ := time.Parse(time.RFC3339, text) + return func() time.Time { return moment } +} + +func TestSuspendedTicketWaitsUntilTheEndThenRetries(t *testing.T) { + cases := []struct { + name string + until string + want time.Duration + }{ + {"future", "2026-01-01T01:00:00Z", time.Hour}, + {"empty", "", 300 * time.Second}, + {"garbage", "tomorrow", 300 * time.Second}, + {"expired", "2025-12-31T00:00:00Z", 60 * time.Second}, + {"far", "2030-01-01T00:00:00Z", 24 * time.Hour}, + } + for _, item := range cases { + t.Run(item.name, func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusForbidden) + body, _ := json.Marshal(map[string]any{"error": map[string]any{"status": 403, "message": "x", "reason": "suspended", "until": item.until}}) + _, _ = w.Write(body) + return + } + w.WriteHeader(http.StatusUnauthorized) + })) + defer server.Close() + cfg := DefaultConfig() + cfg.Server, cfg.Secret = server.URL, testSecret + limiter, _ := NewLimiter(cfg, "", time.Now) + policy := testPolicy(publicAnswer) + session := NewSession(cfg, policy, limiter, NewFetcher(policy), NewStats(time.Now()), NewLogger(testWriter{t}, true)) + session.jitter = func() float64 { return 0 } + session.now = fixedNow("2026-01-01T00:00:00Z") + slept := &sleepLog{} + session.sleep = slept.sleep + var fatal *FatalError + if err := session.Run(context.Background()); !errors.As(err, &fatal) || calls.Load() != 2 { + t.Fatalf("err %v calls %d", err, calls.Load()) + } + if len(slept.waits) != 1 || slept.waits[0] != item.want { + t.Fatalf("waits %v want %v", slept.waits, item.want) + } + }) + } +} + +func TestSuspendedKickHoldsThenReconnects(t *testing.T) { + h := newHarness(t, func(n int, c *Conn) { + readFrame(t, c) + writeFrame(t, c, welcome()) + if n == 1 { + writeFrame(t, c, map[string]any{"type": "kick", "reason": "suspended", "until": "2026-01-01T00:10:00Z"}) + return + } + writeFrame(t, c, map[string]any{"type": "kick", "reason": "revoked"}) + }) + h.session.now = fixedNow("2026-01-01T00:00:00Z") + slept := &sleepLog{} + h.session.sleep = slept.sleep + var fatal *FatalError + if err := h.run(t); !errors.As(err, &fatal) || fatal.Reason != "revoked" { + t.Fatalf("run returned %v", err) + } + if h.conns.Load() != 2 || len(slept.waits) != 1 || slept.waits[0] != 10*time.Minute { + t.Fatalf("conns %d waits %v", h.conns.Load(), slept.waits) + } +} + +func TestSuspendedWaitHonoursCancel(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"error":{"status":403,"message":"x","reason":"suspended","until":""}}`)) + })) + defer server.Close() + cfg := DefaultConfig() + cfg.Server, cfg.Secret = server.URL, testSecret + limiter, _ := NewLimiter(cfg, "", time.Now) + policy := testPolicy(publicAnswer) + session := NewSession(cfg, policy, limiter, NewFetcher(policy), NewStats(time.Now()), NewLogger(testWriter{t}, true)) + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + started := time.Now() + if err := session.Run(ctx); err != nil || time.Since(started) > 5*time.Second { + t.Fatalf("err %v after %v", err, time.Since(started)) + } +} + +func TestWelcomeDecodesDeviceLabel(t *testing.T) { + frame, err := decodeFrame([]byte(`{"type":"welcome","device_uid":"d","device_label":"Laptop","settings":{"pause_on_battery":true}}`)) + if err != nil || frame.DeviceLabel != "Laptop" || !frame.Settings.PauseOnBattery { + t.Fatalf("%+v %v", frame, err) + } + kick, _ := decodeFrame([]byte(`{"type":"kick","reason":"suspended","until":"2026-01-01T00:00:00Z"}`)) + if kick.Until == "" { + t.Fatal("until not decoded") + } +} diff --git a/nodes/extension/README.md b/nodes/extension/README.md index a2675c31..55aa2221 100644 --- a/nodes/extension/README.md +++ b/nodes/extension/README.md @@ -9,7 +9,7 @@ A Manifest V3 extension for Chrome and Firefox from one source tree. It lets a D 3. On your DevPlace profile give consent to bandwidth sharing, register a device and copy the `dpn_...` secret (shown once). 4. Open the popup, enter the server URL and the secret, grant the host permission for that server when asked, then switch sharing on. -Packaged build: `./build.sh <outdir>` writes `devplace-node-extension.zip` and `devplace-node-extension.zip.sha256`. The zip holds `manifest.json`, `src/`, `ui/` and this file; tests are not shipped. +Packaged build: `./build.sh <outdir>` writes `devplace-node-extension.zip` and `devplace-node-extension.zip.sha256`. The zip holds `manifest.json`, `src/`, `ui/`, `icons/` and this file; tests are not shipped. ## Chrome and Firefox from one tree @@ -27,7 +27,7 @@ Both use the `chrome.*` namespace with promises. Firefox store submission additi - The background worker holds the WebSocket. It exchanges the device secret for a 30 second ticket (`POST /nodes/ticket`, `Authorization: Bearer dpn_...`) and opens `WS /nodes/ws?ticket=...`, then sends `hello` with `kind: "extension"` and `modes: ["fetch"]`. - A `pong` is sent every `ping_seconds` as a keepalive and for every server `ping`. A `chrome.alarms` alarm every minute is the reconnect backstop when the worker has been suspended. -- Reconnects use full-jitter backoff between `reconnect_min_seconds` and `reconnect_max_seconds`. Close `4013` retries at the minimum delay. A kick (`4403`), a consent or device refusal (`1008` with a reason) or a rejected secret (HTTP 401/403) **halts** and shows the reason; nothing hammers the server. +- Reconnects use full-jitter backoff between `reconnect_min_seconds` and `reconnect_max_seconds`. Close `4013` retries at the minimum delay. A kick (`4403`), a consent or device refusal (`1008` with a reason) or a rejected secret (HTTP 401/403) **halts** and shows the reason; nothing hammers the server. A timed suspension is the exception: a ticket `403` with reason `suspended`, or a `kick` with reason `suspended`, puts the client in the `SUSPENDED` state, shows the end time in the popup and retries at the `until` instant (clamped to 60 seconds to 24 hours, plus up to 30 seconds of jitter; 5 minutes when `until` is empty or unreadable). - State (connection, last error) is in `chrome.storage.session`. Config (server URL, secret, limits, disclosure flag) and the usage counters are in `chrome.storage.local`. - The secret is never logged and never shown after saving; the popup shows its first 8 characters only. @@ -40,7 +40,7 @@ Independent of anything the server says: - `fetch` with `credentials: "omit"`, `redirect: "manual"`, `cache: "no-store"`, no referrer; only `accept` and `accept-language` are forwarded from the job, so no cookie or authorization header can be injected - each redirect hop is re-checked against the allowlist, at most 5 hops; a redirect that leaves the allowlist is refused with `redirect_not_allowed` - response body capped at `max_body_bytes` and by the remaining daily budget, with `truncated: true`; `set-cookie` style response headers are dropped -- timeout by `AbortController`, daily bandwidth cap, concurrency cap and active hours from the local settings, each answered with its refusal code (`timeout`, `bandwidth_exhausted`, `concurrency_exceeded`, `outside_hours`). A server value can only lower these, never raise them +- timeout by `AbortController`, daily bandwidth cap, concurrency cap and active hours from the local settings (the active hours must be open in the donor's own browser clock AND in the server window evaluated in the server's timezone, an unknown zone falling back to UTC), each answered with its refusal code (`timeout`, `bandwidth_exhausted`, `concurrency_exceeded`, `outside_hours`). A server value can only lower these, never raise them Known limit: browsers hide the target of a `redirect: "manual"` response (an opaque redirect), so the extension cannot read where it points. It fails closed with `redirect_not_allowed`. A followable redirect is only handled when the response exposes its `Location`. @@ -61,10 +61,14 @@ Shared: the URL of the requested public page, the page content that comes back, ## Tests -`node --test nodes/extension/test/*.mjs` (no dependencies). They cover Policy, Limits, Fetcher with a stubbed `fetch`, WsClient with a fake WebSocket, and a drift check of `src/spec.js` against `spec.py`. Not covered by automated tests: `background.js` and the popup, which need a real browser. +`node --test nodes/extension/test/*.mjs` (no dependencies). They cover Policy, Limits (including a server-timezone parity table generated with Python `zoneinfo`), the popup helpers, the packaged icons, WsClient suspension and device label, Fetcher with a stubbed `fetch`, WsClient with a fake WebSocket, and a drift check of `src/spec.js` against `spec.py`. Not covered by automated tests: `background.js` and the popup, which need a real browser. ## Gaps -- No icons yet; the browser shows a default. -- The `welcome` frame carries no device label, so the label in the popup is local only. -- Not tested in a real Chrome or Firefox session in this change. +- Battery-aware pause is not implemented in the extension (browsers expose no reliable power state to a background worker); the Go client honours `pause_on_battery`. +- Verified in real browsers on 2026-10-16: Chromium through `tests/e2e/nodes/extension.py` (disclosure, connect, online, a fetch-mode job answered 403 via `fetch`, suspend, disable) and Firefox 158 driven by geckodriver and selenium (same flow). The Firefox script is not in the repository. Method: start a scratch DevPlace server with a donor account, install the extension as a temporary add-on, open the popup from a chrome-context tab (`--remote-allow-system-access`), and watch the device go online in `/admin/nodes`. +- `manifest.json` sets `content_security_policy.extension_pages` to `script-src 'self'; object-src 'self'`. Firefox's default MV3 policy includes `upgrade-insecure-requests`, which rewrites `ws://127.0.0.1` to `wss://` and breaks a loopback server; a public `wss://` server is unaffected. + +## Icons + +`icons/icon-16|32|48|128.png` are generated by `python3 tools/make_icons.py` (Pillow), a node glyph on a rounded dark-blue square. The popup shows the server's device label from `welcome` and falls back to the label typed locally before the first connect. diff --git a/nodes/extension/build.sh b/nodes/extension/build.sh index b56ece03..02f871fe 100755 --- a/nodes/extension/build.sh +++ b/nodes/extension/build.sh @@ -15,7 +15,7 @@ rm -f "$archive" cd "$here" if command -v zip >/dev/null 2>&1; then - zip -q -X -r "$archive" manifest.json src ui README.md + zip -q -X -r "$archive" manifest.json src ui icons README.md else python3 - "$archive" <<'PY' import sys @@ -24,7 +24,7 @@ from pathlib import Path archive = sys.argv[1] paths = [Path("manifest.json"), Path("README.md")] -for folder in ("src", "ui"): +for folder in ("src", "ui", "icons"): paths.extend(sorted(item for item in Path(folder).rglob("*") if item.is_file())) with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as bundle: for path in paths: diff --git a/nodes/extension/icons/icon-128.png b/nodes/extension/icons/icon-128.png new file mode 100644 index 0000000000000000000000000000000000000000..41e45b7a18d6745db4251fe7af0ca6c178cec636 GIT binary patch literal 6715 zcmZ{JWl$ST)OCPRqy#To2vQt^1_{BP;slBnr&tRVcXwzhu0@JAMT$#t_ux`m+$rw< z@_h6Cd4Ie=c6ZL%*`1v`cjlhGCrU$20fbA13jhE>N{X^tPix?R1N+(2+$C-{1OQN> zE6GYBJb&+h^-6nd75^x4-nw<rV$F!d%Sj&;{tAeWZcYWfW{V8u=i;x*XXyTDMe9G< z4@881^pZrF5?Y-{$h2cpd|{H#f0iV`i$*nIH#cj1u)p9TvekTkyY|PSlUNb<H?6ht z9De_5=d8s~eY!rS9=XVFwx?)gU+wA&3|UHRKBU|+XG`NyBhbLPluQ5KiOALeLdpvD zwoYL;0C>xK;hQCw>?0TnF4sFZ-ZzvBpXGn}KubeIZ`gsLyx9A_0E!`x0Myj=Whw<I z%8KW0=e3ucH*dF0=dqS(kvxz^$^oS$Rl+_6q5mM#XV~?h{OY^v4-@qR{9fRSA^)|) z<4FOqsVBq{=ilm%`e>~#y}@t(?;sM5JeLYxc3CpNm{NSM?hVrB6CUn#v>uOf0fNds zD)9$fYQRj(Ci%S3&^uQEVR}{|C0MDZn+1Zf7$qWEcsdPnqQlqZ2QE;35@3Ewb*$IT zPeA~>KWBHgJzU#^M0_??lxM^P1j(V3Hx#v~*^xBA?(y{baoQ1vCat>tNXWC?tw%s2 zBNLDg^Kd^*oRz?i!87`)FFxCP%WYtpE0ZT<cmZ>ZH?4RPPb0vIl^YycmUczeQvWfS z&b(d3h%oxxT~?S$#J`-(<l3x{+DhpCbHeMS6iOC>hER{h%40?-XJbEfTEL$=hC&+1 zD9A>Y;&>_$(8=*O=5~n-^2LMXA#_ID9b5K*Y;R;dmy~XGx}gQsB?%?urqtfzzRaS% zR-Sd^ZY9AUE*g1Qb-FuXQMmxUX79YT?*wCPDa5BelQW4GLJ4tFw%y_+GOlp*Wj+U> zQ3&mw-HdsAP?Ok~+4z_CF7cAkb_=j#_I4Y{to?=Bjz*=c>ieFOt<=}*H?)TtJ5|rk z{mYmKSJNE{uwJkBY^7kFzHVrjtMX&SUi8GZajgI2Wj#0Ly~;rH+D|^@o5|!IF9<#6 zxo=eAg<|YN$ZFNXVR6gfiRnT|5PH6=`CRj?5*F^FmJWHKfgNRYC76OCFY-8PIXAn{ z(bGm_h8lxT`K-p>OEtm3A$AsNM#zwv$U-rMtHt)FL{}6jU777Rdbq1vbW!BuES$y~ zZ5|lC<Tx><-^*+OJqxiTZ3WCwl|oB`!N4o2z}>y23S}1-J<xsooTa_C*>dcsRkZME z0Z>_vLme?ZtY3I)rzkGcBC!+-lv0G40CK;J%lP?G1sXX7E*jAW-l*WBszdXB2=Xrz zav%@nv~fKSjZb4<1}ZEtyzavciU`0J<TVa4pO(9yp{kmPr>kVF#p$;{idc&K3}>mi zxG)05#Qflmb{HNKKVyHTxF(Ej+S>FZD6zYjJI6c+23|!MW28>h^^-`x8@^J!tGx<k z>Gu95>$bJ4bV_Bhy+3rkBD3kzA;Q8_??we+0(UBp410em$#T<m73=zT_z>&^)|1z^ z5F2eoR-v22HOWi&m5K~@n%N`o?$$%_NvI#c!P=0ajvrp#p#pA%%WpcQdus(IQe8-2 zD-#B4JK_6+apej%w2IUmR!u&OrM+)MfoTn-U|&nW(P;wR*qOy4#_VEx-fkPP=U>u( zUe0!VpE~8mDF3p^f!psKj2x$vd|sV!%vPye(u8niLn#;|n~#Umwtpa;uY#P1iCnyn zli>m0>mPR~*o<leC7DR6NsKgDTboYs&2HERHe=n*_D37WbYzaKV|g@K$6+ookNcB_ zNQzD#;-k$kF448o$hG{(6Ht@x2fDAy%SJkZ?PjI)cn^z-LOyAmNtqEmG~~Lf>)Z@L zsrGqmzl2_If+g$fFbQnBrHEedxN6kRD=9*}4Ll@0XJ=S+|80i0Lw1K#Zdw`|lHoTm zvF#)6ul?>kaF&c<vFBpS+V4}m`7GyctgWAY4x+potY%0opqk3Csqk0{PaD#iR-Tn? z=wCgGGlN<z*y-w>D0cFueBAv-x?H86yoy_46e7)0Hl32KO}A$f2c5O-muiLOd0np~ z2YStcV9SZeciQnaLaU<UpE6U`CNvkSnzEtcH>xS^_1J0v&^e{-tEMFlrambhIWw#f z#n-^VU8Rs8(v=9>%1e%vL4<ybBD>#LuA<-N_DZg`(a^mdnksBV3st~OX@FoAr;ZN0 z7EmeO?pOK76j5Bywozq95V{`!U1AwDJP6&jTu&8F)@3{WBC@1u#$hW-1vYTfr+d0R zcs^A~sjJC?N8@Je>AK;aB2ky20@LS;$`xY5M1F}S8uV|sdh>`(08FgdMi4=M-MmS< zH;o}yf&%VB*zjmRfhZ3eF3rlDA1+nws;>Nh>IY@9U-`EeFb^fqK4yy^_@(tCIY5r< zn_47W*(S;A;R%s_4fPh3+Qn)c%YKUrd3aKU_%8w%Ij?x|Drb1U4><j;pxs<TwFSGf zD?v@{E6f2rueKKsjFrd9@v|fnQ{>}#NTn~mH)USHpO1jT6TnX6asDGhkk)WL&T!sv zI|s_7O)>TI-Ov4slgW#%aGyhH2+0D$MoHHasv$+bQ`w}nR@p=l^`Xsh5EPSoG}|{j zF_rfHW$KS#W;IBz_TD5V7O(4RV~4hL{|x3DCs{*|>tKwqqGw%Y9(U>7zB{+;gW~6I zJIL6p`b;_}?I1H)a=Sg!XJMi|lF$*!_*EJ4p}qbshAEH2ky<B6rv9(ZgF8+j)igjO zp9)}yz>xf82Pn}50iZ~<TzM#9>VNUx#6)oWrMedTqen|oYM<t}fOd)w1DVEuYaZ5= ztVxSw%UqJ*|7q_IXhCl0249@)2O(R*?FO&&G`>lgSl-kqy}UmrIEjk$3&dG7#QutK zxWhh9i||faZ1vCfS%D(x<0Ier$tX^{1LgKj$0lb{|HRzQR%{iH$Ga}9BaYdktyoEy z=7>`wWnehmtp>~GOaUt=_7ACj*+vR37Bt9dQRQ-WG&%-hxAjvwvh0hGqxUB)wsw(; zLo)Lo6kdBjv2~bSmErG7P?C;y!yb^9^r4EzPb$n&MB2%8B`|UxR^KIp>y@yatw;X{ z8|BtQ2{1HR&q8pX$TD_0D*>zlN3E@av%A&@M4e_66uuGh=`=8O{1D%d@C}Fuo7~BX z``mRjkCCkESI^5Z$O2I@fk)19Yt?-LgoF?K_Nn6<Ipt0GE6Cs)YmivWJtur$Pfg;% z)c7AxlvkJN?<j0u)QP!z|4tuxxO4JzVR1berJMSqB!i-zB2R;}aseXhj2a;~i|r7Z z-*st0n#ZnYtwpN)f1TdTW=by22-2LkK$8vC8n9mZ-ZwTy2nwf)D)_7@z=QnWVev$1 zuwUyWb0RaooA#KeQ;Mo@^mkpd%z{BtFB^@5>1PQPXj>)t-Lt7io+F6~5txAYRuIz# zDVKC%aWxSqM%a-&a4^VX^-A~y)`(TEMrQ^UppKF%%SpiTB398f-^_|VU^G(<BYw9L zV(<dsq^9(0^&g)fpOmmSHn#Mf)r}=_lqVPVpCZmc`8OSzRJ+(R@>VHvbfNSD(5d*y zJ|r`T-X7N6=yy9cw8AQN$2zb*WwMulv^m$=yIZ#8``>J)ncmopNy=6;A3V`ed@v#d ztH|yq5Vr<<@{b~uTo@-nQ^});WREAGB><>5Ukk~o)N?^w7-N&nYlZaedq{LI<?OLg zg6Aj7kO{HWg7BQJdJnEbNFp-h+gKV_U~M=L-p&x{=TdXJr>+8%v<I?VO!#cMaUCl< z$rp_Jd_UrCQ3sX;!ch~cBZ<mPSEG%^A6I78Uy;2g1oC~8%7|Rf`l8Z%y_J7y{d;Nv zUh_}Mc<+EQQTg*u!B`qF5c;fG*+cy^Cf4Xi#!%}ZuflLXcX2YVsldL>c$x9ZvXKDv zl^t1-bjQcBR?&~^D_B3>yI#XD*1cCh5}>Lr_=I>i2RpK@2hIryV|s*dO9NIpazx2^ z$CeI{n)@$*NL#ezN}53I&=#~}2wUQ`OssjHRW)w=eIg2Ok5{9HKYwH7L%!5_mWoFj zEih$`9sKhhn*3G?d6}DE8OzWgF^{KLjc#_Y;WG&3Uin+`+iWX#gFd-+CeL!4S<Uiu z&N*5u`@a&1_uwY_!EQ!TmQJ}ucX#$ECfPRu`Hi0yg{X`6%I>&Kt|Ew_t)Y(KN<*y$ z_++$|d4;>OYo>FRKmoA@GX$i{Nkif$ur=ZGtHrj;$(rM{v&OtC$L*vP=B}ZNch?g_ z{)0?!76<=cN=Q4FcXu2veNAB|kcbQ<#8Ke*+ZsRiHAZ0Z6i*c-ou{|ie7M+M&)pdB zBMS4nb3~mycvYvtM7kt9KxOHBV5fh=g;8s9wq2*@cgLu>wOh1-^|=!A=3)S06vRf) z;tMWD8Sl7wSN(WeiO=kjfqIJmFSWJ7L%(=_b@8kOM>aAVUKC-tgQ89SeR<+!n=4L$ z=M}F(232IAg)6)!!G~5)jg!o}*fue3jF=^vv^i~ApPr0_SSb&}RgfU<Sl7-~&%F`z zc)aw-A0;CoXxVS6Rr=4Muj0Xjjf5rE`eE^q(KQ$>+x83bnXKGz{fjPsS{r#>xzW&5 zQh!lm$sf#XZIScg2R{&rMlB{%m^x`%<mnY@aLTnJC??uH|1c(gR@qvn5JDN?77`aa zw-4+`{HMqG%9-TU9ShDG2!WrA<YgR9;vFBBnf)(fekpca6lg`!o4hZxGR;-|{*T{c zi>-!6Ai}HK!zy)0+s3XGE4QAzuL*>zzN@Tt<$ODkagIui>{BdG3>_ENgxj+v53OS< zXC0}7R}D=>qoUUKwg;M&i;t-#P+_Te1QIRBYUaFgXVl_x9U=<;xw4Hf5r4Fjh6sZY zd{XR`;3{PcT0jUo>n+-^eZH`e3wx{0@<|(q(tN!OubcT)Ps?j<c#|_I%si?$92SfH zw?)J~i!bOf%ZGhm;ZR+Wp!M*Gy>-z*#zpMUZ3JYraoHMPX@|k3dfskGDTLi_UO6n8 z2Sgu`&9byIbE;fd;CM$m@f|pzif#JSt@)QwU%9C2#;?a~+t0A~_<r~CM_3YuobPP< z+a3KcS-1SR3})`{y?UWS%ZQz!i$rDSIqOlpi%g-I%YES}OSrtciNuc?@vaGlCcTye zuT|bd5a4AZ;OETP-%Ja&z49!-nU61Tg1lnEsgi=qWw>9s+bz_~x7sS&5_{^!6bVCX z0({ZPvwd(y)ocrjXF)i;*e^ln;aUS9kK-IRStQzMbQIn)pO+IgymzJV7NIe~vR^#A z_jtYx=SFuUgpCUCD30Flwj~U7Zx4mb^mTNktv8ZaWqBN*|G8VhkLCUE`W(?~#dn95 z{ON;L*;2-=N4xe+2al7^{!Tp=`t!9ZDMdG$UyWdPfS<i%nfqG>7mN+t)GU*K(&!Aw zlV-t<T2}nwuF8tN%e$Gh1&E95KS~ig<xX-L8(9qKT%gX0Q8(ZHh1sLADu_YGE=N?x zrF-)pooSPg%W!53_0l~`ee&Yr@AfFWS_gkH7?yw|^A`6>d-=mrC-drDx!YZmQaE21 zN<8RSqqZoic5{m>x@X(`#Dq9oS$1703b&|V0PyS{o+{r)!l`-zH^IiqUaOBb4mly! zZoaIV$${HG9{N*=s1_a)bCx=KByPaUg(rd^4;F>FkCRa}eQ!qL_$b`QhuN|qb^$@z zn5L>m#jO!$cS<>*=P!>N@^~>!L>}cKZJR>*?{MT_0%j<x?lCxpP4U~BE+kk}UXFfv zSlt|xBWW&!Cew|NPuO^xPse1Pj^uoF?8_Grn$fWXEc9wv+vDZAoSmJ!^kZ$Fm~4F= z756#lxA%j&D!Cn1J2=!kC;1@~aJpF3TE1X`jg@5HdUtbmM1nDNk{h-@o)<oj7T}yG zDL2V+{SJB1djTid<?GZD;#{dVxu*{>ycbx@pkA!T_IjPI58UNr)2;AJF+8|p96*F) zT>O0j?EI3~+>!D!y-P!>J50<Yr(<==Jytn?C{h=r>{HF7%=1lN#|u(tFQL*0U!9(6 zWx>p~g0R%14M!uXMx&xidcNxg^8{6>fdc;(Wt;P2vhVKj;dtr@=$Eu@8GWk9aHd!o z0auzrU4W_Vp=hiypd{2?f#$)#pvuEz?$9DMfgs@Slw@kQ_!#b}GJH(X#cK1xuK%e2 zq?k2%9tl);)tbOnpRePRq=4=2P5cwFXBm7rdR@gJ-B_xUsfF@CeaFkA@lu;cIQR>g zl`r5>|A|#^EH&(VZZhXGqc$~Ls9=O6j|X6oCJYytDztSdXTh#>pmB3$BVmqpj786> z3<F4SrSY^9S=2c+DJGePZf<&0?M?vAguk`mGajN_u4nKH7JJvLuH`zT&&^!H*Z~^T z%41r&6%`ZNWrntz=$;$Xt=q%!lqNC(_KW5dJCh#bLEWPlrD3hD{?6CZ8=j>LMVLpW zOqoNcQ@^dF`s%+4sdrRMt2?GYsVqjKI8fn-KJ>;x)xsy3UtAw+9TAYOp%c}YTs$NG zS}+xSEZ$AzftKfTcW@aH)myQ1w8WUGOMAViUAk}tAK(=%SJhEsUi}Zm-JP@X+0wY9 zD?qF0+HUVn<e6yT-vCY!80H_BM?T%MedfdzGp6@Rtah!(DNMp{7FAd#k-GCso<j+P zoYFMUB$8DO^lIxeeGW`?=dHKv-U3RbhpU@s5;qJb8iHhvG~5geTC5~2r4y}=_lFwR zvLkFtQLzT}mWWfd(Q&<;mCru3=1)6mW0q{UD`fgaItmOB-#sj$|7u~HwE`SRkp26z zD|^S&XN)AS@OnG=?*O@5N%h57V{+i<D6W(ZbvUZH?anW&NtH~sS4F-Zo96YKG4~_i zcPfsrQVAlI7Y>uu(bA?Y8?RVAoO){O$KlF7xtg{(+ByjXz)P_?B{m-4?PW0~=KOUv zz3=rL$r2ooYTY0%Q26b`)vIFPm0Q=#O+}_v3-&7?(P%H)dkoBj=0om(jkl2DbSSSK z%jsXkt7Ud**bPi(n55Nhr_9K8UiC@c$d_I4+T8;}tgo^c{qaqH5^VXd<fKb@Fz>ty z#VxODS)jtEdGA@2Y46Y!F;vll!7eqWCvflh!>U)7MuCA7er}O<QK7)&XD~EYt+2mo ziuo-1=ldS^*4uo;hQkg8TL&aHJ6jwZ08gwLYUzUEnca8S$S=H(;ga6C_gP?OR+Ie} zetS0ruPKee0{dsB8Xx3U((?;cR%h58rQcL}F{{3R*Ma8qarsX7sl>(0!sSiMb9C%; zHi9I|Oy{GYT0W98e%@1i;6o+t?Pb;MJIQ$G`M%TlNn=jYU|DhAL^a|VFP*lAd4rHc ztS5mgTs%(qnlUtw5nFoom?GVknDHcAul>>z@oza~%1SG(>Fmo=NQbH7l3S%qtp)|G zv{nE*gU08oASFu(EFRYwJQ9ocTb$JI7!~=GsqMc&4_XvAr(O>Go~?-mr5fd(W;G}Y zGQA~j{e~H0=TZ3XJGAJ%gfVMEqdRpFIY*El2(X|@N{smxZjI1;kqj(&7P#McXaCCO zt%a<<u0{U^6=px9e3m|P)MKhB+(F%P>KZ?}uxJ4f9=+?$t1J2TlaS8g<5voflJY}{ zP&tn)7vzUCc$JME3P7~n_0VS{ox%402)^XgC9V<@S$l`vn>C3Lm}Wy*^ipc8D^ljI z*C&dK&1cBPJD*kQ038$9G#Yz&om|T34__(&SWiB7Awv6_PyOHs&){+Jq*t=~!nM1T zB39yNRvHBJf{m9C6ZE2B6CbH^OT@8zb{$GS>TSF62_pm|X=`82UDnJ?up(z}9<1ep zp;K~iZjQbp_aCWln2Cu9cSh$VzVPc1B*z`<?%lMZ2*s}JZA23Kv(8lS-nCu4(!QM( z5ea=xN)dgNd;!TjWe0bqBX<cYvrG{Al@Na-ooOCY@z+~A;`?ty_`eHI9F;=~%SJfF zofjqtj;p7;6C#Li&={Z`JViYfLEQ<wofD0I>VF#2F5HJ!dt}H-7yn?>)zs?4V8NR+ ziH_vl$Go6uafV7NmcD``zvF}_)ON7bv_eu->*eYiN^N*izXv4JU>*@+dyD@sr#oh3 z`u}xsg<&(HIgz#$uFQ$YnB1;v5d53_PXWWtWvFFeucu&m?FGldwmhOzk2KEhjc5!z zJw^|!@&eADe|-aLhAh<E+!mGm?Ob(|-eVI5${&!DWEM+RzQd_I4^)Sea_`t~bDtvO z{0sM*$g!-h-MKv@EK#e3zxO%b&bDkf-l2o;7rI}22g=y*tbRBy!J;#sm~3x@I)un9 zs$LzMqdl{Y4|@Jma>%$Cf-V9k5;zNCRi~@{JP}a3P7_T+eTsm#H=cSomH)aLsahcY z&`VU2&o}20&(?g`g+)waEdi*ipX(--!gP}Y=2H<qpam23=y3>|A-dWmJ$+FmvyXhl zzJ0m>=9B^NfuXGGKU;-I9^gc=J}(M%-hY#mY!^Xvmel@y7Cuhm-EBdHiu^?F=FswO z>ry;hqWbDYRef@1vvQ?rgjSXZPpaiWo60%>IiDb`qa(p+Zb*ayk%R(Ee6h<)WlP?Q z&zV=yf+ELp2|h()t_8ueo`Z#)piGY$2hg^Dhw=e18USHDyXX+kr+j-rZO=9;hUlSr zeq)yOHh(c#>a~tniorXt(;hwnvQfm`Hb=jmY)WL)l55=3k<U{KD6;&2=R&XUCQ+9! z;{P&H5Fp^;=)`s%nOOTI!@FdLugvSHzZedf)a4xb_uC7W7j;Pw5^pxd7ne_Lb)QTj OfRdb=Y`L^)@c#f3oa&VT literal 0 HcmV?d00001 diff --git a/nodes/extension/icons/icon-16.png b/nodes/extension/icons/icon-16.png new file mode 100644 index 0000000000000000000000000000000000000000..c6059ce4c55ed6ad1ecc17e3a173a9e5577d3316 GIT binary patch literal 720 zcmV;>0x$iEP)<h;3K|Lk000e1NJLTq000mG000mO1^@s6AM^iV0007)Nkl<Zc-n1~ zO=wj|6vuyOX70UtNzC&Zja7p+x)4o4kQVHsNDEn21goMhBCQKiH=;Xn<3{MJ?V=#K zQkqQ#kyZsou!{&a3NEZD3Vtj^ZT#T*65q?acV>=@mlxy5e>XGy&N=^c&Imwd)$lf) zJ4y@#fB+&Q#25iE#-OSI3KB8qo_c?%dgqV7WZvrGtr&L-Ks9I^Zsr9YSqi{ZGbc-v znIK}EsA?Vw+f9UDwFQSZTzJod-ZE!S9O2B#BP{4GQ+SVv0Ek8(L`aSDn-h^GXaKMP zMua>sSk}Lgb!$H-0?Yar^7Q|gWJyA3Rbe#-1(v#&FNA4@80qfn<jS>M{QUb_I!hU& zW4D-F?j}t4WLg5Kn+>;RMDT@YbnHJqn_nRm1;z;P^Q|E6qZ^>`o=T;M9h<(TBTYDW z=_*CxF(T-P$z~s{F(hfi#AKa)KW$|Ahp*YP;Y;@ZxRKgqoit6b<~@X&3Pgn3WE~M9 z<OPE(2WB4E3@#`7z^m#60Nq`s_DGS+;7%10n$4W=zFA9Ss=?6uuXtU1Lo1ok%ySm? z&F8}9G14sKpV1p+S&FLStTnu@z2W<zb(}takO1sDaEy^ZFXGY!MDUF!+qVvJZsaf) z*u47}&R@Dpxu*-K3eH)^UsQSas?Oc}<D9+tFO}Xh&f3;!yGvZSe2pvDAF*;pALB2o zIA>9PXF(f{rt~lFquyvxo2Zi{?wwB&io(+~r<2Z72algTr&P*RRYfYRcRs+{0nr%p zqOisooV7FX>?ZpVh%u5TNl-DC5FQ$pxSyy6Vr-fu`YE{e(G+JbX_6>nEK05t_uHxG zH}7SAU)&b#8dT@CHv8l4Y{vnKh}_lS_f&6>{QVYq8X#GFVZQSK0000<MNUMnLSTYH CmQS_- literal 0 HcmV?d00001 diff --git a/nodes/extension/icons/icon-32.png b/nodes/extension/icons/icon-32.png new file mode 100644 index 0000000000000000000000000000000000000000..60dfcebf1564500f8f21c691aa9d3c0bb9d0933c GIT binary patch literal 1548 zcmV+n2J`ueP)<h;3K|Lk000e1NJLTq001BW001Be1^@s6b9#F8000HkNkl<Zc-pm> zU5H&*6~}*T@AGlby?5r$Og?H-H6%7A>Db9UOwr(jMJ=&`1Pi?&rYRw5>ytF}rC7x; z(uxDAd8jEVAp|>GaikxqkV3VzAqGkYYtyMyQfo1RiE)Pcyx(Vk_;6=RGNYM0I@wQ$ zv(MRUt^Zp8|5{rBC>Aw?gSP9=J)d?y{Eombyt@qu0O9{z9x#ad8(v;+==4bW!oa`g zZOZ&ZVe13mP&)X9NU+{}1KurpvPjc6@BPwlh(bhzcbBY9ADtc<c$s2R3qVKr1G|)p zPvITj8Pojo#*u5SNK*qq7zS9ol*8WRaiRi^D4eVBo_c@aRaw1d&pNC11w^mLS&N`s z!L?Mbvbw7P!1%;8`CQU!LM%#aydBrp_ULGB?+=xZzlKZgT;xl_UaL1a_{jYndGZnV zee)iYIN?v{hY7==g;b#M&Weun7Uy-cw)ZIb&QG>)1iMnqlq%f&)h}@DXAcoX3At>- z&bzz$^YA5xKlqp=6E9s8B7jHW76pD2-U*;ui9SsYecNuwct^RCQm&*J@95ihJ85b@ zDa<97YTERh6o_h-PtJSdIONT9@8VJ9l8{^y;!(VL?p@+I#I?eRYp9`?Lult5`CP); zx8LL8$DU<!dWOmA86JM@S<b%w9{F5?b8cB85iD0#ob$BhvkaU(#g4Ds1;D_`Q>@vr z4rA=HRVw(Oq|JMDd!8_8l18`ZTN9&|azH7CH3nmw5nE%L@2tD|4k#s*D>cF>qR^IY zDq$2+uGGlHA<iui3AJ3~m1>>M8*k;zEB$Qm+sgL7t(<wKpUoR@rBbb<l-wi-q{)IL z6LGZvAbmX>Fb0f)zMc&n?LSD8iJB~so8W*JMY&R^uXhVQcdTb@(xH@~lrT2w=(%G( zeZ5;KSL*22rDb^tXswu?tx%~Nt`$5Hs8kIzvlX;fD}#X63Z)cUDFUr=&QUE@S-q-2 zds{$0r*O{WoF|`C6xsqlwYHOLsfu$Btu<QBouU7GoH|*%?MMG<V`jFDF_v1rL8Vp) z;LsEMIP~~lo;h(E>om7*>>`eYbLT%|@YGqJ{O(@Xtm)$QGw)EYRH-*oQe&xB>%>vm z8Vv=gHBt`j*~xuhzl-6ESNOwQLp=4peI#+nt_Oe2*>e|Iy{41i&u;?YFT;Old~A}} z-WcNe;YYdqOLy}4PfoD&o-fk9c^xmkcAn=4UndiVEF`IPc0cH^g?(ze#QtyH%dw~S zQ#LT)EshWTk?;TXB(-{jjzXSBBc)PpB9l#Gq9|l)x<n=u)Bl45>@9v75pWK20et(3 z6P)<<X*vq|>jX<Jl31<Sy<<D2)G{?~n4YmrOc{QD>?P`HN=JK+G&LwCwB@t3<+CW& zY||a>IqHoD{m%?CHPifeYT8grExUJYM{Bi6vem+VZ;a*2)d}*U#sk@;S%69)jzY|w z_ue;k!8ynLnZ`QeC`15j9oeLLMn2SBxjKO{cG0|WJp#^?i6f3Y_aZ|hqpT_fq-ldg zPY-Z;bc`qnZrF6Z_e4R!<<T(?Jv~60Hds{%7#bPn$a61}i6i_1slKiT0F(%oTAlW` z9DO~X<MNdWKKSqw`CQU+#}g4s<tke?f0oaz>tg7_N6gHY$!6k31B-Na@3jl3BBcal z9F=OFDAZ&!@v;XEr4+SVoknWOCNV*v7B?#I(*V!MD79%08#O<4Xssx;W%1rG+xYzS zl!+sfjNrUqXxt%6!2LrZGAN>DA->*wtaHnp@fOdJ*15&0^WKXD0{l*;X?ob%^eQUQ zcq})2c`Pc>&ei{!rs-i-y7<C4;vW)Gh|(JG&D@1@r7e3rR+QEv3URW(bn%681w}pe z{`0@LrhcEtkBW|iW`0(9i5AhJ1}-^Q-8DJ#%hOQQ8vg1j7Ipot-;88$>He+rVg$Ot ycgz*#N?gzDh<u3WctfWLW`}?E4iq&E+J6K1`n26rVcQD;0000<MNUMnLSTY=+XUnQ literal 0 HcmV?d00001 diff --git a/nodes/extension/icons/icon-48.png b/nodes/extension/icons/icon-48.png new file mode 100644 index 0000000000000000000000000000000000000000..946814d269fe4b5bfdb1e0ae3994f6bb5d6c9354 GIT binary patch literal 2202 zcmV;L2xa$)P)<h;3K|Lk000e1NJLTq001xm001xu1^@s6R|5Hm000PKNkl<Zc-qaH zZH$!F6~}+)+~=9uncbb;fd#7pQ;ob6AlWL|no83|Ybg<%t{Pd<q%l@Z+O))4Yg@tA zwD^+xp~Vk^6-BgcF^Y}WMpCe;Z4-%618cEsL}1g!eH~`!<$1ZcA7&R3Sx{yc_k5bk zlic(F-*cXG?)l$GINsXY3>_UYz@$0ti=CQw2TKKaA&>%uvt!OdUPRs#$4+d#t-o{2 zzfZ)-#{i8y$joiO$P@2Ahr349^KlmEV$Qw^A_}578--3}J8|%n!OpD*jwd0<gKwVK zzFhD(i}A89E`uXRoPr3Si#dxEhhW4caW?EH4%!Dgx4txrSw??k=dWBQlKiuC7Hk|L z;#IweBfOn+wMeQ+)Od*CZ1jzRy-&VOTbmJZ(zM{lFKTSwKqP~;7Qs{tTtx`N2!Mp| zVV$d<h$BW-aSS;dEE?>5>H`IF66u3t5>41Rt`@wi5QY(hxnVMmX)=vz26MxNVT7tw z`&t-m9E<Ur5$^#ICzI#j_$BM~pWqbGYQYyv0aIJDti59;OIqgw@UK1lSo^?J9O)e( zRhPips#D5>I2F6Z)U_<STa3SWBz3f!w29-G#`-jW-1sxTwR9fp*HNFYW66T)TypV3 zcD(c&<sd{=t4$><#y8H|NS(OLM+j9FU!zJcU*NhGOSyExG!FMgM3Ezk9EbZNE?qE< z>sBl!moH$dB_K|~;>2C<0T(%I0nzF<#i1h1n{@$YYw?U8t3sYp%GNS()&-~tj%rg# zw1Sz7RFEW8U*-hCT1R(JFP<uK#Q^}ZwRoy@_w-_|Lu%5{9fBm48kx10bUI1Lj@|V1 zm&j%+rPDe`HsjILU!r5jZqn%__Be9YTI~opM=F_Qf7c;a-Tok5@AqI7j6&D@J*>L@ zLH2hYB9%;HYth(7BcGkOs&=aZs!D#iKvSlXSGM0zOE$x0%YVoRhyF)%QzLO)+jEst zW3$%MP@kq$3MiLDqSz6JA@%jCnx@XU5f~+5MyilVrNNB{_;Dj};sMS%#?8EO*_9I- zl`&d<_dQz?Fh&_J6d4%IAB)1^&@jV=BF5;%CZMVe<?<|=KZA|G{Q>8m(@Zj9c=%Vh z(z;+4gG2e6N1|qB)EH$bH_UgIU(Sa0-y@we<nmxGG-aSzw5-1K5&rbTZkjUn#Ic<? z5vbJuBCQK&^859xvDVN#5aV2B80{a3u~ylzel@KNXObT-)+`b=Sw#fGFyfZ>Wz?sH zQV`>NAI(LcXD9_R>6CCw`!d2Xs@arX69RE;X>3R{XXg2o15SEeHmZ~Zm^1Ty8XM9R zD}_eWNJUPq9F07#<%_J<r;m_Gc=Y%5vFELK@r^xc*R{^#8Oxh{-=n{$kA&|Xn-|U+ zffF4XqgZR{`EZacuKY5KTW4TY@jOEmS*&$f>xd$YXAGVyUs*hx6<06i!@eQnIQ~R$ z_^hP#)0Rl6yr`r9J<o77UqD6q<()V2+~&JjH2XYW`0Lx0$^lceN;)N^Q^M4&QZ9#V z+wnRxE}TNgqieYTo?A#H5)9>r@jP=<AD;?aYU;wZr&AhoM+<nK$K-4yg<`<be33$- z#Kntdv+3a<apTpC={}O@`#;~r+Vzj|+UuPd&&N6CNN<i;{{9v}z3(x8_vo|i-QP{? z!Z|EoI-jpz*2;hWdx*|=KOhJ~{NsG&hl|wJB~E*eI%OkM6>|9kH+=i++;;O-v}CjN z_7AY(4=<8!ZsguOSCUD>GcWGp7r)s|*MV*(H8(OmTp|b~8XHmo9L*Q;eUFCv6azy? znRf0J*8O-DH(j%Y;Rx0}u#F>q1Kf7gRkTdX(mOD~Bagqt)@OH;X-J<^h8dH<XlK3o z`YYM+>+ezwAq*jzs7%nlfq|hi_dfJAk8S-6o>9{2I-)2>RYqhMdu%P|D#vWk7>eaG z<#NcX>#ktk9V=;>oFb}Z4&?xRA5uQ7zH1YYKlAb_JNHQmh(K%|b%_Ljd+s65J10Y- zXh&XJhbn|ozzw&q=hc6_!Q_@~MXdRhg4HO&-rhkjy<|SyHr`FbPhf53SsYu^DW$t7 z$JbV@p%erdm6N8Ej#X{SLCExJQ)rpgL^%*V&mbZrgNkA_N&nDM8Zz~t34CSXspzB) zjcM}3MLaKAS#x9v@;pO15LzZRF@4$;%0W10b<(kJ7ktmq*Po+MEMwFj18A*7#S(-8 z#X<=YIirE4a!wKog%Uv!pz?7aMlFS6nZEuUzGuetp<_)`$%N0r!$;`Yv5UzKib!Q) z5t!VdY<qqe2M+a+O!#M{I*<A|aHxlE&!5~!$BtbbJbdJoKE^Ji&f)tWyI*^YhD<XH z=AKWg&R}iKre}7s_WmbGCjII(u^RX|Wf>Kz4C94jnb|X@GyS|44tDqQ&btRmr;-(^ zzRH2OS|6t!Xe!ErP%M=R!wBE^NYy2AwYadTfe+6)R}j(kSaBL%B$Z5%9$B>363LnQ z5S+6G73dOGz}eGDhq1Lal?Bh-hqI!JK$kjXXQdd)II1_bHgTXhWT$#owm2KboV`oz z>UDJxXQLRgk8AHcoA)?pw~I+C&P5X+zKg{q)j7L;aPO8k6`ZKCYg`--qQ>BeC-C0K zqMqk$(67d>fqPVGyH@+(e(Dg!*NcdtY9?ePDi__x;R-Hx*Hkcr($Nvq)@BCwKKZhX z;&0*@5U<V~Wjwyp>}gTk(TotU&U3gAtqYe8c0TbcZEXhD*@}3xZ=a*B&6IXOy+8Sd zxg9t;M_`s1->V4Zp9cp)#E7V;K;WG15F1}RxNpmwpZwTk)F$ZXHnFmG=<UsWMnD?) c=yRL@10BIr^c-hX{{R3007*qoM6N<$f{u+m4gdfE literal 0 HcmV?d00001 diff --git a/nodes/extension/manifest.json b/nodes/extension/manifest.json index d5d3a7ef..23a63550 100644 --- a/nodes/extension/manifest.json +++ b/nodes/extension/manifest.json @@ -8,9 +8,22 @@ "scripts": ["src/background.js"], "type": "module" }, + "icons": { + "16": "icons/icon-16.png", + "32": "icons/icon-32.png", + "48": "icons/icon-48.png", + "128": "icons/icon-128.png" + }, "action": { "default_title": "DevPlace Node", - "default_popup": "ui/popup.html" + "default_popup": "ui/popup.html", + "default_icon": { + "16": "icons/icon-16.png", + "32": "icons/icon-32.png" + } + }, + "content_security_policy": { + "extension_pages": "script-src 'self'; object-src 'self'" }, "permissions": ["storage", "alarms"], "host_permissions": [ diff --git a/nodes/extension/src/Limits.js b/nodes/extension/src/Limits.js index e93b05b3..79165479 100644 --- a/nodes/extension/src/Limits.js +++ b/nodes/extension/src/Limits.js @@ -27,6 +27,26 @@ function minutes(text) { return match ? Number(match[1]) * 60 + Number(match[2]) : null; } +function windowOpen(from, to, current) { + const start = minutes(from); + const end = minutes(to); + if (start === null || end === null || start === end) { + return true; + } + return start < end ? current >= start && current < end : current >= start || current < end; +} + +function zoneMinutes(moment, timeZone) { + let format; + try { + format = new Intl.DateTimeFormat("en-GB", { timeZone, hour: "2-digit", minute: "2-digit", hourCycle: "h23" }); + } catch (error) { + format = new Intl.DateTimeFormat("en-GB", { timeZone: "UTC", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }); + } + const parts = Object.fromEntries(format.formatToParts(moment).map((part) => [part.type, part.value])); + return Number(parts.hour) * 60 + Number(parts.minute); +} + export class Limits { constructor(options = {}) { this.baked = options.baked || LIMITS; @@ -39,6 +59,7 @@ export class Limits { this.reconnectMax = this.baked.reconnect_max_seconds; this.serverBandwidthMb = this.baked.max_donor_mb_per_day; this.serverConcurrent = this.baked.max_concurrent_per_node; + this.serverWindow = { from: "", to: "", timeZone: "UTC" }; this.local = Limits.normaliseLocal({}, this.baked); this.inflight = 0; if (options.local) { @@ -97,6 +118,11 @@ export class Limits { this.baked.max_concurrent_per_node, this.baked.max_concurrent_per_node, ); + this.serverWindow = { + from: typeof values.active_from === "string" ? values.active_from : "", + to: typeof values.active_to === "string" ? values.active_to : "", + timeZone: typeof values.timezone === "string" && values.timezone ? values.timezone : "UTC", + }; } get maxBodyBytes() { @@ -137,14 +163,12 @@ export class Limits { } inActiveHours() { - const from = minutes(this.local.activeFrom); - const to = minutes(this.local.activeTo); - if (from === null || to === null || from === to) { - return true; - } const now = this.clock(); - const current = now.getHours() * 60 + now.getMinutes(); - return from < to ? current >= from && current < to : current >= from || current < to; + if (!windowOpen(this.local.activeFrom, this.local.activeTo, now.getHours() * 60 + now.getMinutes())) { + return false; + } + const { from, to, timeZone } = this.serverWindow; + return windowOpen(from, to, zoneMinutes(now, timeZone)); } admit(usedBytes) { diff --git a/nodes/extension/src/State.js b/nodes/extension/src/State.js index 9d7a84bf..a7716dbf 100644 --- a/nodes/extension/src/State.js +++ b/nodes/extension/src/State.js @@ -6,13 +6,14 @@ export const STATES = Object.freeze({ ONLINE: "online", BACKOFF: "backoff", HALTED: "halted", + SUSPENDED: "suspended", }); export class State { constructor(area, clock = () => Date.now()) { this.area = area; this.clock = clock; - this.data = { state: STATES.OFF, detail: "", since: clock(), deviceUid: "", lastError: "" }; + this.data = { state: STATES.OFF, detail: "", since: clock(), deviceUid: "", deviceLabel: "", suspendedUntil: "", lastError: "" }; this.chain = Promise.resolve(); } diff --git a/nodes/extension/src/WsClient.js b/nodes/extension/src/WsClient.js index 4faec080..d4d5bc22 100644 --- a/nodes/extension/src/WsClient.js +++ b/nodes/extension/src/WsClient.js @@ -15,6 +15,7 @@ import { PROTOCOL_VERSION, REFUSALS, SERVER_FRAMES, + SUSPENSION, } from "./spec.js"; const OPEN = 1; @@ -35,6 +36,7 @@ export class WsClient { this.WebSocketImpl = options.WebSocketImpl || globalThis.WebSocket; this.timers = options.timers || globalThis; this.random = options.random || Math.random; + this.clock = options.clock || (() => Date.now()); this.clientVersion = options.clientVersion || "0.0.0"; this.platform = options.platform || "chrome"; this.baked = options.baked || DESTINATIONS; @@ -120,6 +122,37 @@ export class WsClient { await this.state.set({ state: STATES.HALTED, detail, lastError: detail }); } + suspensionDelaySeconds(until) { + const end = Date.parse(until); + const seconds = Number.isNaN(end) ? SUSPENSION.default_seconds : (end - this.clock()) / 1000; + const bounded = Math.min(SUSPENSION.max_seconds, Math.max(SUSPENSION.min_seconds, seconds)); + return bounded + this.random() * SUSPENSION.jitter_seconds; + } + + async suspend(until) { + if (!this.wanted) { + return; + } + const seconds = this.suspensionDelaySeconds(until); + this.welcomed = false; + this.clearTimers(); + this.generation += 1; + this.closeSocket(CLOSE_NORMAL, "suspended"); + this.retryTimer = this.timers.setTimeout(() => { + this.retryTimer = null; + this.ensure(); + }, seconds * 1000); + const ends = Number.isNaN(Date.parse(until)) + ? new Date(this.clock() + seconds * 1000).toISOString() + : new Date(until).toISOString(); + await this.state.set({ + state: STATES.SUSPENDED, + detail: "This device is suspended by the server", + suspendedUntil: ends, + retryInSeconds: Math.round(seconds), + }); + } + async fetchTicket() { const { serverUrl, secret } = this.config.credentials(); const response = await this.fetchImpl(serverUrl + TICKET_PATH, { @@ -129,14 +162,18 @@ export class WsClient { cache: "no-store", }); if (!response.ok) { - let reason = ""; + let detail = {}; try { - reason = (await response.json()).reason || ""; + const body = await response.json(); + detail = body && typeof body.error === "object" && body.error ? body.error : body || {}; } catch (error) { - reason = ""; + detail = {}; } + const reason = typeof detail.reason === "string" ? detail.reason : ""; const error = new Error(`ticket refused with ${response.status} ${reason}`.trim()); error.status = response.status; + error.reason = reason; + error.until = typeof detail.until === "string" ? detail.until : ""; throw error; } const body = await response.json(); @@ -164,6 +201,10 @@ export class WsClient { if (generation !== this.generation) { return; } + if (error.status === 403 && error.reason === SUSPENSION.reason) { + await this.suspend(error.until); + return; + } if (HALT_STATUSES.includes(error.status)) { await this.halt(error.message); return; @@ -276,6 +317,8 @@ export class WsClient { this.runJob(frame); } else if (frame.type === "settings_update") { this.limits.applyServerSettings(frame.settings); + } else if (frame.type === "kick" && frame.reason === SUSPENSION.reason) { + await this.suspend(typeof frame.until === "string" ? frame.until : ""); } else if (frame.type === "kick") { await this.halt(`kicked by the server: ${frame.reason}`); } else { @@ -303,6 +346,8 @@ export class WsClient { state: STATES.ONLINE, detail: notes, deviceUid: frame.device_uid, + deviceLabel: typeof frame.device_label === "string" ? frame.device_label : "", + suspendedUntil: "", lastError: notes, }); } @@ -344,6 +389,9 @@ export class WsClient { } classifyClose(code, reason) { + if (code === CLOSE_KICKED && reason === SUSPENSION.reason) { + return { action: "suspend", detail: "suspended by the server" }; + } if (code === CLOSE_KICKED) { return { action: "halt", detail: `closed by the server: ${reason || "kicked"}` }; } @@ -366,6 +414,10 @@ export class WsClient { return; } const verdict = this.classifyClose(code, reason); + if (verdict.action === "suspend") { + await this.suspend(""); + return; + } if (verdict.action === "halt") { await this.halt(verdict.detail); return; diff --git a/nodes/extension/src/spec.js b/nodes/extension/src/spec.js index 346bb8c8..7493342b 100644 --- a/nodes/extension/src/spec.js +++ b/nodes/extension/src/spec.js @@ -65,10 +65,19 @@ export const REFUSALS = Object.freeze({ BANDWIDTH_EXHAUSTED: "bandwidth_exhausted", CONCURRENCY_EXCEEDED: "concurrency_exceeded", OUTSIDE_HOURS: "outside_hours", + ON_BATTERY: "on_battery", BODY_TOO_LARGE: "body_too_large", TIMEOUT: "timeout", }); +export const SUSPENSION = Object.freeze({ + reason: "suspended", + min_seconds: 60, + max_seconds: 24 * 60 * 60, + default_seconds: 300, + jitter_seconds: 30, +}); + export const FETCH_FAILED = "fetch_failed"; export const SERVER_FRAMES = Object.freeze({ diff --git a/nodes/extension/test/limits.test.mjs b/nodes/extension/test/limits.test.mjs index 21440aba..046183a6 100644 --- a/nodes/extension/test/limits.test.mjs +++ b/nodes/extension/test/limits.test.mjs @@ -89,3 +89,70 @@ test("hello limits advertise the effective concurrency", () => { const limits = new Limits({ local: { maxConcurrent: 1 } }); assert.deepEqual(limits.helloLimits(), { max_concurrent: 1 }); }); + +function serverLimits(window, clock, local = {}) { + const limits = new Limits({ clock, local }); + limits.applyServerSettings({ active_from: window.from, active_to: window.to, timezone: window.zone }); + return limits; +} + +test("the server window is evaluated in the server timezone", () => { + const moment = () => new Date(Date.UTC(2026, 0, 15, 12, 0)); + assert.equal(serverLimits({ from: "09:00", to: "17:00", zone: "UTC" }, moment).inActiveHours(), true); + assert.equal(serverLimits({ from: "09:00", to: "17:00", zone: "Asia/Tokyo" }, moment).inActiveHours(), false); + assert.equal(serverLimits({ from: "20:00", to: "23:00", zone: "Asia/Tokyo" }, moment).inActiveHours(), true); + assert.equal(serverLimits({ from: "22:00", to: "06:00", zone: "America/New_York" }, moment).inActiveHours(), false); + assert.equal(serverLimits({ from: "21:00", to: "06:00", zone: "Asia/Tokyo" }, moment).inActiveHours(), true); +}); + +test("an invalid server timezone falls back to UTC and empty or equal bounds mean always", () => { + const moment = () => new Date(Date.UTC(2026, 0, 15, 12, 0)); + assert.equal(serverLimits({ from: "09:00", to: "17:00", zone: "Not/AZone" }, moment).inActiveHours(), true); + assert.equal(serverLimits({ from: "13:00", to: "17:00", zone: "Not/AZone" }, moment).inActiveHours(), false); + assert.equal(serverLimits({ from: "", to: "", zone: "Asia/Tokyo" }, moment).inActiveHours(), true); + assert.equal(serverLimits({ from: "10:00", to: "10:00", zone: "Asia/Tokyo" }, moment).inActiveHours(), true); + const bare = new Limits({ clock: moment }); + bare.applyServerSettings({}); + assert.equal(bare.inActiveHours(), true); +}); + +test("both the donor window and the server window must be open", () => { + const noon = at(12); + const open = { from: "00:00", to: "23:59", zone: "UTC" }; + const shut = { from: "00:00", to: "00:01", zone: "UTC" }; + assert.equal(serverLimits(open, noon, { activeFrom: "09:00", activeTo: "17:00" }).admit(0), null); + assert.equal(serverLimits(shut, noon, { activeFrom: "09:00", activeTo: "17:00" }).admit(0), REFUSALS.OUTSIDE_HOURS); + assert.equal(serverLimits(open, noon, { activeFrom: "13:00", activeTo: "17:00" }).admit(0), REFUSALS.OUTSIDE_HOURS); +}); + +test("a settings_update replaces the server window", () => { + const limits = serverLimits({ from: "13:00", to: "17:00", zone: "UTC" }, () => new Date(Date.UTC(2026, 0, 15, 12, 0))); + assert.equal(limits.inActiveHours(), false); + limits.applyServerSettings({ active_from: "", active_to: "", timezone: "UTC" }); + assert.equal(limits.inActiveHours(), true); +}); + +const BASE = 1767225600; +const STEP = 40009; +const SAMPLES = 200; +const PARITY = [ + { from: "09:00", to: "17:00", zone: "Europe/Amsterdam", bits: "01010000001010101010000101010101000000101010100000010101010000001010101010000101010101000000101010100000010101010000001010101010000101010101000000101010100000010101010000001010101010000001010101010000" }, + { from: "22:00", to: "06:00", zone: "America/New_York", bits: "00010101010000001010101000000101010101000010101010100000010101010000001010101000000101010101000010101010100000010101010000001010101000000101010101000000101010101000010101010100000010101010000001010101" }, + { from: "08:30", to: "12:15", zone: "Asia/Kolkata", bits: "00000001010000000000101000000000010101000000000010100000000001010000000000101000000000000101000000000010100000000001010000000000101000000000000101000000000010100000000001010000000000101010000000000101" }, + { from: "00:00", to: "12:00", zone: "Pacific/Auckland", bits: "01101010101010110101010101010010101010101001010101010101101010101010110101010101010010101010101001010101010101101010101010110101010101010010101010101011010101010101101010101010100101010101010010101010" }, + { from: "10:00", to: "10:00", zone: "UTC", bits: "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" }, + { from: "09:00", to: "17:00", zone: "Not/AZone", bits: "01010000101010101000000101010100000010101010000001010101010000001010101000000101010100000010101010100001010101010000001010101000000101010100000010101010100001010101010000001010101000000101010100000010" }, + { from: "23:00", to: "01:00", zone: "Australia/Lord_Howe", bits: "00000000000010000000000001010000000000001000000000000100000000000010000000000001010000000000001000000000000100000000000010000000000000010000000000001000000000000100000000000010000000000000010000000000" }, + { from: "", to: "", zone: "Europe/Amsterdam", bits: "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" }, +]; + +test("server window parity with the Python scheduler over 200 sampled instants", () => { + for (const item of PARITY) { + assert.equal(item.bits.length, SAMPLES); + for (let index = 0; index < SAMPLES; index += 1) { + const moment = new Date((BASE + index * STEP) * 1000); + const limits = serverLimits({ from: item.from, to: item.to, zone: item.zone }, () => moment); + assert.equal(limits.inActiveHours(), item.bits[index] === "1", `${item.from}-${item.to} ${item.zone} #${index}`); + } + } +}); diff --git a/nodes/extension/test/package.test.mjs b/nodes/extension/test/package.test.mjs new file mode 100644 index 00000000..40103ab8 --- /dev/null +++ b/nodes/extension/test/package.test.mjs @@ -0,0 +1,29 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const root = (path) => fileURLToPath(new URL(`../${path}`, import.meta.url)); +const manifest = JSON.parse(readFileSync(root("manifest.json"), "utf8")); +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +test("the manifest declares icons and the action icon, and every file is a PNG of its declared size", () => { + assert.deepEqual(Object.keys(manifest.icons), ["16", "32", "48", "128"]); + for (const [size, path] of Object.entries(manifest.icons)) { + assert.ok(existsSync(root(path)), path); + const data = readFileSync(root(path)); + assert.equal(data.subarray(0, 8).equals(PNG_SIGNATURE), true, path); + assert.equal(data.subarray(12, 16).toString(), "IHDR", path); + assert.equal(data.readUInt32BE(16), Number(size), `${path} width`); + assert.equal(data.readUInt32BE(20), Number(size), `${path} height`); + } + for (const [size, path] of Object.entries(manifest.action.default_icon)) { + assert.equal(manifest.icons[size], path); + } +}); + +test("build.sh ships the icons", () => { + const script = readFileSync(root("build.sh"), "utf8"); + assert.match(script, /manifest\.json src ui icons README\.md/); + assert.match(script, /\("src", "ui", "icons"\)/); +}); diff --git a/nodes/extension/test/popup.test.mjs b/nodes/extension/test/popup.test.mjs new file mode 100644 index 00000000..fd5f0bfa --- /dev/null +++ b/nodes/extension/test/popup.test.mjs @@ -0,0 +1,20 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { describeState, displayLabel } from "../ui/popup.js"; + +test("a suspended state shows the end time in DD/MM/YYYY and says it retries", () => { + const text = describeState({ state: "suspended", suspendedUntil: "2026-03-05T10:07:00Z", detail: "x" }); + assert.match(text, /^Suspended until \d{2}\/\d{2}\/2026 \d{2}:\d{2}, retrying automatically$/); + assert.equal(describeState({ state: "suspended", suspendedUntil: "" }), "Suspended, retrying automatically"); +}); + +test("other states show detail then last error", () => { + assert.equal(describeState({ state: "online", detail: "ok", lastError: "e" }), "ok"); + assert.equal(describeState({ state: "backoff", detail: "", lastError: "e" }), "e"); +}); + +test("the server label is the authority and the local label is the fallback", () => { + assert.equal(displayLabel({ deviceLabel: "Server name" }, { label: "typed" }), "Server name"); + assert.equal(displayLabel({ deviceLabel: "" }, { label: "typed" }), "typed"); + assert.equal(displayLabel({}, { label: "typed" }), "typed"); +}); diff --git a/nodes/extension/test/wsclient.test.mjs b/nodes/extension/test/wsclient.test.mjs index 8e828149..711a8ae0 100644 --- a/nodes/extension/test/wsclient.test.mjs +++ b/nodes/extension/test/wsclient.test.mjs @@ -78,6 +78,8 @@ class FakeTimers { } } +const NOW = Date.parse("2026-01-01T00:00:00Z"); + function build(overrides = {}) { FakeSocket.instances = []; const timers = new FakeTimers(); @@ -104,6 +106,7 @@ function build(overrides = {}) { WebSocketImpl: FakeSocket, timers, random: overrides.random || (() => 0.5), + clock: () => NOW, clientVersion: "9.9.9", platform: "firefox", }); @@ -321,3 +324,109 @@ test("settings_update can only lower the effective limits", async () => { assert.equal(context.limits.concurrency, 1); assert.equal(context.limits.bandwidthBytes, 10 * 1024 * 1024); }); + +function suspendedTicket(until) { + return async () => + new Response(JSON.stringify({ error: { status: 403, message: "x", reason: "suspended", until } }), { + status: 403, + }); +} + +test("a suspended ticket waits until the end instead of halting, then retries", async () => { + const context = build({ random: () => 0, ticketFetch: suspendedTicket("2026-01-01T01:00:00Z") }); + await context.client.start(); + const snapshot = context.state.snapshot(); + assert.equal(snapshot.state, STATES.SUSPENDED); + assert.equal(snapshot.suspendedUntil, "2026-01-01T01:00:00.000Z"); + assert.equal(context.timers.timeouts.at(-1).ms, 3600 * 1000); + assert.equal(context.client.wanted, true); + await context.client.ensure(); + assert.equal(FakeSocket.instances.length, 0); +}); + +test("suspension waits are clamped and an unusable until waits five minutes", async () => { + const cases = [ + ["2025-12-31T00:00:00Z", 60], + ["2030-01-01T00:00:00Z", 24 * 3600], + ["", 300], + ["tomorrow", 300], + ]; + for (const [until, seconds] of cases) { + const context = build({ random: () => 0, ticketFetch: suspendedTicket(until) }); + await context.client.start(); + assert.equal(context.timers.timeouts.at(-1).ms, seconds * 1000, until); + assert.equal(context.state.snapshot().state, STATES.SUSPENDED); + } + const jittered = build({ random: () => 1, ticketFetch: suspendedTicket("") }); + await jittered.client.start(); + assert.equal(jittered.timers.timeouts.at(-1).ms, 330 * 1000); +}); + +test("the retry after a suspension asks for a new ticket", async () => { + let calls = 0; + const context = build({ + random: () => 0, + ticketFetch: async () => { + calls += 1; + if (calls === 1) { + return suspendedTicket("2026-01-01T00:02:00Z")(); + } + return new Response(JSON.stringify({ ticket: "tk2", ws_path: "/nodes/ws" }), { status: 200 }); + }, + }); + await context.client.start(); + context.timers.timeouts.at(-1).fn(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(calls, 2); + assert.equal(FakeSocket.instances.length, 1); +}); + +test("a revoked ticket or other 403 reason still halts", async () => { + const context = build({ + ticketFetch: async () => + new Response(JSON.stringify({ error: { status: 403, message: "x", reason: "revoked" } }), { status: 403 }), + }); + await context.client.start(); + assert.equal(context.state.snapshot().state, STATES.HALTED); +}); + +test("a suspended kick with until closes the socket and holds until then", async () => { + const context = build({ random: () => 0 }); + const socket = await online(context); + await socket.receive({ type: "kick", reason: "suspended", until: "2026-01-01T00:10:00Z" }); + assert.equal(context.state.snapshot().state, STATES.SUSPENDED); + assert.equal(context.timers.timeouts.at(-1).ms, 600 * 1000); + assert.equal(socket.closed.code, 1000); + assert.equal(context.timers.intervals[0].cleared, true); +}); + +test("a suspended kick without until waits five minutes", async () => { + const context = build({ random: () => 0 }); + const socket = await online(context); + await socket.receive({ type: "kick", reason: "suspended" }); + assert.equal(context.timers.timeouts.at(-1).ms, 300 * 1000); +}); + +test("a close 4403 with reason suspended holds, any other 4403 halts", async () => { + const context = build({ random: () => 0 }); + const socket = await online(context); + await socket.serverClose(4403, "suspended"); + assert.equal(context.state.snapshot().state, STATES.SUSPENDED); +}); + +test("welcome stores the server device label and clears an old suspension", async () => { + const context = build(); + await context.state.set({ suspendedUntil: "2026-01-01T00:00:00.000Z" }); + await context.client.start(); + const socket = FakeSocket.instances[0]; + socket.open(); + await socket.receive(welcome({ device_label: "Work laptop" })); + assert.equal(context.state.snapshot().deviceLabel, "Work laptop"); + assert.equal(context.state.snapshot().suspendedUntil, ""); +}); + +test("a welcome without a label leaves the label empty so the popup falls back", async () => { + const context = build(); + await online(context); + assert.equal(context.state.snapshot().deviceLabel, ""); +}); diff --git a/nodes/extension/tools/make_icons.py b/nodes/extension/tools/make_icons.py new file mode 100644 index 00000000..254f1b48 --- /dev/null +++ b/nodes/extension/tools/make_icons.py @@ -0,0 +1,43 @@ +import sys +from pathlib import Path + +from PIL import Image, ImageDraw + +SIZES = (16, 32, 48, 128) +SCALE = 8 +BACKGROUND = (17, 43, 94, 255) +GLYPH = (234, 242, 255, 255) +CENTER = (0.5, 0.5) +OUTER = ((0.5, 0.2), (0.24, 0.68), (0.76, 0.68)) +CORNER_RADIUS = 0.22 +CENTER_RADIUS = 0.1 +OUTER_RADIUS = 0.075 +LINE_WIDTH = 0.04 + + +def render(size: int) -> Image.Image: + big = size * SCALE + image = Image.new("RGBA", (big, big), (0, 0, 0, 0)) + draw = ImageDraw.Draw(image) + draw.rounded_rectangle((0, 0, big - 1, big - 1), radius=big * CORNER_RADIUS, fill=BACKGROUND) + + def point(pair): + return (pair[0] * big, pair[1] * big) + + for outer in OUTER: + draw.line((point(CENTER), point(outer)), fill=GLYPH, width=max(1, round(LINE_WIDTH * big))) + for pair, radius in [(CENTER, CENTER_RADIUS)] + [(outer, OUTER_RADIUS) for outer in OUTER]: + x, y = point(pair) + r = radius * big + draw.ellipse((x - r, y - r, x + r, y + r), fill=GLYPH) + return image.resize((size, size), Image.LANCZOS) + + +def main(out_dir: Path) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + for size in SIZES: + render(size).save(out_dir / f"icon-{size}.png", optimize=True) + + +if __name__ == "__main__": + main(Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).resolve().parent.parent / "icons") diff --git a/nodes/extension/ui/popup.css b/nodes/extension/ui/popup.css index 8f8e7fd3..5ae009ce 100644 --- a/nodes/extension/ui/popup.css +++ b/nodes/extension/ui/popup.css @@ -108,7 +108,8 @@ p { } .dot.connecting, -.dot.backoff { +.dot.backoff, +.dot.suspended { background: var(--warn); } diff --git a/nodes/extension/ui/popup.js b/nodes/extension/ui/popup.js index c7e4d567..472e60e0 100644 --- a/nodes/extension/ui/popup.js +++ b/nodes/extension/ui/popup.js @@ -6,6 +6,7 @@ const STATE_LABELS = { online: "Online", backoff: "Reconnecting", halted: "Stopped", + suspended: "Suspended", }; function formatBytes(value) { @@ -18,6 +19,30 @@ function formatBytes(value) { return `${unit === 0 ? size : size.toFixed(1)} ${UNITS[unit]}`; } +function pad(value) { + return String(value).padStart(2, "0"); +} + +function formatInstant(iso) { + const moment = new Date(iso); + if (Number.isNaN(moment.getTime())) { + return ""; + } + return `${pad(moment.getDate())}/${pad(moment.getMonth() + 1)}/${moment.getFullYear()} ${pad(moment.getHours())}:${pad(moment.getMinutes())}`; +} + +export function describeState(state) { + if (state.state === "suspended") { + const ends = formatInstant(state.suspendedUntil); + return ends ? `Suspended until ${ends}, retrying automatically` : "Suspended, retrying automatically"; + } + return state.detail || state.lastError || ""; +} + +export function displayLabel(state, config) { + return state.deviceLabel || config.label; +} + class Popup { constructor(root) { this.root = root; @@ -55,8 +80,8 @@ class Popup { const { config, state, usage } = view; this.$("dot").className = `dot ${state.state}`; this.$("state").textContent = STATE_LABELS[state.state] || state.state; - this.$("detail").textContent = state.detail || state.lastError || ""; - this.$("label").textContent = config.label; + this.$("detail").textContent = describeState(state); + this.$("label").textContent = displayLabel(state, config); this.$("disclosure-card").hidden = config.disclosed; const toggle = this.$("toggle"); toggle.checked = config.enabled; @@ -152,4 +177,6 @@ class Popup { } } -new Popup(document).start(); +if (typeof document !== "undefined") { + new Popup(document).start(); +} diff --git a/pyproject.toml b/pyproject.toml index 516ce3a1..eeb2c175 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "devplacepy" -version = "1.1.4" +version = "1.1.5" description = "DevPlace - The Developer Social Network" requires-python = ">=3.12" dependencies = [ @@ -62,4 +62,5 @@ filterwarnings = [ "ignore:'crypt' is deprecated", "ignore:Using `httpx` with `starlette.testclient` is deprecated", "ignore:invalid escape sequence", + "ignore:Skipped unsupported reflection of expression-based index", ] diff --git a/tests/api/admin/communities/index.py b/tests/api/admin/communities/index.py index 0bc8fc12..458eda7c 100644 --- a/tests/api/admin/communities/index.py +++ b/tests/api/admin/communities/index.py @@ -209,6 +209,27 @@ def test_password_is_stored_but_never_returned(client, admin, fake_world): assert store.get("forum-test")["login_password"] == "" +def test_the_authenticator_secret_is_stored_but_never_returned(client, admin, fake_world): + client.post( + "/admin/communities", + data={"query": "https://forum.test/api/posts", "login_username": "bot", "login_password": "s3cret-pw", "login_totp_secret": "JBSWY3DPEHPK3PXP"}, + headers=admin["headers"], + ) + row = wait_for(lambda: (found := store.get("forum-test")) and found["status"] == "ready" and found) + assert row["login_totp_secret"] == "JBSWY3DPEHPK3PXP" + assert store.credentials(row)["totp"] == "JBSWY3DPEHPK3PXP" + for url in ("/admin/communities", "/admin/communities/forum-test", "/admin/communities/forum-test/ledger"): + for headers in (admin["headers"], {"X-API-KEY": admin["headers"]["X-API-KEY"]}): + body = client.get(url, headers=headers) + assert "JBSWY3DPEHPK3PXP" not in body.text + as_json = client.get("/admin/communities/forum-test", headers=admin["headers"]).json() + assert as_json["community"]["has_totp"] is True and "login_totp_secret" not in as_json["community"] + client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "login_username": "other"}, headers=admin["headers"]) + assert store.get("forum-test")["login_totp_secret"] == "JBSWY3DPEHPK3PXP" + client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "clear_login": "on"}, headers=admin["headers"]) + assert store.get("forum-test")["login_totp_secret"] == "" + + def test_devii_tools_are_admin_only_and_delete_needs_confirmation(): from devplacepy.services.devii.actions.catalog.community import COMMUNITY_ACTIONS from devplacepy.services.devii.actions.dispatcher import CONFIRM_REQUIRED diff --git a/tests/api/admin/communities/schema.py b/tests/api/admin/communities/schema.py new file mode 100644 index 00000000..c4952ad9 --- /dev/null +++ b/tests/api/admin/communities/schema.py @@ -0,0 +1,97 @@ +# retoor <retoor@molodetz.nl> + +import json + +from devplacepy.database import get_table +from devplacepy.services.community.files import files +from devplacepy.services.community.store import store +from tests.api.admin.communities.index import add_ready, admin, client, fake_world, make_account # noqa: F401 + + +def save(client, admin, schema_text, slug="forum-test"): + return client.post(f"/admin/communities/{slug}/schema", data={"schema": schema_text}, headers=admin["headers"]) + + +def test_authz(client, admin, fake_world): + add_ready(client, admin) + member = make_account() + assert client.get("/admin/communities/forum-test/schema", headers={"Accept": "application/json"}).status_code in (401, 303) + assert client.get("/admin/communities/forum-test/schema", headers=member["headers"]).status_code == 403 + assert client.post("/admin/communities/forum-test/schema", data={"schema": "{}"}, headers=member["headers"]).status_code == 403 + assert client.post("/admin/communities/forum-test/schema/restore", headers=member["headers"]).status_code == 403 + + +def test_the_page_shows_the_schema_and_never_a_credential(client, admin, fake_world): + add_ready(client, admin) + store.update(store.get("forum-test")["uid"], login_password="topsecret-pw", login_totp_secret="TOTPSECRETVALUE") + body = client.get("/admin/communities/forum-test/schema", headers=admin["headers"]).json() + assert json.loads(body["schema_text"])["origin"] == "forum.test" and body["has_previous"] is False + page = client.get("/admin/communities/forum-test/schema", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]}) + assert page.status_code == 200 and "community-schema" in page.text + assert "topsecret-pw" not in page.text and "TOTPSECRETVALUE" not in page.text + assert "Edit schema" in client.get("/admin/communities/forum-test", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]}).text + + +def test_a_verified_schema_is_stored_regenerated_and_the_old_one_kept(client, admin, fake_world): + row = add_ready(client, admin) + schema = json.loads(json.dumps(row["schema"])) + schema["feeds"][0]["post"]["fields"]["title"] = ["$.title", "$.id"] + response = save(client, admin, json.dumps(schema)) + assert response.status_code == 200, response.text + assert response.json()["data"]["posts"] >= 1 + after = store.get("forum-test") + assert after["schema"]["feeds"][0]["post"]["fields"]["title"] == ["$.title", "$.id"] + assert after["schema_previous"] == row["schema"] and after["status"] == "ready" + assert files.load("forum-test", after["schema"]).SCHEMA["feeds"][0]["post"]["fields"]["title"] == ["$.title", "$.id"] + rows = list(get_table("audit_log").find(event_key="admin.community.schema", target_label="forum-test")) + assert rows + + +def test_a_structurally_invalid_schema_is_refused_with_every_reason_and_changes_nothing(client, admin, fake_world): + row = add_ready(client, admin) + broken = json.loads(json.dumps(row["schema"])) + broken["feeds"][0]["type"] = "soap" + broken["origin"] = "NOT A HOST" + response = save(client, admin, json.dumps(broken)) + assert response.status_code == 400 + assert store.get("forum-test")["schema"] == row["schema"] + html = client.post("/admin/communities/forum-test/schema", data={"schema": json.dumps(broken)}, headers={"X-API-KEY": admin["headers"]["X-API-KEY"]}) + assert html.status_code == 400 and "Not saved" in html.text and "feeds[0].type" in html.text and "origin" in html.text + assert 'name="schema"' in html.text and "NOT A HOST" in html.text + + +def test_invalid_json_is_refused(client, admin, fake_world): + add_ready(client, admin) + html = client.post("/admin/communities/forum-test/schema", data={"schema": "{not json"}, headers={"X-API-KEY": admin["headers"]["X-API-KEY"]}) + assert html.status_code == 400 and "not valid JSON" in html.text + + +def test_a_schema_that_collects_nothing_live_is_not_saved(client, admin, fake_world): + row = add_ready(client, admin) + dead = json.loads(json.dumps(row["schema"])) + dead["feeds"][0]["list_path"] = "$.nothing_here" + response = client.post("/admin/communities/forum-test/schema", data={"schema": json.dumps(dead)}, headers={"X-API-KEY": admin["headers"]["X-API-KEY"]}) + assert response.status_code == 422 and "no posts" in response.text and "not saved" in response.text + assert store.get("forum-test")["schema"] == row["schema"] and not store.get("forum-test")["schema_previous"] + + +def test_restore_swaps_back_and_refuses_when_there_is_nothing(client, admin, fake_world): + row = add_ready(client, admin) + assert client.post("/admin/communities/forum-test/schema/restore", headers=admin["headers"]).status_code == 409 + changed = json.loads(json.dumps(row["schema"])) + changed["feeds"][0]["post"]["fields"]["title"] = ["$.title", "$.id"] + assert save(client, admin, json.dumps(changed)).status_code == 200 + assert client.post("/admin/communities/forum-test/schema/restore", headers=admin["headers"]).status_code == 200 + restored = store.get("forum-test") + assert restored["schema"] == row["schema"] and restored["schema_previous"] == changed + assert list(get_table("audit_log").find(event_key="admin.community.schema_restore", target_label="forum-test")) + + +def test_the_devii_tools_exist_and_the_save_needs_confirmation(): + from devplacepy.services.devii.actions.catalog.community import COMMUNITY_ACTIONS + from devplacepy.services.devii.actions.dispatcher import CONFIRM_REQUIRED + + tools = {action.name: action for action in COMMUNITY_ACTIONS} + assert tools["community_schema_get"].requires_admin and tools["community_schema_set"].requires_admin + assert "community_schema_set" in CONFIRM_REQUIRED + assert any(param.name == "confirm" for param in tools["community_schema_set"].params) diff --git a/tests/api/admin/nodes/index.py b/tests/api/admin/nodes/index.py index 6af25760..7fdaf36d 100644 --- a/tests/api/admin/nodes/index.py +++ b/tests/api/admin/nodes/index.py @@ -28,6 +28,8 @@ def test_admin_overview_shape(client, admin): "nodes_cache_ttl_seconds", "nodes_daily_social_user", "nodes_social_via_nodes", + "nodes_allow_unknown_class", + "nodes_redundancy_percent", } listed = {row["uid"]: row for row in body["devices"]} assert listed[device["uid"]]["owner_username"] == owner["username"] @@ -40,3 +42,26 @@ def test_html_page_renders(client, admin): assert response.status_code == 200 assert "Community nodes" in response.text assert "data-live=\"summary\"" in response.text + + +def test_overview_carries_circuits_outcomes_lifetime_geo_and_cache(client, admin): + from devplacepy.database.nodes import flush_counters, save_circuits + + flush_counters({"outcome:reddit:residential:ok": 3, "outcome:reddit:residential:blocked": 1, "kicked": 2}) + save_circuits([{"key": "destination:reddit.com", "failures": 9, "state": "open", "opened_at": 4_000_000_000.0}]) + try: + body = client.get("/admin/nodes", headers=admin_headers(admin)).json() + row = next(item for item in body["outcomes"] if item["adapter"] == "reddit" and item["asn_class"] == "residential") + assert row["ok"] >= 3 and row["blocked"] >= 1 and row["total"] == row["ok"] + row["blocked"] + row["failed"] + assert body["lifetime"]["kicked"] >= 2 + assert not any(name.startswith("outcome:") for name in body["lifetime"]) + assert body["circuits"][0]["key"] == "destination:reddit.com" and body["circuits"][0]["state"] == "open" + assert set(body["geo"]) >= {"available", "path"} + assert set(body["cache"]) == {"entries", "bytes"} + finally: + save_circuits([]) + + +def test_html_shows_the_new_sections_and_the_empty_outcome_state(client, admin): + text = client.get("/admin/nodes", headers=admin["headers"]).text + assert "Circuit breakers" in text and "Success by address class" in text and "Address database" in text diff --git a/tests/api/admin/nodes/override.py b/tests/api/admin/nodes/override.py new file mode 100644 index 00000000..ed57c3ae --- /dev/null +++ b/tests/api/admin/nodes/override.py @@ -0,0 +1,53 @@ +# retoor <retoor@molodetz.nl> + +from devplacepy.database.nodes import get_device +from tests.api.admin.nodes.conftest import admin_headers, make_admin, make_device +from tests.api.nodes.conftest import make_member + + +def test_member_cannot_override(client): + member = make_member() + device = make_device(member["uid"]) + response = client.post(f"/admin/nodes/devices/{device['uid']}/class", json={"value": "allow"}, headers=admin_headers(member)) + assert response.status_code == 403 + + +def test_override_round_trip_is_visible_in_the_overview(client, admin): + owner = make_member() + device = make_device(owner["uid"]) + url = f"/admin/nodes/devices/{device['uid']}/class" + response = client.post(url, json={"value": "allow"}, headers=admin_headers(admin)) + assert response.status_code == 200 and response.json()["data"] == {"uid": device["uid"], "class_override": "allow"} + assert get_device(device["uid"])["class_override"] == "allow" + listed = {row["uid"]: row for row in client.get("/admin/nodes", headers=admin_headers(admin)).json()["devices"]} + assert listed[device["uid"]]["class_override"] == "allow" + client.post(url, json={"value": "clear"}, headers=admin_headers(admin)) + assert not get_device(device["uid"])["class_override"] + + +def test_only_known_values_are_accepted(client, admin): + owner = make_member() + device = make_device(owner["uid"]) + response = client.post(f"/admin/nodes/devices/{device['uid']}/class", json={"value": "deny"}, headers=admin_headers(admin)) + assert response.status_code in (400, 422) + + +def test_unknown_device_is_404(client, admin): + assert client.post("/admin/nodes/devices/nope/class", json={"value": "allow"}, headers=admin_headers(admin)).status_code == 404 + + +def test_junior_admin_cannot_override_a_senior_admins_device(client): + senior, junior = make_admin(), make_admin() + device = make_device(senior["uid"]) + client.post(f"/admin/nodes/devices/{device['uid']}/class", json={"value": "allow"}, headers=admin_headers(junior)) + assert not get_device(device["uid"]).get("class_override") + + +def test_the_change_is_audited(client, admin): + from devplacepy.database import get_table + + owner = make_member() + device = make_device(owner["uid"]) + client.post(f"/admin/nodes/devices/{device['uid']}/class", json={"value": "allow"}, headers=admin_headers(admin)) + rows = list(get_table("audit_log").find(event_key="node.device.override", target_uid=device["uid"])) + assert rows and rows[0]["new_value"] == "allow" diff --git a/tests/api/game/mutations.py b/tests/api/game/mutations.py index d7a57a2b..d490cfd6 100644 --- a/tests/api/game/mutations.py +++ b/tests/api/game/mutations.py @@ -429,3 +429,62 @@ def test_grant_ineligible_when_rich_returns_400(app_server, seeded_db): ) r = session.post(f"{BASE_URL}/game/grant", headers=JSON) assert r.status_code == 400 + + +def test_plot_stop_requires_auth(app_server, seeded_db): + r = requests.post( + f"{BASE_URL}/game/plot/stop", data={"slot": 0, "stop": "true"}, headers=JSON + ) + assert r.status_code in (401, 303) + + +def test_plot_stop_without_continuous_delivery_returns_400(app_server, seeded_db): + session, name = _signup() + _reset_farm(name) + session.post(f"{BASE_URL}/game/plant", data={"slot": 0, "crop": "shell"}, headers=JSON) + r = session.post( + f"{BASE_URL}/game/plot/stop", data={"slot": 0, "stop": "true"}, headers=JSON + ) + assert r.status_code == 400 + + +def test_plot_stop_empty_plot_returns_400(app_server, seeded_db): + session, name = _signup() + _reset_farm(name) + _set_farm(name, mastery_autoreplant=1) + r = session.post( + f"{BASE_URL}/game/plot/stop", data={"slot": 0, "stop": "true"}, headers=JSON + ) + assert r.status_code == 400 + + +def test_plot_stop_leaves_plot_empty_after_harvest(app_server, seeded_db): + session, name = _signup() + _reset_farm(name) + _set_farm(name, mastery_autoreplant=1) + assert session.post( + f"{BASE_URL}/game/plant", data={"slot": 0, "crop": "shell"}, headers=JSON + ).status_code == 200 + r = session.post( + f"{BASE_URL}/game/plot/stop", data={"slot": 0, "stop": "true"}, headers=JSON + ) + assert r.status_code == 200, r.text[:200] + plot = r.json()["farm"]["plots"][0] + assert plot["can_stop_replant"] is True and plot["stop_after_harvest"] is True + refresh_snapshot() + user = get_table("users").find_one(username=name) + farm = store.get_farm(user["uid"]) + row = get_table("game_plots").find_one(farm_uid=farm["uid"], slot_index=0) + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + get_table("game_plots").update({"uid": row["uid"], "ready_at": past}, ["uid"]) + refresh_snapshot() + harvested = session.post( + f"{BASE_URL}/game/harvest", data={"slot": 0}, headers=JSON + ) + assert harvested.status_code == 200, harvested.text[:200] + assert harvested.json()["farm"]["plots"][0]["state"] == "empty" + resumed = session.post( + f"{BASE_URL}/game/plant", data={"slot": 0, "crop": "python"}, headers=JSON + ) + assert resumed.status_code == 200 + assert resumed.json()["farm"]["plots"][0]["stop_after_harvest"] is False diff --git a/tests/api/nodes/jobs.py b/tests/api/nodes/jobs.py index 5040f5ae..fbce7f5f 100644 --- a/tests/api/nodes/jobs.py +++ b/tests/api/nodes/jobs.py @@ -7,7 +7,7 @@ import time import pytest from starlette.websockets import WebSocketDisconnect -from devplacepy.database import get_table +from devplacepy.database import get_table, set_setting from devplacepy.database import nodes as store from devplacepy.services.manager import service_manager from devplacepy.services.nodes.broker import JobBroker @@ -26,7 +26,9 @@ def relaxed(client): original = service.scheduler.destination_ready service.scheduler.destination_ready = lambda target: True get_table("node_requests").delete() + set_setting("nodes_allow_unknown_class", "1") yield service + set_setting("nodes_allow_unknown_class", "0") service.scheduler.destination_ready = original get_table("node_requests").delete() diff --git a/tests/api/nodes/ticket.py b/tests/api/nodes/ticket.py index 567325ca..2596abf0 100644 --- a/tests/api/nodes/ticket.py +++ b/tests/api/nodes/ticket.py @@ -59,6 +59,7 @@ def test_a_suspended_device_gets_no_ticket(client, member): response = client.post("/nodes/ticket", headers=bearer(secret)) assert response.status_code == 403 assert response.json()["error"]["reason"] == "suspended" + assert response.json()["error"]["until"].startswith("20") def test_a_deleted_device_secret_stops_working(client, member): diff --git a/tests/api/nodes/ws.py b/tests/api/nodes/ws.py index e1a25914..c6a1fba8 100644 --- a/tests/api/nodes/ws.py +++ b/tests/api/nodes/ws.py @@ -55,6 +55,7 @@ def test_handshake_welcomes_with_the_published_numbers(client, member): item.suffix for item in SPEC.destinations } assert welcome["settings"]["enabled"] is True + assert welcome["device_label"] == "Home laptop" service = service_manager.get_service("nodes") assert service.registry.get(device_uid) is not None ws.send_text(json.dumps({"type": "pong"})) @@ -153,3 +154,44 @@ def test_capacity_is_a_hard_ceiling(client, member): assert service.counters.snapshot()["rejected_capacity"] >= 1 finally: set_setting("nodes_max_total", previous) + + +def test_the_donor_address_is_classed_through_the_geo_database(client, member, tmp_path, monkeypatch): + from devplacepy import config + from devplacepy.database.nodes import get_device + from devplacepy.services.nodes import geo + + monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=1, min_country_rows=1, checks=())) + target = tmp_path / "geo.db" + built = geo.build( + ["8.8.8.0,8.8.8.255,15169,Google LLC", "20.0.0.0,20.0.0.255,64500,Home Fiber BV"], + ["8.8.8.0,8.8.8.255,US", "20.0.0.0,20.0.0.255,NL"], + target, + "2026-10", + ) + geo.install(built, target) + monkeypatch.setattr(config, "NODE_GEO_DB", target) + for address, expected in (("8.8.8.8", ("US", "datacenter")), ("20.0.0.9", ("NL", "residential")), ("9.9.9.9", ("", "unknown"))): + device_uid, _, ticket = ticket_for(client, make_fresh(member)) + with client.websocket_connect(f"/nodes/ws?ticket={ticket}", headers={"x-real-ip": address}) as ws: + ws.send_text(hello()) + ws.receive_text() + session = service_manager.get_service("nodes").registry.get(device_uid) + assert session.asn_class == expected[1] + row = get_device(device_uid) + assert (row["country"], row["asn_class"]) == expected + assert "x-real-ip" not in str(row) and address not in str(row.values()) + + +def test_a_missing_geo_database_is_counted_and_classes_unknown(client, member, tmp_path, monkeypatch): + from devplacepy import config + + monkeypatch.setattr(config, "NODE_GEO_DB", tmp_path / "absent.db") + device_uid, _, ticket = ticket_for(client, member) + service = service_manager.get_service("nodes") + before = service.counters.snapshot()["geo_unavailable"] + with client.websocket_connect(f"/nodes/ws?ticket={ticket}", headers={"x-real-ip": "8.8.8.8"}) as ws: + ws.send_text(hello()) + ws.receive_text() + assert service.registry.get(device_uid).asn_class == "unknown" + assert service.counters.snapshot()["geo_unavailable"] == before + 1 diff --git a/tests/api/sync/attachments.py b/tests/api/sync/attachments.py new file mode 100644 index 00000000..5e3dc762 --- /dev/null +++ b/tests/api/sync/attachments.py @@ -0,0 +1,80 @@ +# retoor <retoor@molodetz.nl> + +import base64 +from pathlib import Path + +from devplacepy.config import ATTACHMENTS_DIR +from devplacepy.database import get_table +from tests.api.sync.conftest import PNG_1X1, attachment_record, seed_world, sync_ok + + +def local(origin, kind, origin_uid): + return get_table("sync_links").find_one(origin=origin, kind=kind, origin_uid=origin_uid)["local_uid"] + + +def test_an_attachment_is_stored_and_linked_to_its_target(client, admin, origin): + seed_world(client, admin, origin) + result = sync_ok(client, admin, "attachments", origin, [attachment_record()])["results"][0] + assert result["status"] == "created" + row = get_table("attachments").find_one(uid=result["local_uid"]) + assert (row["target_type"], row["target_uid"]) == ("post", local(origin, "posts", "p1")) + assert row["user_uid"] == local(origin, "users", "u1") and row["mime_type"] == "image/png" + assert (Path(ATTACHMENTS_DIR) / row["directory"] / row["stored_name"]).read_bytes() == base64.b64decode(PNG_1X1) + assert row["created_at"].startswith("2025-01-05") + + +def test_replay_is_unchanged_and_a_content_change_is_refused(client, admin, origin): + seed_world(client, admin, origin) + sync_ok(client, admin, "attachments", origin, [attachment_record()]) + assert sync_ok(client, admin, "attachments", origin, [attachment_record()])["results"][0]["status"] == "unchanged" + other = base64.b64encode(b"GIF89a" + b"x" * 40).decode() + changed = sync_ok(client, admin, "attachments", origin, [attachment_record(content=other, filename="b.gif", updated_at="2025-02-01T00:00:00Z")])["results"][0] + assert changed["status"] == "rejected" and changed["reason"] == "immutable" + + +def test_tombstone_soft_deletes_and_restores(client, admin, origin): + seed_world(client, admin, origin) + created = sync_ok(client, admin, "attachments", origin, [attachment_record()])["results"][0] + gone = attachment_record(updated_at="2025-02-01T00:00:00Z", deleted=True) + assert sync_ok(client, admin, "attachments", origin, [gone])["results"][0]["status"] == "updated" + assert get_table("attachments").find_one(uid=created["local_uid"])["deleted_by"] == "sync" + back = attachment_record(updated_at="2025-03-01T00:00:00Z") + sync_ok(client, admin, "attachments", origin, [back]) + assert get_table("attachments").find_one(uid=created["local_uid"])["deleted_at"] is None + + +def test_validation_and_type_rules(client, admin, origin): + seed_world(client, admin, origin) + body = sync_ok( + client, admin, "attachments", origin, + [ + attachment_record("a1", content="not base64!!"), + attachment_record("a2", content=""), + attachment_record("a3", filename="run.exe", content=base64.b64encode(b"MZ" + b"0" * 10).decode()), + attachment_record("a4", filename="../../etc/passwd.png"), + ], + ) + statuses = [item["status"] for item in body["results"]] + assert statuses[:3] == ["rejected", "rejected", "rejected"] + stored = get_table("attachments").find_one(uid=body["results"][3].get("local_uid") or "x") + assert statuses[3] == "created" and stored["original_filename"] == "passwd.png" + + +def test_an_oversize_file_is_refused(client, admin, origin): + from devplacepy.services.sync.spec import LIMITS + + seed_world(client, admin, origin) + big = base64.b64encode(b"\x00" * (LIMITS.max_attachment_bytes + 1)).decode() + result = sync_ok(client, admin, "attachments", origin, [attachment_record(content=big)])["results"][0] + assert result["status"] == "rejected" and "exceeds" in result["errors"][0] + + +def test_an_unknown_target_waits_and_is_released_when_the_target_arrives(client, admin, origin): + from tests.api.sync.conftest import post_record, user_record + + sync_ok(client, admin, "users", origin, [user_record("u1", "attuser1")]) + waiting = sync_ok(client, admin, "attachments", origin, [attachment_record(target="p1")])["results"][0] + assert waiting["status"] == "deferred" + body = sync_ok(client, admin, "posts", origin, [post_record("p1", "u1")]) + assert body["released"] == {"created": 1} + assert get_table("attachments").count(target_uid=local(origin, "posts", "p1")) == 1 diff --git a/tests/api/sync/comments.py b/tests/api/sync/comments.py index 8824f744..3bcc630b 100644 --- a/tests/api/sync/comments.py +++ b/tests/api/sync/comments.py @@ -9,7 +9,7 @@ def seed_thread(client, admin, origin): sync_ok(client, admin, "posts", origin, [post_record("p1", "u1"), post_record("p2", "u1")]) -def test_unknown_references_are_rejected_with_specific_reasons(client, admin, origin): +def test_unknown_references_are_deferred_with_specific_reasons(client, admin, origin): seed_thread(client, admin, origin) body = sync_ok( client, admin, "comments", origin, @@ -20,6 +20,7 @@ def test_unknown_references_are_rejected_with_specific_reasons(client, admin, or ], ) assert [r["reason"] for r in body["results"]] == ["unknown_author", "unknown_post", "unknown_parent"] + assert [r["status"] for r in body["results"]] == ["deferred", "deferred", "deferred"] def test_create_with_nested_reply_in_one_batch(client, admin, origin): diff --git a/tests/api/sync/conftest.py b/tests/api/sync/conftest.py index 9c9f6695..39dbf554 100644 --- a/tests/api/sync/conftest.py +++ b/tests/api/sync/conftest.py @@ -96,3 +96,96 @@ def comment_record(uid="c1", post="p1", author="u1", **extra) -> dict: "created_at": "2025-01-03T00:00:00Z", **extra, } + + +def project_record(uid="pr1", author="u1", **extra) -> dict: + return { + "origin_uid": uid, + "author_uid": author, + "title": "Synced project", + "description": "A project that came in from another community", + "created_at": "2025-01-02T00:00:00Z", + **extra, + } + + +def poll_record(uid="po1", post="p1", author="u1", **extra) -> dict: + return { + "origin_uid": uid, + "post_uid": post, + "author_uid": author, + "question": "Tabs or spaces?", + "options": [{"origin_uid": "a", "label": "Tabs"}, {"origin_uid": "b", "label": "Spaces"}], + "created_at": "2025-01-03T00:00:00Z", + **extra, + } + + +def vote_record(uid="v1", voter="u2", target="p1", kind="post", value=1, **extra) -> dict: + return { + "origin_uid": uid, + "voter_uid": voter, + "target_kind": kind, + "target_uid": target, + "value": value, + "created_at": "2025-01-04T00:00:00Z", + **extra, + } + + +def reaction_record(uid="r1", author="u2", target="p1", kind="post", emoji="🔥", **extra) -> dict: + return { + "origin_uid": uid, + "author_uid": author, + "target_kind": kind, + "target_uid": target, + "emoji": emoji, + "created_at": "2025-01-04T00:00:00Z", + **extra, + } + + +def follow_record(uid="f1", follower="u2", following="u1", **extra) -> dict: + return { + "origin_uid": uid, + "follower_uid": follower, + "following_uid": following, + "created_at": "2025-01-04T00:00:00Z", + **extra, + } + + +def poll_vote_record(uid="pv1", voter="u2", poll="po1", option="a", **extra) -> dict: + return { + "origin_uid": uid, + "voter_uid": voter, + "poll_uid": poll, + "option_uid": option, + "created_at": "2025-01-05T00:00:00Z", + **extra, + } + + +PNG_1X1 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + + +def attachment_record(uid="at1", author="u1", target="p1", kind="post", filename="pixel.png", content=PNG_1X1, **extra) -> dict: + return { + "origin_uid": uid, + "author_uid": author, + "target_kind": kind, + "target_uid": target, + "filename": filename, + "content_b64": content, + "created_at": "2025-01-05T00:00:00Z", + **extra, + } + + +def seed_world(client, admin, origin): + sync_ok(client, admin, "users", origin, [user_record("u1", unique("wu1")), user_record("u2", unique("wu2")), user_record("u3", unique("wu3"))]) + sync_ok(client, admin, "projects", origin, [project_record("pr1", "u1")]) + sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", project_uid="pr1")]) + sync_ok(client, admin, "comments", origin, [comment_record("c1", "p1", "u2")]) diff --git a/tests/api/sync/deletions.py b/tests/api/sync/deletions.py new file mode 100644 index 00000000..12f113c4 --- /dev/null +++ b/tests/api/sync/deletions.py @@ -0,0 +1,109 @@ +# retoor <retoor@molodetz.nl> + +from devplacepy.database import get_table +from tests.api.sync.conftest import comment_record, post_record, sync_ok, unique, user_record + + +def listing(client, admin, origin, **params): + return client.get("/sync/deletions", params={"origin": origin, **params}, headers=admin["headers"]) + + +def ack(client, admin, origin, items): + return client.post("/sync/deletions/ack", json={"origin": origin, "items": items}, headers=admin["headers"]) + + +def seed(client, admin, origin): + sync_ok(client, admin, "users", origin, [user_record("u1", unique("del"))]) + sync_ok(client, admin, "posts", origin, [post_record("p1", "u1"), post_record("p2", "u1"), post_record("p3", "u1")]) + sync_ok(client, admin, "comments", origin, [comment_record("c1", "p1", "u1")]) + + +def local(origin, kind, origin_uid): + return get_table("sync_links").find_one(origin=origin, kind=kind, origin_uid=origin_uid)["local_uid"] + + +def test_authz(client, admin): + from tests.api.sync.conftest import make_account + + assert client.get("/sync/deletions", params={"origin": "a.b"}).status_code == 401 + assert listing(client, make_account("Member"), "a.b").status_code == 403 + assert client.get("/sync/deletions", headers=admin["headers"]).status_code == 422 + + +def test_nothing_is_listed_for_a_clean_origin_and_sync_own_deletions_never_are(client, admin, origin): + seed(client, admin, origin) + assert listing(client, admin, origin).json()["items"] == [] + sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", updated_at="2025-05-01T00:00:00Z", deleted=True)]) + assert listing(client, admin, origin).json()["items"] == [] + + +def test_a_member_deletion_a_system_deletion_and_a_purge_are_told_apart(client, admin, origin): + seed(client, admin, origin) + get_table("posts").update({"uid": local(origin, "posts", "p1"), "deleted_at": "2026-01-01T00:00:00+00:00", "deleted_by": "someone"}, ["uid"]) + get_table("posts").update({"uid": local(origin, "posts", "p2"), "deleted_at": "2026-01-02T00:00:00+00:00", "deleted_by": "system"}, ["uid"]) + purged_uid = local(origin, "posts", "p3") + get_table("posts").delete(uid=purged_uid) + body = listing(client, admin, origin).json() + found = {(item["kind"], item["origin_uid"]): item["by"] for item in body["items"]} + assert found == {("posts", "p1"): "member", ("posts", "p2"): "system", ("posts", "p3"): "purged"} + assert body["count"] == 3 and body["has_more"] is False + + +def test_paging_with_a_cursor_covers_every_entry_once(client, admin, origin): + seed(client, admin, origin) + for origin_uid in ("p1", "p2", "p3"): + get_table("posts").update({"uid": local(origin, "posts", origin_uid), "deleted_at": "2026-01-01T00:00:00+00:00", "deleted_by": "m"}, ["uid"]) + get_table("comments").update({"uid": local(origin, "comments", "c1"), "deleted_at": "2026-01-01T00:00:00+00:00", "deleted_by": "m"}, ["uid"]) + seen, cursor = [], "" + while True: + body = listing(client, admin, origin, limit=2, cursor=cursor).json() + seen += [(item["kind"], item["origin_uid"]) for item in body["items"]] + cursor = body["next_cursor"] + if not body["has_more"]: + break + assert sorted(seen) == [("comments", "c1"), ("posts", "p1"), ("posts", "p2"), ("posts", "p3")] + assert listing(client, admin, origin, cursor="zz").status_code == 400 + + +def test_other_origins_are_never_listed(client, admin, origin): + seed(client, admin, origin) + get_table("posts").update({"uid": local(origin, "posts", "p1"), "deleted_at": "2026-01-01T00:00:00+00:00", "deleted_by": "m"}, ["uid"]) + assert listing(client, admin, f"{unique('x')}.example.org").json()["items"] == [] + + +def test_acknowledging_stops_the_listing_and_quiets_the_resend(client, admin, origin): + seed(client, admin, origin) + purged = local(origin, "posts", "p3") + get_table("posts").delete(uid=purged) + resend = sync_ok(client, admin, "posts", origin, [post_record("p3", "u1", updated_at="2025-09-01T00:00:00Z")])["results"][0] + assert resend["status"] == "rejected" and resend["reason"] == "purged_locally" + response = ack(client, admin, origin, [{"kind": "posts", "origin_uid": "p3"}]) + assert response.status_code == 200 and response.json()["acknowledged"] == 1 + assert listing(client, admin, origin).json()["items"] == [] + again = sync_ok(client, admin, "posts", origin, [post_record("p3", "u1", updated_at="2025-10-01T00:00:00Z")])["results"][0] + assert again["status"] == "unchanged" and again["reason"] == "acknowledged_deleted" + assert ack(client, admin, origin, [{"kind": "posts", "origin_uid": "p3"}]).json()["acknowledged"] == 0 + + +def test_a_locally_deleted_row_after_ack_answers_acknowledged(client, admin, origin): + seed(client, admin, origin) + get_table("posts").update({"uid": local(origin, "posts", "p1"), "deleted_at": "2026-01-01T00:00:00+00:00", "deleted_by": "m"}, ["uid"]) + ack(client, admin, origin, [{"kind": "posts", "origin_uid": "p1"}]) + result = sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", content="edited after local delete", updated_at="2025-09-01T00:00:00Z")])["results"][0] + assert result["status"] == "unchanged" and result["reason"] == "acknowledged_deleted" + + +def test_ack_validation_and_audit(client, admin, origin): + assert ack(client, admin, origin, []).status_code == 422 + assert ack(client, admin, origin, [{"kind": "gists", "origin_uid": "x"}]).status_code == 422 + assert ack(client, admin, origin, [{"kind": "posts", "origin_uid": "bad id"}]).status_code == 422 + assert ack(client, admin, origin, [{"kind": "posts", "origin_uid": "unknown-one"}]).json()["acknowledged"] == 0 + rows = list(get_table("audit_log").find(event_key="sync.deletions.ack", actor_uid=admin["uid"])) + assert rows + + +def test_a_purged_user_is_listed(client, admin, origin): + seed(client, admin, origin) + get_table("users").delete(uid=local(origin, "users", "u1")) + items = listing(client, admin, origin).json()["items"] + assert ("users", "u1", "purged") in {(item["kind"], item["origin_uid"], item["by"]) for item in items} diff --git a/tests/api/sync/export.py b/tests/api/sync/export.py new file mode 100644 index 00000000..48c8ae91 --- /dev/null +++ b/tests/api/sync/export.py @@ -0,0 +1,203 @@ +# retoor <retoor@molodetz.nl> + +from datetime import datetime, timedelta, timezone + +import pytest + +from devplacepy.database import get_table, set_setting +from devplacepy.utils import generate_uid, register_account +from tests.api.sync.conftest import ( + comment_record, + post_record, + sync, + sync_ok, + unique, + user_record, +) + + +def export(client, admin, kind, **params): + return client.get(f"/sync/export/{kind}", params=params, headers=admin["headers"]) + + +def pull_all(client, admin, kind, since=None): + records, cursor, pages = [], "", 0 + while True: + params = {"cursor": cursor, "limit": 50} + if since: + params["since"] = since + response = export(client, admin, kind, **params) + assert response.status_code == 200, response.text + body = response.json() + records.extend(body["records"]) + pages += 1 + cursor = body["next_cursor"] + if not body["has_more"]: + return records, pages + + +def native_user(prefix="expu"): + name = unique(prefix) + uid, _, _ = register_account(name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True) + return uid, name + + +def native_post(user_uid, content="native post content here", stamp=None, **extra): + stamp = stamp or datetime.now(timezone.utc).isoformat() + uid = generate_uid() + get_table("posts").insert( + { + "uid": uid, "user_uid": user_uid, "slug": f"s-{uid[-12:]}", "title": "T", "content": content, + "topic": "random", "project_uid": None, "image": None, "stars": 0, "created_at": stamp, + "updated_at": stamp, "deleted_at": None, "deleted_by": None, **extra, + } + ) + return uid + + +def test_authz_and_kill_switch(client, admin): + assert client.get("/sync/export/posts").status_code == 401 + member = {"headers": {**admin["headers"]}} + from tests.api.sync.conftest import make_account + + plain = make_account("Member") + assert export(client, plain, "posts").status_code == 403 + set_setting("sync_enabled", "0") + try: + assert export(client, admin, "posts").status_code == 503 + finally: + set_setting("sync_enabled", "1") + assert member + + +def test_unknown_and_private_kinds_are_not_exportable(client, admin): + for kind in ("votes", "reactions", "poll_votes", "nope"): + response = export(client, admin, kind) + assert response.status_code == 404 and response.json()["error"]["reason"] == "not_exportable" + + +def test_a_post_exports_in_the_inbound_shape_and_no_private_user_field_leaks(client, admin): + uid, name = native_user() + post = native_post(uid, "exported post content here") + users, _ = pull_all(client, admin, "users") + row = next(item for item in users if item["origin_uid"] == uid) + assert row["username"] == name and row["deleted"] is False + assert not set(row) & {"email", "password_hash", "api_key", "role", "xp", "level"} + posts, _ = pull_all(client, admin, "posts") + found = next(item for item in posts if item["origin_uid"] == post) + assert found["author_uid"] == uid and found["content"] == "exported post content here" + from devplacepy.services.sync.records import PostRecord + + assert PostRecord.model_validate(found).origin_uid == post + + +def test_a_soft_deleted_post_is_a_tombstone_and_only_after_the_since_cursor(client, admin): + uid, _ = native_user() + post = native_post(uid) + before = datetime.now(timezone.utc).isoformat() + get_table("posts").update({"uid": post, "deleted_at": datetime.now(timezone.utc).isoformat(), "deleted_by": uid}, ["uid"]) + posts, _ = pull_all(client, admin, "posts", since=before) + mine = [item for item in posts if item["origin_uid"] == post] + assert len(mine) == 1 and mine[0]["deleted"] is True + + +def test_pagination_is_gapless_and_repeat_free(client, admin): + uid, _ = native_user() + base = datetime.now(timezone.utc) - timedelta(days=1) + made = {native_post(uid, f"paged post number {index}", (base + timedelta(seconds=index // 3)).isoformat()) for index in range(120)} + records, pages = pull_all(client, admin, "posts") + ids = [item["origin_uid"] for item in records] + assert len(ids) == len(set(ids)) and made <= set(ids) and pages >= 3 + + +def test_rows_that_arrived_by_sync_are_never_echoed(client, admin, origin): + sync_ok(client, admin, "users", origin, [user_record("u1", unique("echo"))]) + sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", content="synced in, must not come back out")]) + sync_ok(client, admin, "comments", origin, [comment_record("c1", "p1", "u1")]) + posts, _ = pull_all(client, admin, "posts") + assert not any(item["content"] == "synced in, must not come back out" for item in posts) + local_post = get_table("sync_links").find_one(origin=origin, kind="posts", origin_uid="p1")["local_uid"] + comments, _ = pull_all(client, admin, "comments") + assert not any(item["post_uid"] == local_post for item in comments) + + +def test_private_projects_are_left_out_and_post_links_to_them_are_cut(client, admin): + uid, _ = native_user() + public, private = generate_uid(), generate_uid() + now = datetime.now(timezone.utc).isoformat() + for project, is_private in ((public, 0), (private, 1)): + get_table("projects").insert( + {"uid": project, "user_uid": uid, "slug": f"p-{project[-10:]}", "title": "P", "description": "d", "project_type": "software", + "status": "x", "platforms": "", "website_url": "", "repo_url": "", "is_private": is_private, "read_only": 0, "stars": 0, + "created_at": now, "updated_at": now, "deleted_at": None, "deleted_by": None} + ) + linked = native_post(uid, "post that points at a private project", project_uid=private) + open_post = native_post(uid, "post that points at a public project", project_uid=public) + projects, _ = pull_all(client, admin, "projects") + ids = {item["origin_uid"] for item in projects} + assert public in ids and private not in ids + posts, _ = pull_all(client, admin, "posts") + by_id = {item["origin_uid"]: item for item in posts} + assert by_id[linked]["project_uid"] is None and by_id[open_post]["project_uid"] == public + + +def test_polls_and_follows_and_attachments_export(client, admin): + alice, _ = native_user() + bob, _ = native_user() + post = native_post(alice, "post that carries a poll for export") + now = datetime.now(timezone.utc).isoformat() + poll = generate_uid() + get_table("polls").insert({"uid": poll, "post_uid": post, "user_uid": alice, "question": "Q?", "created_at": now, "deleted_at": None, "deleted_by": None}) + first, second = generate_uid(), generate_uid() + get_table("poll_options").insert_many([ + {"uid": first, "poll_uid": poll, "label": "yes", "position": 0, "deleted_at": None, "deleted_by": None}, + {"uid": second, "poll_uid": poll, "label": "no", "position": 1, "deleted_at": None, "deleted_by": None}, + ]) + follow = generate_uid() + get_table("follows").insert({"uid": follow, "follower_uid": bob, "following_uid": alice, "created_at": now, "deleted_at": None, "deleted_by": None}) + attachment = generate_uid() + get_table("attachments").insert({"uid": attachment, "target_type": "post", "target_uid": post, "user_uid": alice, "original_filename": "a.png", + "stored_name": f"{attachment}.png", "directory": "ab/cd", "file_size": 10, "mime_type": "image/png", "created_at": now, "deleted_at": None}) + polls, _ = pull_all(client, admin, "polls") + exported = next(item for item in polls if item["origin_uid"] == poll) + assert [option["label"] for option in exported["options"]] == ["yes", "no"] + from devplacepy.services.sync.records import PollRecord + + assert PollRecord.model_validate(exported).question == "Q?" + follows, _ = pull_all(client, admin, "follows") + assert any(item["origin_uid"] == follow and item["follower_uid"] == bob for item in follows) + attachments, _ = pull_all(client, admin, "attachments") + found = next(item for item in attachments if item["origin_uid"] == attachment) + assert found["url"] == f"/static/uploads/attachments/ab/cd/{attachment}.png" and "content_b64" not in found + + +def test_bad_cursor_since_and_limit(client, admin): + assert export(client, admin, "posts", cursor="!!!").status_code == 400 + assert export(client, admin, "posts", since="yesterday-ish").status_code == 400 + body = export(client, admin, "posts", limit=1000).json() + assert body["count"] <= 50 + + +def test_the_changed_expression_uses_an_index(local_db): + from devplacepy.database import db + + plan = " ".join( + str(row) + for row in db.query( + "EXPLAIN QUERY PLAN SELECT * FROM posts t WHERE COALESCE(deleted_at, updated_at, created_at) > '2020' " + "ORDER BY COALESCE(deleted_at, updated_at, created_at), uid LIMIT 51" + ) + ) + assert "idx_posts_changed" in plan and "TEMP B-TREE" not in plan + + +def test_every_export_is_audited(client, admin): + pull_all(client, admin, "follows") + rows = list(get_table("audit_log").find(event_key="sync.export", actor_uid=admin["uid"])) + assert rows + + +@pytest.mark.parametrize("kind", ["users", "projects", "posts", "comments", "polls", "follows", "attachments"]) +def test_every_exportable_kind_answers(client, admin, kind): + body = export(client, admin, kind).json() + assert body["kind"] == kind and isinstance(body["records"], list) diff --git a/tests/api/sync/follows.py b/tests/api/sync/follows.py new file mode 100644 index 00000000..d7ecc928 --- /dev/null +++ b/tests/api/sync/follows.py @@ -0,0 +1,31 @@ +# retoor <retoor@molodetz.nl> + +from devplacepy.database import get_table +from tests.api.sync.conftest import follow_record, seed_world, sync_ok + + +def local(origin, uid): + return get_table("sync_links").find_one(origin=origin, kind="users", origin_uid=uid)["local_uid"] + + +def test_follow_unfollow_refollow_keeps_one_row_and_has_no_side_effects(client, admin, origin): + seed_world(client, admin, origin) + notifications = get_table("notifications").count() + xp = get_table("users").find_one(uid=local(origin, "u1"))["xp"] + created = sync_ok(client, admin, "follows", origin, [follow_record("f1")])["results"][0] + assert created["status"] == "created" + sync_ok(client, admin, "follows", origin, [follow_record("f1", updated_at="2025-02-01T00:00:00Z", deleted=True)]) + assert get_table("follows").find_one(uid=created["local_uid"])["deleted_by"] == "sync" + sync_ok(client, admin, "follows", origin, [follow_record("f1", updated_at="2025-03-01T00:00:00Z")]) + assert get_table("follows").count(follower_uid=local(origin, "u2"), following_uid=local(origin, "u1")) == 1 + assert get_table("follows").find_one(uid=created["local_uid"])["deleted_at"] is None + assert get_table("notifications").count() == notifications + assert get_table("users").find_one(uid=local(origin, "u1"))["xp"] == xp + + +def test_self_follow_and_unknown_users(client, admin, origin): + seed_world(client, admin, origin) + body = sync_ok(client, admin, "follows", origin, [follow_record("f1", "u1", "u1"), follow_record("f2", "ghost", "u1"), follow_record("f3", "u2", "ghost")]) + assert [(item["status"], item["reason"]) for item in body["results"]] == [ + ("rejected", "self_follow"), ("deferred", "unknown_author"), ("deferred", "unknown_author"), + ] diff --git a/tests/api/sync/pending.py b/tests/api/sync/pending.py new file mode 100644 index 00000000..dbdb7d0c --- /dev/null +++ b/tests/api/sync/pending.py @@ -0,0 +1,120 @@ +# retoor <retoor@molodetz.nl> + +import dataclasses +import random + +from devplacepy.database import get_table +from devplacepy.services.sync import processors +from devplacepy.services.sync.pending import PendingQueue +from devplacepy.services.sync.spec import LIMITS +from tests.api.sync.conftest import comment_record, post_record, sync_ok, unique, user_record + + +def link_count(origin, kind): + return get_table("sync_links").count(origin=origin, kind=kind) + + +def test_a_chain_of_waiting_records_is_released_by_the_first_dependency(client, admin, origin): + reply = sync_ok(client, admin, "comments", origin, [comment_record("c2", "p1", "u1", parent_uid="c1")])["results"][0] + parent = sync_ok(client, admin, "comments", origin, [comment_record("c1", "p1", "u1")])["results"][0] + post = sync_ok(client, admin, "posts", origin, [post_record("p1", "u1")])["results"][0] + assert (reply["status"], parent["status"], post["status"]) == ("deferred", "deferred", "deferred") + assert get_table("sync_pending").count(origin=origin) == 3 + body = sync_ok(client, admin, "users", origin, [user_record("u1", unique("chain"))]) + assert body["results"][0]["status"] == "created" + assert body["released"] == {"created": 3} + assert get_table("sync_pending").count(origin=origin) == 0 + assert [link_count(origin, kind) for kind in ("users", "posts", "comments")] == [1, 1, 2] + comments = {row["origin_uid"]: row for row in get_table("sync_links").find(origin=origin, kind="comments")} + reply_row = get_table("comments").find_one(uid=comments["c2"]["local_uid"]) + assert reply_row["parent_uid"] == comments["c1"]["local_uid"] + + +def test_any_arrival_order_ends_in_the_same_final_state(client, admin, origin): + records = [ + ("users", user_record("u1", unique("order1"))), + ("users", user_record("u2", unique("order2"))), + ("posts", post_record("p1", "u1")), + *[("comments", comment_record(f"c{index}", "p1", "u2")) for index in range(15)], + ] + random.Random(7).shuffle(records) + for kind, record in records: + sync_ok(client, admin, kind, origin, [record]) + assert [link_count(origin, kind) for kind in ("users", "posts", "comments")] == [2, 1, 15] + assert get_table("sync_pending").count(origin=origin) == 0 + + +def test_a_resend_replaces_the_waiting_row_instead_of_stacking(client, admin, origin): + for _ in range(3): + sync_ok(client, admin, "posts", origin, [post_record("p1", "ghost")]) + assert get_table("sync_pending").count(origin=origin) == 1 + + +def test_a_resend_carries_the_newest_content(client, admin, origin): + sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", content="the first version of it")]) + sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", content="the second version of it", updated_at="2025-02-01T00:00:00Z")]) + sync_ok(client, admin, "users", origin, [user_record("u1", unique("newest"))]) + link = get_table("sync_links").find_one(origin=origin, kind="posts", origin_uid="p1") + assert get_table("posts").find_one(uid=link["local_uid"])["content"] == "the second version of it" + + +def test_the_waiting_list_is_capped_per_origin_and_says_so(client, admin, origin, monkeypatch): + monkeypatch.setattr(processors, "LIMITS", dataclasses.replace(LIMITS, pending_max_per_origin=2)) + body = sync_ok(client, admin, "posts", origin, [post_record(f"p{index}", "ghost") for index in range(4)]) + assert [item["status"] for item in body["results"]] == ["deferred", "deferred", "rejected", "rejected"] + assert "waiting list" in body["results"][2]["errors"][-1] + assert get_table("sync_pending").count(origin=origin) == 2 + + +def test_the_cap_is_per_origin(client, admin, origin, monkeypatch): + monkeypatch.setattr(processors, "LIMITS", dataclasses.replace(LIMITS, pending_max_per_origin=1)) + other = f"{unique('o')}.example.org" + sync_ok(client, admin, "posts", origin, [post_record("p1", "ghost")]) + assert sync_ok(client, admin, "posts", other, [post_record("p1", "ghost")])["results"][0]["status"] == "deferred" + + +def test_a_dry_run_never_queues(client, admin, origin): + body = sync_ok(client, admin, "posts", origin, [post_record("p1", "ghost")], dry_run=True) + assert body["results"][0]["status"] == "rejected" and body["released"] == {} + assert get_table("sync_pending").count(origin=origin) == 0 + + +def test_a_tombstone_for_an_unknown_record_is_not_queued(client, admin, origin): + body = sync_ok(client, admin, "comments", origin, [comment_record("c1", "nopost", "nobody", deleted=True)]) + assert body["results"][0]["status"] == "unchanged" and body["results"][0]["reason"] == "tombstone_for_unknown" + assert get_table("sync_pending").count(origin=origin) == 0 + + +def test_an_oversize_record_is_not_queued(client, admin, origin, monkeypatch): + monkeypatch.setattr(processors, "LIMITS", dataclasses.replace(LIMITS, pending_max_record_bytes=100)) + result = sync_ok(client, admin, "posts", origin, [post_record("p1", "ghost")])["results"][0] + assert result["status"] == "rejected" and "too large" in result["errors"][-1] + + +def test_expired_and_exhausted_rows_are_dropped_hard(client, admin, origin): + sync_ok(client, admin, "posts", origin, [post_record("p1", "ghost"), post_record("p2", "ghost")]) + rows = list(get_table("sync_pending").find(origin=origin)) + get_table("sync_pending").update({"uid": rows[0]["uid"], "expires_at": "2000-01-01T00:00:00+00:00"}, ["uid"]) + get_table("sync_pending").update({"uid": rows[1]["uid"], "attempts": LIMITS.pending_max_attempts}, ["uid"]) + assert PendingQueue().sweep() >= 2 + assert get_table("sync_pending").count(origin=origin) == 0 + + +def test_the_release_budget_bounds_the_work_per_call(client, admin, origin): + for index in range(5): + sync_ok(client, admin, "posts", origin, [post_record(f"p{index}", "u1")]) + from devplacepy.database.sync import claim_link + + author = get_table("users").insert({"uid": "budget-author", "username": unique("budget"), "created_at": "2025-01-01T00:00:00+00:00"}) + claim_link(origin, "users", "u1", "budget-author", "x", "2025-01-01T00:00:00+00:00") + released = PendingQueue().release(origin, cap=2) + assert sum(released.values()) == 2 and author + assert get_table("sync_pending").count(origin=origin) == 3 + + +def test_pending_rows_of_one_origin_are_never_released_by_another(client, admin, origin): + other = f"{unique('o')}.example.org" + sync_ok(client, admin, "posts", origin, [post_record("p1", "u1")]) + sync_ok(client, admin, "users", other, [user_record("u1", unique("isolated"))]) + assert get_table("sync_pending").count(origin=origin) == 1 + assert link_count(origin, "posts") == 0 diff --git a/tests/api/sync/poll_votes.py b/tests/api/sync/poll_votes.py new file mode 100644 index 00000000..1201d4c2 --- /dev/null +++ b/tests/api/sync/poll_votes.py @@ -0,0 +1,41 @@ +# retoor <retoor@molodetz.nl> + +from devplacepy.database import get_table +from tests.api.sync.conftest import poll_record, poll_vote_record, seed_world, sync_ok + + +def setup(client, admin, origin): + seed_world(client, admin, origin) + return sync_ok(client, admin, "polls", origin, [poll_record()])["results"][0]["local_uid"] + + +def test_a_ballot_changes_option_without_duplicating(client, admin, origin): + poll = setup(client, admin, origin) + first = sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1", option="a")])["results"][0] + assert first["status"] == "created" + sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1", option="b", updated_at="2025-02-01T00:00:00Z")]) + rows = list(get_table("poll_votes").find(poll_uid=poll)) + assert len(rows) == 1 + option = get_table("poll_options").find_one(uid=rows[0]["option_uid"]) + assert option["label"] == "Spaces" + + +def test_unknown_poll_waits_but_an_unknown_option_is_final(client, admin, origin): + setup(client, admin, origin) + body = sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1", poll="ghost"), poll_vote_record("pv2", option="zzz")]) + assert (body["results"][0]["status"], body["results"][0]["reason"]) == ("deferred", "unknown_poll") + assert (body["results"][1]["status"], body["results"][1]["reason"]) == ("rejected", "unknown_option") + + +def test_one_ballot_per_voter_even_with_a_second_origin_id(client, admin, origin): + poll = setup(client, admin, origin) + sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1", option="a")]) + sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv7", option="b")]) + assert get_table("poll_votes").count(poll_uid=poll) == 1 + + +def test_clearing_a_ballot(client, admin, origin): + poll = setup(client, admin, origin) + sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1")]) + sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1", updated_at="2025-02-01T00:00:00Z", deleted=True)]) + assert get_table("poll_votes").find_one(poll_uid=poll)["deleted_by"] == "sync" diff --git a/tests/api/sync/polls.py b/tests/api/sync/polls.py new file mode 100644 index 00000000..72db54c7 --- /dev/null +++ b/tests/api/sync/polls.py @@ -0,0 +1,75 @@ +# retoor <retoor@molodetz.nl> + +from devplacepy.database import get_table +from tests.api.sync.conftest import poll_record, seed_world, sync_ok + + +def options_of(poll_uid): + return {row["label"]: row for row in get_table("poll_options").find(poll_uid=poll_uid)} + + +def test_a_poll_is_created_with_its_options(client, admin, origin): + seed_world(client, admin, origin) + result = sync_ok(client, admin, "polls", origin, [poll_record()])["results"][0] + assert result["status"] == "created" + poll = get_table("polls").find_one(uid=result["local_uid"]) + assert poll["question"] == "Tabs or spaces?" and poll["deleted_at"] is None + assert sorted(options_of(poll["uid"])) == ["Spaces", "Tabs"] + assert sorted(row["position"] for row in options_of(poll["uid"]).values()) == [0, 1] + + +def test_option_validation(client, admin, origin): + seed_world(client, admin, origin) + one = poll_record(options=[{"origin_uid": "a", "label": "Only"}]) + dup = poll_record(options=[{"origin_uid": "a", "label": "X"}, {"origin_uid": "a", "label": "Y"}]) + many = poll_record(options=[{"origin_uid": str(index), "label": f"o{index}"} for index in range(7)]) + body = sync_ok(client, admin, "polls", origin, [one, dup, many]) + assert [item["status"] for item in body["results"]] == ["rejected"] * 3 + + +def test_edit_adds_renames_and_removes_options(client, admin, origin): + seed_world(client, admin, origin) + created = sync_ok(client, admin, "polls", origin, [poll_record()])["results"][0] + edited = poll_record( + question="Tabs, spaces or both?", + options=[{"origin_uid": "a", "label": "Tabs!"}, {"origin_uid": "c", "label": "Both"}], + updated_at="2025-02-01T00:00:00Z", + ) + assert sync_ok(client, admin, "polls", origin, [edited])["results"][0]["status"] == "updated" + rows = list(get_table("poll_options").find(poll_uid=created["local_uid"])) + live = sorted(row["label"] for row in rows if not row["deleted_at"]) + assert live == ["Both", "Tabs!"] + assert [row["label"] for row in rows if row["deleted_at"]] == ["Spaces"] + assert get_table("polls").find_one(uid=created["local_uid"])["question"] == "Tabs, spaces or both?" + + +def test_one_poll_per_post(client, admin, origin): + seed_world(client, admin, origin) + sync_ok(client, admin, "polls", origin, [poll_record("po1")]) + second = sync_ok(client, admin, "polls", origin, [poll_record("po2")])["results"][0] + assert second["status"] == "rejected" and "already has a poll" in second["errors"][0] + assert get_table("sync_links").count(origin=origin, kind="polls") == 1 + + +def test_a_tombstone_removes_poll_options_and_votes_together_and_restores(client, admin, origin): + seed_world(client, admin, origin) + created = sync_ok(client, admin, "polls", origin, [poll_record()])["results"][0] + from tests.api.sync.conftest import poll_vote_record + + sync_ok(client, admin, "poll_votes", origin, [poll_vote_record()]) + gone = poll_record(updated_at="2025-02-01T00:00:00Z", deleted=True) + sync_ok(client, admin, "polls", origin, [gone]) + uid = created["local_uid"] + assert get_table("polls").find_one(uid=uid)["deleted_at"] + assert all(row["deleted_at"] for row in get_table("poll_options").find(poll_uid=uid)) + assert all(row["deleted_at"] for row in get_table("poll_votes").find(poll_uid=uid)) + back = poll_record(updated_at="2025-03-01T00:00:00Z") + sync_ok(client, admin, "polls", origin, [back]) + assert get_table("polls").find_one(uid=uid)["deleted_at"] is None + assert all(row["deleted_at"] is None for row in get_table("poll_options").find(poll_uid=uid)) + + +def test_an_unknown_post_waits(client, admin, origin): + seed_world(client, admin, origin) + result = sync_ok(client, admin, "polls", origin, [poll_record(post="nope")])["results"][0] + assert result["status"] == "deferred" and result["reason"] == "unknown_post" diff --git a/tests/api/sync/posts.py b/tests/api/sync/posts.py index f1db1850..acd9950b 100644 --- a/tests/api/sync/posts.py +++ b/tests/api/sync/posts.py @@ -9,11 +9,13 @@ def seed_author(client, admin, origin, uid="u1"): return body["results"][0]["local_uid"] -def test_unknown_author_is_rejected_with_reason(client, admin, origin): +def test_unknown_author_is_deferred_with_the_reason_and_what_it_waits_for(client, admin, origin): body = sync_ok(client, admin, "posts", origin, [post_record(author="missing")]) result = body["results"][0] - assert result["status"] == "rejected" and result["reason"] == "unknown_author" + assert result["status"] == "deferred" and result["reason"] == "unknown_author" + assert (result["needs_kind"], result["needs_uid"]) == ("users", "missing") assert get_table("sync_links").count(origin=origin) == 0 + assert get_table("sync_pending").count(origin=origin) == 1 def test_create_normalizes_and_has_no_side_effects(client, admin, origin): diff --git a/tests/api/sync/projects.py b/tests/api/sync/projects.py new file mode 100644 index 00000000..ee205c07 --- /dev/null +++ b/tests/api/sync/projects.py @@ -0,0 +1,64 @@ +# retoor <retoor@molodetz.nl> + +from devplacepy.database import get_table +from tests.api.sync.conftest import post_record, project_record, sync_ok, unique, user_record + + +def seed(client, admin, origin): + sync_ok(client, admin, "users", origin, [user_record("u1", unique("pj"))]) + + +def test_a_project_is_created_public_writable_and_without_side_effects(client, admin, origin): + seed(client, admin, origin) + before = get_table("notifications").count() + body = sync_ok(client, admin, "projects", origin, [project_record("pr1", project_type="game", stars=99)]) + result = body["results"][0] + assert result["status"] == "created" and result["ignored"] == ["stars"] + row = get_table("projects").find_one(uid=result["local_uid"]) + assert (row["is_private"], row["read_only"], row["stars"], row["project_type"]) == (0, 0, 0, "game") + assert row["slug"].endswith("synced-project") and row["deleted_at"] is None + assert get_table("notifications").count() == before + + +def test_replay_edit_and_tombstone(client, admin, origin): + seed(client, admin, origin) + created = sync_ok(client, admin, "projects", origin, [project_record("pr1")])["results"][0] + assert sync_ok(client, admin, "projects", origin, [project_record("pr1")])["results"][0]["status"] == "unchanged" + edited = sync_ok(client, admin, "projects", origin, [project_record("pr1", title="Renamed", updated_at="2025-02-01T00:00:00Z")])["results"][0] + assert edited["status"] == "updated" + assert get_table("projects").find_one(uid=created["local_uid"])["title"] == "Renamed" + gone = sync_ok(client, admin, "projects", origin, [project_record("pr1", title="Renamed", updated_at="2025-03-01T00:00:00Z", deleted=True)])["results"][0] + assert gone["status"] == "updated" + row = get_table("projects").find_one(uid=created["local_uid"]) + assert row["deleted_at"] and row["deleted_by"] == "sync" + + +def test_unknown_author_and_bad_input(client, admin, origin): + body = sync_ok(client, admin, "projects", origin, [project_record("pr1", "ghost"), project_record("pr2", title="")]) + assert body["results"][0]["status"] == "deferred" + assert body["results"][1]["status"] == "rejected" and body["results"][1]["reason"] == "invalid" + + +def test_unknown_type_falls_back_and_urls_are_cleaned(client, admin, origin): + seed(client, admin, origin) + result = sync_ok(client, admin, "projects", origin, [project_record("pr1", project_type="quantum", website_url="not a url at all", repo_url="example.org/r")])["results"][0] + row = get_table("projects").find_one(uid=result["local_uid"]) + assert row["project_type"] == "software" and row["website_url"] == "" and row["repo_url"].startswith("http") + + +def test_a_post_links_to_its_project_and_waits_for_it(client, admin, origin): + seed(client, admin, origin) + waiting = sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", project_uid="pr1")])["results"][0] + assert waiting["status"] == "deferred" and waiting["reason"] == "unknown_project" + body = sync_ok(client, admin, "projects", origin, [project_record("pr1")]) + assert body["released"] == {"created": 1} + post = get_table("posts").find_one(title=None, content="a synced post with enough content", user_uid=get_table("sync_links").find_one(origin=origin, kind="users", origin_uid="u1")["local_uid"]) + project = get_table("sync_links").find_one(origin=origin, kind="projects", origin_uid="pr1")["local_uid"] + assert post["project_uid"] == project + + +def test_a_post_without_a_project_keeps_the_old_fingerprint(client, admin, origin): + seed(client, admin, origin) + sync_ok(client, admin, "posts", origin, [post_record("p1", "u1")]) + again = sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", project_uid=None)])["results"][0] + assert again["status"] == "unchanged" diff --git a/tests/api/sync/reactions.py b/tests/api/sync/reactions.py new file mode 100644 index 00000000..6633e4f1 --- /dev/null +++ b/tests/api/sync/reactions.py @@ -0,0 +1,33 @@ +# retoor <retoor@molodetz.nl> + +from devplacepy.database import get_table +from tests.api.sync.conftest import reaction_record, seed_world, sync_ok + + +def local(origin, kind, origin_uid): + return get_table("sync_links").find_one(origin=origin, kind=kind, origin_uid=origin_uid)["local_uid"] + + +def test_reactions_are_created_toggled_and_revived(client, admin, origin): + seed_world(client, admin, origin) + created = sync_ok(client, admin, "reactions", origin, [reaction_record("r1")])["results"][0] + assert created["status"] == "created" + row = get_table("reactions").find_one(uid=created["local_uid"]) + assert (row["emoji"], row["target_type"], row["target_uid"]) == ("🔥", "post", local(origin, "posts", "p1")) + sync_ok(client, admin, "reactions", origin, [reaction_record("r1", updated_at="2025-02-01T00:00:00Z", deleted=True)]) + assert get_table("reactions").find_one(uid=created["local_uid"])["deleted_by"] == "sync" + sync_ok(client, admin, "reactions", origin, [reaction_record("r1", updated_at="2025-03-01T00:00:00Z")]) + assert get_table("reactions").find_one(uid=created["local_uid"])["deleted_at"] is None + + +def test_a_different_emoji_is_a_different_reaction_and_text_is_refused(client, admin, origin): + seed_world(client, admin, origin) + body = sync_ok(client, admin, "reactions", origin, [reaction_record("r1"), reaction_record("r2", emoji="👍"), reaction_record("r3", emoji="nope")]) + assert [item["status"] for item in body["results"]] == ["created", "created", "rejected"] + assert get_table("reactions").count(user_uid=local(origin, "users", "u2"), target_uid=local(origin, "posts", "p1")) == 2 + + +def test_reactions_on_comments_and_projects_wait_for_their_target(client, admin, origin): + seed_world(client, admin, origin) + body = sync_ok(client, admin, "reactions", origin, [reaction_record("r1", target="c1", kind="comment"), reaction_record("r2", target="pr1", kind="project"), reaction_record("r3", target="ghost")]) + assert [item["status"] for item in body["results"]] == ["created", "created", "deferred"] diff --git a/tests/api/sync/votes.py b/tests/api/sync/votes.py new file mode 100644 index 00000000..d2aa46ca --- /dev/null +++ b/tests/api/sync/votes.py @@ -0,0 +1,82 @@ +# retoor <retoor@molodetz.nl> + +from devplacepy.database import get_table +from tests.api.sync.conftest import seed_world, sync_ok, vote_record + + +def local(origin, kind, origin_uid): + return get_table("sync_links").find_one(origin=origin, kind=kind, origin_uid=origin_uid)["local_uid"] + + +def stars(table, uid): + return get_table(table).find_one(uid=uid)["stars"] + + +def test_votes_derive_stars_from_synced_voters_only(client, admin, origin): + seed_world(client, admin, origin) + post = local(origin, "posts", "p1") + body = sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2"), vote_record("v2", "u3")]) + assert [item["status"] for item in body["results"]] == ["created", "created"] + assert stars("posts", post) == 2 + sync_ok(client, admin, "votes", origin, [vote_record("v3", "u1", value=-1)]) + assert stars("posts", post) == 1 + row = get_table("votes").find_one(uid=local(origin, "votes", "v1")) + assert row["target_type"] == "post" and row["user_uid"] == local(origin, "users", "u2") + + +def test_a_changed_and_a_cleared_vote_move_the_total(client, admin, origin): + seed_world(client, admin, origin) + post = local(origin, "posts", "p1") + sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2")]) + sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2", value=-1, updated_at="2025-02-01T00:00:00Z")]) + assert stars("posts", post) == -1 + sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2", value=-1, updated_at="2025-03-01T00:00:00Z", deleted=True)]) + assert stars("posts", post) == 0 + sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2", value=1, updated_at="2025-04-01T00:00:00Z")]) + assert stars("posts", post) == 1 + assert get_table("votes").count(user_uid=local(origin, "users", "u2"), target_uid=post) == 1 + + +def test_a_second_origin_id_for_the_same_voter_revives_the_same_row(client, admin, origin): + seed_world(client, admin, origin) + post = local(origin, "posts", "p1") + sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2")]) + sync_ok(client, admin, "votes", origin, [vote_record("v9", "u2", value=-1)]) + assert get_table("votes").count(user_uid=local(origin, "users", "u2"), target_uid=post, target_type="post") == 1 + assert stars("posts", post) == -1 + + +def test_votes_on_comments_and_projects(client, admin, origin): + seed_world(client, admin, origin) + comment_votes = sync_ok(client, admin, "votes", origin, [vote_record("v1", "u3", "c1", "comment")]) + assert comment_votes["results"][0]["status"] == "created" + project = local(origin, "projects", "pr1") + sync_ok(client, admin, "votes", origin, [vote_record("v2", "u2", "pr1", "project")]) + assert stars("projects", project) == 1 + + +def test_value_target_and_voter_validation(client, admin, origin): + seed_world(client, admin, origin) + body = sync_ok( + client, admin, "votes", origin, + [vote_record("v1", value=5), vote_record("v2", kind="gist"), vote_record("v3", voter="ghost"), vote_record("v4", target="ghost")], + ) + statuses = [(item["status"], item["reason"]) for item in body["results"]] + assert statuses == [("rejected", "invalid"), ("rejected", "invalid"), ("deferred", "unknown_author"), ("deferred", "unknown_target")] + assert body["results"][3]["needs_kind"] == "posts" + + +def test_a_locally_deleted_vote_stays_deleted(client, admin, origin): + seed_world(client, admin, origin) + created = sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2")])["results"][0] + get_table("votes").update({"uid": created["local_uid"], "deleted_at": "2025-06-01T00:00:00+00:00", "deleted_by": "some-moderator"}, ["uid"]) + again = sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2", value=-1, updated_at="2025-07-01T00:00:00Z")])["results"][0] + assert again["status"] == "unchanged" and again["reason"] == "locally_deleted" + + +def test_dry_run_writes_nothing(client, admin, origin): + seed_world(client, admin, origin) + body = sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2")], dry_run=True) + assert body["results"][0]["status"] == "created" + assert get_table("votes").count(target_uid=local(origin, "posts", "p1")) == 0 + assert stars("posts", local(origin, "posts", "p1")) == 0 diff --git a/tests/e2e/admin/communities/__init__.py b/tests/e2e/admin/communities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/admin/communities/live.py b/tests/e2e/admin/communities/live.py new file mode 100644 index 00000000..5b4bf487 --- /dev/null +++ b/tests/e2e/admin/communities/live.py @@ -0,0 +1,81 @@ +# retoor <retoor@molodetz.nl> + +from datetime import datetime, timezone + +from devplacepy.database import get_table +from devplacepy.utils import generate_uid +from tests.conftest import BASE_URL + + +def make_community(slug): + uid = generate_uid() + get_table("communities").insert( + { + "uid": uid, "slug": slug, "name": slug, "query": slug, "url": f"https://{slug}.example.org", "origin": f"{slug}.example.org", + "status": "ready", "enabled": 1, "interval_seconds": 86400, "run_count": 0, "failure_count": 0, + "created_at": datetime.now(timezone.utc).isoformat(), "deleted_at": None, "deleted_by": None, "schema": "{}", + } + ) + return uid + + +def make_run(community_uid, **extra): + uid = generate_uid() + get_table("community_runs").insert( + { + "uid": uid, "community_uid": community_uid, "started_at": datetime.now(timezone.utc).isoformat(), "finished_at": "", + "status": "running", "dry_run": 0, "trigger": "admin", "collected": "", "pushed": "", "errors": "", "log": "first log line", + **extra, + } + ) + return uid + + +def test_the_run_page_updates_itself_and_stops_polling_when_the_run_ends(alice): + page, _ = alice + slug = f"live{generate_uid()[-8:]}" + community = make_community(slug) + run = make_run(community) + page.goto(f"{BASE_URL}/admin/communities/{slug}/ledger?run={run}", wait_until="domcontentloaded") + root = page.locator("[data-community-root]") + root.wait_for(state="attached") + assert root.get_attribute("data-busy") == "true" + get_table("community_runs").update( + { + "uid": run, "status": "ok", "finished_at": datetime.now(timezone.utc).isoformat(), "verdict": "ok", + "headline": "Everything synced and updated by itself.", "tally": '{"created": 2}', "log": "final log line", + }, + ["uid"], + ) + page.locator("text=Everything synced and updated by itself.").wait_for(state="visible", timeout=15000) + page.wait_for_function("document.querySelector('[data-community-root]').dataset.busy === 'false'", timeout=15000) + assert "final log line" in page.locator("pre.community-log").text_content() + + +def test_a_finished_run_page_does_not_poll(alice): + page, _ = alice + slug = f"idle{generate_uid()[-8:]}" + community = make_community(slug) + run = make_run(community, status="ok", finished_at=datetime.now(timezone.utc).isoformat(), verdict="ok", headline="Done.") + requests = [] + page.on("request", lambda request: requests.append(request.url) if "/ledger" in request.url else None) + page.goto(f"{BASE_URL}/admin/communities/{slug}/ledger?run={run}", wait_until="domcontentloaded") + assert page.locator("[data-community-root]").get_attribute("data-busy") == "false" + page.wait_for_timeout(4500) + assert len(requests) == 1 + + +def test_the_community_page_follows_a_running_run(alice): + page, _ = alice + slug = f"page{generate_uid()[-8:]}" + community = make_community(slug) + run = make_run(community) + page.goto(f"{BASE_URL}/admin/communities/{slug}", wait_until="domcontentloaded") + banner = page.locator("text=A run or a discovery is in progress") + banner.wait_for(state="visible") + get_table("community_runs").update( + {"uid": run, "status": "ok", "finished_at": datetime.now(timezone.utc).isoformat(), "verdict": "ok", "headline": "Finished while you watched."}, + ["uid"], + ) + page.locator("text=Finished while you watched.").wait_for(state="visible", timeout=15000) + assert banner.count() == 0 diff --git a/tests/e2e/game/index.py b/tests/e2e/game/index.py index 695372b7..7c6b8233 100644 --- a/tests/e2e/game/index.py +++ b/tests/e2e/game/index.py @@ -496,3 +496,31 @@ def test_mobile_no_horizontal_overflow(mobile_page): client_w = page.evaluate("document.documentElement.clientWidth") assert scroll_w - client_w <= 1, f"horizontal overflow: {scroll_w} vs {client_w}" assert page.locator("[data-game-grid]").first.is_visible() + + +def test_plot_stop_toggle_hidden_without_continuous_delivery(alice): + page, _ = alice + reset_farm("alice_test") + open_game(page) + page.select_option("[data-slot='0'] select[name='crop']", "shell") + page.click("[data-slot='0'] button:has-text('Plant')") + page.locator("[data-slot='0'].game-plot-growing").wait_for(state="visible") + assert page.locator("form[data-game-action='plot-stop']").count() == 0 + + +def test_plot_stop_toggle_flips_and_leaves_plot_empty_after_harvest(alice): + page, _ = alice + reset_farm("alice_test") + set_farm("alice_test", mastery_autoreplant=1) + open_game(page) + page.select_option("[data-slot='0'] select[name='crop']", "shell") + page.click("[data-slot='0'] button:has-text('Plant')") + toggle = page.locator("[data-slot='0'] form[data-game-action='plot-stop'] button") + toggle.wait_for(state="visible") + expect(toggle).to_contain_text("Stop after harvest") + toggle.click() + expect(toggle).to_contain_text("Resume auto-replant") + warp_ready("alice_test") + open_game(page) + page.locator("[data-slot='0'] form[data-game-action='harvest'] button").click() + page.locator("[data-slot='0'].game-plot-empty").wait_for(state="visible") diff --git a/tests/e2e/nodes/__init__.py b/tests/e2e/nodes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/nodes/extension.py b/tests/e2e/nodes/extension.py new file mode 100644 index 00000000..93980807 --- /dev/null +++ b/tests/e2e/nodes/extension.py @@ -0,0 +1,168 @@ +# retoor <retoor@molodetz.nl> + +import json +import os +import shutil +import socket +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +import pytest + +from devplacepy.database import get_table, init_db, set_consent +from devplacepy.database.nodes import create_device, save_node_settings, suspend_device, unsuspend_device +from devplacepy.services.nodes.broker import JobBroker +from devplacepy.services.nodes.policy import DestinationPolicy +from devplacepy.utils import register_account +from tests.conftest import PROJECT_ROOT + +EXTENSION_SOURCE = Path(PROJECT_ROOT) / "nodes" / "extension" +JOB_URL = "https://www.reddit.com/r/python.json" + + +def free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +@pytest.fixture(scope="module") +def donor(): + init_db() + name = f"extdonor{int(time.time() * 1000)}" + uid, _, _ = register_account(name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True) + set_consent("user", uid, "bandwidth_sharing", True) + save_node_settings(uid, {"enabled": True}) + device, secret = create_device(uid, "Browser under test", "extension") + return {"uid": uid, "device": device, "secret": secret} + + +@pytest.fixture(scope="module") +def node_server(donor): + port = free_port() + log = open(f"/tmp/node_app_{port}.log", "w") + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "--factory", "tests.e2e.nodes.node_app:build", "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], + cwd=str(PROJECT_ROOT), + env={**os.environ, "PYTHONUNBUFFERED": "1"}, + stdout=log, + stderr=subprocess.STDOUT, + ) + base = f"http://127.0.0.1:{port}" + deadline = time.time() + 30 + while time.time() < deadline: + try: + urllib.request.urlopen(f"{base}/stats", timeout=2) + break + except Exception: + if proc.poll() is not None: + raise RuntimeError(f"node app died, see /tmp/node_app_{port}.log") + time.sleep(0.3) + else: + proc.terminate() + raise RuntimeError("node app did not start") + yield base + proc.terminate() + proc.wait(timeout=10) + from devplacepy.database import refresh_snapshot, set_setting + + refresh_snapshot() + set_setting("nodes_enabled", "0") + set_setting("nodes_allow_unknown_class", "0") + + +def stats(base: str) -> dict: + return json.loads(urllib.request.urlopen(f"{base}/stats", timeout=5).read()) + + +def wait_for(predicate, seconds: float = 20.0, message: str = "condition"): + deadline = time.time() + seconds + while time.time() < deadline: + value = predicate() + if value: + return value + time.sleep(0.2) + raise AssertionError(f"{message} never became true") + + +@pytest.fixture +def extension_dir(tmp_path): + target = tmp_path / "extension" + shutil.copytree(EXTENSION_SOURCE, target, ignore=shutil.ignore_patterns("test", "node_modules", "*.zip")) + manifest = json.loads((target / "manifest.json").read_text()) + manifest["host_permissions"].append("http://127.0.0.1/*") + (target / "manifest.json").write_text(json.dumps(manifest)) + return target + + +@pytest.fixture +def browser_with_extension(playwright_instance, extension_dir, tmp_path): + context = playwright_instance.chromium.launch_persistent_context( + str(tmp_path / "profile"), + headless=False, + args=[ + "--headless=new", + f"--disable-extensions-except={extension_dir}", + f"--load-extension={extension_dir}", + "--no-sandbox", + ], + ) + worker = context.service_workers[0] if context.service_workers else context.wait_for_event("serviceworker", timeout=20000) + extension_id = worker.url.split("/")[2] + yield context, extension_id + context.close() + + +def command(page, message: dict) -> dict: + return page.evaluate("(message) => chrome.runtime.sendMessage(message)", message) + + +def popup(context, extension_id): + page = context.new_page() + page.goto(f"chrome-extension://{extension_id}/ui/popup.html", wait_until="domcontentloaded") + return page + + +def test_the_extension_connects_serves_a_job_gets_suspended_and_disconnects(node_server, donor, browser_with_extension): + context, extension_id = browser_with_extension + page = popup(context, extension_id) + assert command(page, {"cmd": "status"})["state"]["state"] == "off" + page.locator("#disclosure-card").wait_for(state="visible") + disclosure = context.new_page() + disclosure.goto(f"chrome-extension://{extension_id}/ui/disclosure.html", wait_until="domcontentloaded") + disclosure.locator("#accept").click() + disclosure.locator("#done").filter(has_text="Accepted").wait_for(state="visible") + reply = command(page, {"cmd": "connect", "serverUrl": node_server, "secret": donor["secret"], "label": "typed locally"}) + assert reply["config"]["hasSecret"] is True and reply["config"]["secretPrefix"].startswith("dpn_") + assert "dpn_" not in json.dumps(reply)[len('{"config":'):].replace(reply["config"]["secretPrefix"], "") + assert command(page, {"cmd": "enable", "enabled": True})["config"]["enabled"] is True + wait_for(lambda: stats(node_server)["connected"] == 1, message="the extension connecting") + page.reload(wait_until="domcontentloaded") + page.locator("#state").filter(has_text="Online").wait_for(state="visible", timeout=15000) + assert page.locator("#label").inner_text() == "Browser under test" + target = DestinationPolicy().check(JOB_URL, "GET", "fetch") + uid = JobBroker().enqueue("guest", "browser-e2e", target, "fetch") + + def finished(): + row = get_table("node_requests").find_one(uid=uid) + return row if row and row["status"] in ("done", "failed") else None + + row = wait_for(finished, seconds=40, message="the job being served") + assert row["mode"] == "fetch" + assert row["device_uid"] == donor["device"]["uid"] and row["device_asn_class"] == "unknown" + assert row["error"] not in ("node_disconnected", "timeout", "send_failed") + view = command(page, {"cmd": "status"}) + assert view["usage"]["requests"] + view["usage"]["failed"] >= 1 + suspend_device(donor["device"]["uid"], "browser e2e", 30) + wait_for(lambda: stats(node_server)["connected"] == 0, message="the suspension disconnecting the extension") + suspended = wait_for(lambda: (state := command(page, {"cmd": "status"})["state"])["state"] == "suspended" and state, message="the suspended state") + assert suspended["suspendedUntil"].startswith("20") + page.reload(wait_until="domcontentloaded") + page.locator("#detail").filter(has_text="Suspended until").wait_for(state="visible", timeout=10000) + unsuspend_device(donor["device"]["uid"]) + assert command(page, {"cmd": "enable", "enabled": False})["config"]["enabled"] is False + wait_for(lambda: stats(node_server)["connected"] == 0, message="the switch disconnecting") + assert command(page, {"cmd": "status"})["state"]["state"] == "off" diff --git a/tests/e2e/nodes/node_app.py b/tests/e2e/nodes/node_app.py new file mode 100644 index 00000000..d8f5436a --- /dev/null +++ b/tests/e2e/nodes/node_app.py @@ -0,0 +1,38 @@ +# retoor <retoor@molodetz.nl> + +import asyncio + +from fastapi import FastAPI + + +def build() -> FastAPI: + from devplacepy.database import init_db, refresh_snapshot, set_setting + from devplacepy.routers.nodes import router + from devplacepy.services.manager import service_manager + from devplacepy.services.nodes.service import NodesService + + app = FastAPI() + app.include_router(router, prefix="/nodes") + service = NodesService() + + @app.on_event("startup") + async def start() -> None: + init_db() + set_setting("nodes_enabled", "1") + set_setting("nodes_allow_unknown_class", "1") + service_manager.register(service) + service_manager.set_lock_owner(True) + service_manager.supervise() + service.scheduler.destination_ready = lambda target: True + await asyncio.sleep(0) + + @app.middleware("http") + async def fresh_snapshot(request, call_next): + refresh_snapshot() + return await call_next(request) + + @app.get("/stats") + async def stats() -> dict: + return {"connected": service.registry.count(), "counters": service.counters.snapshot()} + + return app diff --git a/tests/unit/database/nodes.py b/tests/unit/database/nodes.py index b8ad604d..7307a0fa 100644 --- a/tests/unit/database/nodes.py +++ b/tests/unit/database/nodes.py @@ -392,3 +392,65 @@ def test_recent_devices_is_capped_and_excludes_deleted(local_db): assert len(store.recent_devices(10_000)) <= store.ADMIN_LIST_LIMIT store.revoke_device(row["uid"], owner) assert row["uid"] not in {item["uid"] for item in store.recent_devices(store.ADMIN_LIST_LIMIT)} + + +def test_touch_devices_updates_only_the_given_devices_in_one_statement(owner): + first, _ = store.create_device(owner, "a", "native") + second, _ = store.create_device(owner, "b", "native") + assert store.touch_devices([first["uid"]]) == 1 + assert store.get_device(first["uid"])["last_seen_at"] + assert not store.get_device(second["uid"]).get("last_seen_at") + assert store.touch_devices([]) == 0 + many = [generate_uid() for _ in range(store.BATCH_SIZE + 5)] + assert store.touch_devices(many) == 0 + + +def test_class_override_round_trip(owner): + device, _ = store.create_device(owner, "a", "native") + assert store.set_class_override(device["uid"], "allow") + assert store.get_device(device["uid"])["class_override"] == "allow" + assert store.set_class_override(device["uid"], "") + assert not store.get_device(device["uid"])["class_override"] + assert not store.set_class_override(generate_uid(), "allow") + + +def test_connection_records_country_asn_and_class(owner): + device, _ = store.create_device(owner, "a", "native") + store.record_device_connection(device["uid"], platform="linux/amd64", country="nl", asn="3320", asn_class="residential") + row = store.get_device(device["uid"]) + assert (row["country"], row["asn"], row["asn_class"]) == ("NL", "3320", "residential") + + +def test_counters_accumulate_and_ignore_empty_or_non_positive(local_db): + name = f"test_counter_{generate_uid()[:8]}" + assert store.flush_counters({}) == 0 + assert store.flush_counters({name: 0}) == 0 + assert store.flush_counters({name: 3}) == 1 + assert store.flush_counters({name: 4}) == 1 + assert store.lifetime_counters(name)[name] == 7 + + +def test_lifetime_prefix_is_a_literal_prefix(local_db): + store.flush_counters({"outcome:reddit:residential:ok": 1, "outcomeXreddit:residential:ok": 1}) + found = store.lifetime_counters("outcome:") + assert "outcome:reddit:residential:ok" in found and "outcomeXreddit:residential:ok" not in found + assert store.lifetime_counters("100%") == {} + + +def test_circuit_snapshots_replace_each_other_and_are_capped(local_db): + rows = [{"key": f"destination:d{index}", "failures": 5, "state": "open", "opened_at": 1.5} for index in range(LIMITS.circuit_snapshot_max + 20)] + assert store.save_circuits(rows) == LIMITS.circuit_snapshot_max + assert len(store.load_circuits()) == LIMITS.circuit_snapshot_max + assert store.save_circuits([]) == 0 + assert store.load_circuits() == [] + + +def test_the_request_queue_cap_counts_shadows_too(local_db): + get_table("node_requests").delete() + try: + for index in range(LIMITS.max_queue_depth): + store.enqueue_request("user", "c", f"https://reddit.com/{index}", "GET", "reddit", "any") + with pytest.raises(store.QueueFull): + store.enqueue_request("shadow", "x", "https://reddit.com/z", "GET", "reddit", "fetch", shadow_of="u") + finally: + get_table("node_requests").delete() diff --git a/tests/unit/services/community/alias.py b/tests/unit/services/community/alias.py index b36b96a1..14e82496 100644 --- a/tests/unit/services/community/alias.py +++ b/tests/unit/services/community/alias.py @@ -60,11 +60,12 @@ def enable(**changes): store.update(row["uid"], sync_options=SyncOptions.from_row(row).merged(**changes).to_dict()) -def test_aliasing_off_by_default_keeps_original_names(world): +def test_aliasing_off_by_default_creates_no_identities_and_derives_from_originals(world): report = go(world) assert report.status == "ok", report.errors - assert "writer0" in local_text() + assert report.pushed["users"].get("created", 0) == 4 assert get_table("sync_identities").count(origin="forum.test") == 0 + assert "writer" in local_text().replace("wr1ter", "writer") def test_alias_run_never_stores_an_original_username(world): diff --git a/tests/unit/services/community/checkpoint.py b/tests/unit/services/community/checkpoint.py new file mode 100644 index 00000000..3d41090e --- /dev/null +++ b/tests/unit/services/community/checkpoint.py @@ -0,0 +1,152 @@ +# retoor <retoor@molodetz.nl> + +import asyncio +from dataclasses import replace + +import pytest + +from devplacepy.database import get_setting, get_table, init_db, set_setting +from devplacepy.services.community import checkpoint as checkpoint_module +from devplacepy.services.community.ledger import Ledger +from devplacepy.services.community.ledger_store import TABLE, ledger_store +from devplacepy.services.community.runner import CommunityRunner +from devplacepy.services.community.spec import LIMITS, RUN_INTERRUPTED +from devplacepy.services.community.store import iso, now, store +from tests.conftest import run_async +from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep + + +@pytest.fixture +def world(): + init_db() + previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")} + set_setting("sync_enabled", "1") + set_setting("sync_origins", "") + for table in ("communities", "community_runs", TABLE): + get_table(table).delete() + get_table("sync_links").delete(origin="forum.test") + yield + for key, value in previous.items(): + set_setting(key, value) + + +class SlowFetcher(FakeFetcher): + async def get(self, url, params=None, timeout=None, **kwargs): + await asyncio.sleep(0.12) + return await super().get(url, params, timeout, **kwargs) + + +def learned(runner): + store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum") + run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test")) + return store.get("forum-test") + + +def test_a_running_run_exposes_its_log_and_ledger_before_it_finishes(world, monkeypatch): + runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(json_community()), sleep=no_sleep) + row = learned(runner) + monkeypatch.setattr(checkpoint_module, "LIMITS", replace(LIMITS, checkpoint_seconds=0.05)) + seen = [] + original = store.update_run + + def spy(uid, **fields): + original(uid, **fields) + seen.append((get_table("community_runs").find_one(uid=uid)["status"], fields.get("log", ""), get_table(TABLE).count(run_uid=uid))) + + monkeypatch.setattr(store, "update_run", spy) + runner.fetcher_factory = lambda: SlowFetcher(json_community()) + report = run_async(runner.run(row, dry_run=False, trigger="test")) + assert seen, "no checkpoint was written while the run was going" + assert all(status == "running" for status, _, _ in seen) + assert any("collect" in log or "run[" in log for _, log, _ in seen) + final = store.run(report.run_uid) + assert final["status"] in ("ok", "partial") and final["verdict"] != "running" + assert final["items_stored"] == get_table(TABLE).count(run_uid=report.run_uid) > 0 + + +def test_checkpointed_ledger_rows_are_never_written_twice(world): + community = store.create("keep-cp", "q", "tester", name="cp") + run_uid = store.start_run(community["uid"], "test", False) + ledger = Ledger() + from devplacepy.services.community.checkpoint import RunCheckpoint + from devplacepy.services.community.logbook import LogBook + from devplacepy.services.community.pusher import PushReport + from devplacepy.services.community.runner import RunReport + + report = RunReport(status="running", dry_run=False, run_uid=run_uid, ledger=ledger) + live = PushReport(dry_run=False) + with LogBook() as book: + checkpoint = RunCheckpoint(store, ledger_store, run_uid, community["uid"], report, live, book) + for number in range(3): + ledger.add("posts", "created", origin_uid=f"a{number}") + checkpoint.flush() + checkpoint.flush() + assert get_table(TABLE).count(run_uid=run_uid) == 3 + for number in range(2): + ledger.add("posts", "rejected", reason="invalid", origin_uid=f"b{number}") + checkpoint.flush() + assert get_table(TABLE).count(run_uid=run_uid) == 5 + final = ledger_store.append(run_uid, community["uid"], ledger, checkpoint.saved) + assert final == 0 + seqs = sorted(row["seq"] for row in get_table(TABLE).find(run_uid=run_uid)) + assert seqs == [0, 1, 2, 3, 4] + + +def test_a_killed_run_is_marked_interrupted_and_keeps_what_was_saved(world): + community = store.create("keep-kill", "q", "tester", name="kill") + run_uid = store.start_run(community["uid"], "test", False) + store.update_run(run_uid, log="line one\nline two", tally={"created": 4}, items_stored=4) + assert store.reap_interrupted() == 0 + stale = iso(now().replace(year=2000)) + get_table("community_runs").update({"uid": run_uid, "started_at": stale}, ["uid"]) + assert store.reap_interrupted() == 1 + found = store.run(run_uid) + assert found["status"] == RUN_INTERRUPTED and found["verdict"] == "bad" and "Interrupted" in found["headline"] + assert found["log"] == "line one\nline two" and found["tally"] == {"created": 4} and found["finished_at"] + assert store.reap_interrupted() == 0 + assert not store.busy(community["uid"]) + + +def test_the_interrupted_verdict_is_bad(): + from devplacepy.services.community.ledger import verdict + + assert verdict("interrupted", False, {})["level"] == "bad" + + +def test_the_search_index_follows_inserts_and_prunes(world): + ledger = Ledger() + ledger.add("posts", "created", origin_uid="needle-1", label="Alpha Beta Gamma", detail="") + ledger.add("posts", "created", origin_uid="hay-2", label="Delta", detail="contains the NEEDLE word") + ledger.add("posts", "created", origin_uid="hay-3", label="100% done", detail="") + ledger_store.save("search-run", "search-community", ledger) + assert ledger_store.search_ready() + + def find(text): + rows, page = ledger_store.page({"q": text}, 1, community_uid="search-community") + scoped, _ = ledger_store.page({"q": text}, 1, run_uid="search-run") + assert sorted(row["origin_uid"] for row in scoped) == sorted(row["origin_uid"] for row in rows) + return sorted(row["origin_uid"] for row in rows) + + assert find("needle") == ["hay-2", "needle-1"] + assert find("beta g") == ["needle-1"] + assert find("100%") == ["hay-3"] + assert find("a\"b") == [] + assert find("ne") == ["hay-2", "hay-3", "needle-1"] + get_table(TABLE).delete(run_uid="search-run") + assert find("needle") == [] + + +def test_a_community_wide_text_search_goes_through_the_index(world): + from devplacepy.database import db + + ledger = Ledger() + for number in range(50): + ledger.add("posts", "created", origin_uid=f"p{number}", label=f"label {number}") + ledger_store.save("plan-run-a", "plan-community-a", ledger) + where, params, order = ledger_store.compile({"q": "label 4"}, community_uid="plan-community-a") + plan = " | ".join(row["detail"] for row in db.query(f"EXPLAIN QUERY PLAN SELECT * FROM {TABLE} WHERE {where} ORDER BY {order} LIMIT 50", **params)) + assert "community_run_items_fts" in plan + where, params, order = ledger_store.compile({"q": "la"}, community_uid="plan-community-a") + assert "community_run_items_fts" not in where and "LIKE" in where + where, params, order = ledger_store.compile({"q": "label 4"}, run_uid="plan-run-a") + assert "community_run_items_fts" not in where diff --git a/tests/unit/services/community/dedupe.py b/tests/unit/services/community/dedupe.py new file mode 100644 index 00000000..af6f3ca4 --- /dev/null +++ b/tests/unit/services/community/dedupe.py @@ -0,0 +1,92 @@ +# retoor <retoor@molodetz.nl> + +import json + +from devplacepy.services.community.generator import CodeGenerator +from devplacepy.services.community.runtime import Collector +from tests.conftest import run_async +from tests.unit.services.community.fakes import FakeFetcher +from tests.unit.services.community.generator import load, no_sleep + + +def feed(identifier, url, comments="none"): + return { + "id": identifier, + "url": url, + "type": "json_api", + "list_path": "$.items", + "post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"], "url": ["$.link"]}}, + "author": {"fields": {"username": ["$.who"]}}, + "comments": {"mode": comments}, + } + + +def schema(limits=None): + return { + "community": "twins", + "origin": "twins.test", + "limits": {"delay_seconds": 0, "retries": 1, **(limits or {})}, + "feeds": [feed("latest", "https://twins.test/latest"), feed("top", "https://twins.test/top")], + } + + +def entry(identifier, link, title="A shared story", at="2026-10-01T10:00:00Z"): + return {"id": identifier, "title": title, "body": "a body that is long enough to sync", "at": at, "who": "alice", "link": link} + + +def routes(latest, top): + return {"https://twins.test/latest": {"items": latest}, "https://twins.test/top": {"items": top}} + + +def collect(fetcher_routes, state=None, only=None, limits=None): + return run_async(Collector(FakeFetcher(fetcher_routes), schema(limits), state, sleep=no_sleep).run(only)) + + +def uids(result): + return {record["origin_uid"] for record in result["records"]["posts"]} + + +def test_the_same_story_in_a_second_feed_in_a_later_run_is_not_a_second_post(): + shared = "https://news.test/story/42?utm=a" + sources = routes([entry("L1", shared)], [entry("T9", "https://www.news.test/story/42/?utm=b")]) + first = collect(sources, only="latest") + assert len(first["records"]["posts"]) == 1 and first["state"]["links"] + second = collect(sources, state=json.loads(json.dumps(first["state"])), only="top") + assert second["records"]["posts"] == [] + + +def test_without_the_remembered_map_the_twin_would_become_a_post(): + shared = "https://news.test/story/42" + sources = routes([entry("L1", shared)], [entry("T9", shared)]) + first = collect(sources, only="latest") + assert len(collect(sources, state={"feeds": first["state"]["feeds"]}, only="top")["records"]["posts"]) == 1 + + +def test_one_feed_never_merges_its_own_posts_that_share_a_link(): + shared = "https://news.test/root" + result = collect(routes([entry("A", shared, "first"), entry("B", shared, "second")], [])) + assert len(result["records"]["posts"]) == 2 + + +def test_the_same_feed_keeps_its_own_post_when_it_changes_in_a_later_run(): + shared = "https://news.test/story/7" + first = collect(routes([entry("L1", shared)], [])) + changed = routes([entry("L1", shared, "A shared story, edited", "2026-10-05T10:00:00Z")], []) + again = collect(changed, state=first["state"]) + assert uids(again) == uids(first) and again["records"]["posts"][0]["title"].endswith("edited") + + +def test_the_remembered_map_is_bounded_and_keeps_the_newest(): + items = [entry(f"L{number}", f"https://news.test/story/{number}") for number in range(40)] + result = collect(routes(items, []), limits={"max_remembered_links": 10}) + kept = list(result["state"]["links"]) + assert len(kept) == 10 and kept[-1].endswith("/story/39") and not any(key.endswith("/story/0") for key in kept) + + +def test_a_generated_collector_remembers_links_too(tmp_path): + module = load(CodeGenerator(schema()).write(tmp_path)) + shared = "https://news.test/story/42" + sources = routes([entry("L1", shared)], [entry("T9", shared)]) + first = run_async(module.collect_all(FakeFetcher(sources), only="latest", sleep=no_sleep)) + second = run_async(module.collect_all(FakeFetcher(sources), state=json.loads(json.dumps(first["state"])), only="top", sleep=no_sleep)) + assert len(first["records"]["posts"]) == 1 and second["records"]["posts"] == [] diff --git a/tests/unit/services/community/discoverer.py b/tests/unit/services/community/discoverer.py index 9d443ec7..e14359e7 100644 --- a/tests/unit/services/community/discoverer.py +++ b/tests/unit/services/community/discoverer.py @@ -131,3 +131,44 @@ def test_feeds_listing_the_same_posts_are_learned_once_and_comment_feeds_are_ski def test_generic_probe_paths_are_skipped_when_a_feed_is_advertised(): _, fetcher = discover(rss_community(), "https://blog.test/") assert not any(request.endswith("/api/posts") or request.endswith("/rss.xml") for request in fetcher.requests) + + +def test_a_captcha_on_the_login_wall_is_reported_and_never_attempted(): + routes = club_community({}) + captcha_wall = routes["https://club.test/"]().replace("<form", '<div class="g-recaptcha" data-sitekey="k"></div><form', 1) + routes["https://club.test/"] = captcha_wall + routes["https://club.test/login"] = captcha_wall + result, fetcher = discover(routes, "https://club.test/") + assert result.status == "login_required" and "captcha" in result.error + assert result.schema["login"]["captcha"] is True and fetcher.posts == [] + result, fetcher = discover(routes, "https://club.test/", {"username": "bob", "password": "pw"}) + assert result.status == "login_required" and "captcha" in result.error and fetcher.posts == [] + + +def test_a_second_factor_with_a_secret_logs_in_and_without_one_asks_a_human(monkeypatch): + from devplacepy.services.community import runtime + from tests.unit.services.community.login import OTP_PAGE + + monkeypatch.setattr(runtime, "totp", lambda secret: "123456") + state = {} + routes = club_community(state) + password_step = routes["post:https://club.test/login"] + + def first_step(data): + answer = password_step(data) + if state.get("in"): + state["in"] = False + return (200, OTP_PAGE) + return answer + + def second_step(data): + state["in"] = data.get("otp_code") == "123456" + return (200, "welcome" if state["in"] else OTP_PAGE) + + routes["post:https://club.test/login"] = first_step + routes["post:https://club.test/2fa"] = second_step + result, _ = discover(routes, "https://club.test/", {"username": "bob", "password": "pw", "totp": "JBSWY3DPEHPK3PXP"}) + assert result.status == "ready", result.error + state.clear() + result, _ = discover(routes, "https://club.test/", {"username": "bob", "password": "pw"}) + assert result.status == "login_required" and "authenticator secret" in result.error diff --git a/tests/unit/services/community/engagement.py b/tests/unit/services/community/engagement.py new file mode 100644 index 00000000..f5afb4c9 --- /dev/null +++ b/tests/unit/services/community/engagement.py @@ -0,0 +1,199 @@ +# retoor <retoor@molodetz.nl> + +import base64 + +import pytest + +from devplacepy.database import get_setting, get_table, init_db, set_setting +from devplacepy.services.community.discoverer import CommunityDiscoverer +from devplacepy.services.community.generator import CodeGenerator +from devplacepy.services.community.learner import learn_engagement +from devplacepy.services.community.runner import CommunityRunner +from devplacepy.services.community.runtime import Collector +from devplacepy.services.community.store import store +from devplacepy.services.sync.options import SyncOptions +from devplacepy.services.sync.records import RECORD_MODELS +from tests.conftest import run_async +from tests.unit.services.community.fakes import FakeBinary, FakeFetcher +from tests.unit.services.community.generator import load, no_sleep + +PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + + +def person(number): + return {"id": 500 + number, "username": f"fan{number}"} + + +def post(number, voters=3, with_engagement=True): + item = { + "id": 9000 + number, + "title": f"Engaged post number {number}", + "content_html": f'<p>Body of engaged post {number}, long enough to sync.</p><img src="/img/p{number}.png"><a href="https://cdn.test/doc{number}.pdf">doc</a><a href="/page">page</a>', + "created_at": f"2026-10-{number:02d}T10:00:00Z", + "author": {"id": 77, "username": "writer"}, + "comments": [ + {"id": 7000 + number, "body": f"A comment on {number}", "created_at": f"2026-10-{number:02d}T11:00:00Z", "user": {"id": 88, "username": "replier"}, + "votes": [{"user": person(1), "value": 1}, {"user": person(2), "value": -1}] if with_engagement else []}, + ], + } + if with_engagement: + item["votes"] = [{"user": person(index), "value": -1 if index == 3 else 1, "created_at": f"2026-10-{number:02d}T12:00:00Z"} for index in range(1, voters + 1)] + item["reactions"] = [{"user": person(1), "emoji": "\U0001f525"}, {"user": person(2), "emoji": "\U0001f44d"}, {"user": person(3), "emoji": "nope"}] + return item + + +def routes(count=4, **kwargs): + return { + "https://eng.test/": '<html lang="en"><head><title>Eng

Join 100 members discussing code every day on this site.

', + "https://eng.test/api/posts": {"items": [post(number, **kwargs) for number in range(1, count + 1)], "total": count}, + "https://eng.test/about": "

About: 100 members.

", + **{f"https://eng.test/img/p{number}.png": FakeBinary(PNG) for number in range(1, count + 1)}, + **{f"https://cdn.test/doc{number}.pdf": FakeBinary(b"%PDF-1.4 " + b"x" * 40, "application/pdf") for number in range(1, count + 1)}, + "https://eng.test/page": "not a file", + } + + +def learned(source=None): + result = run_async(CommunityDiscoverer(FakeFetcher(source or routes())).discover("https://eng.test/api/posts")) + assert result.status == "ready", result.error + return result.schema + + +def test_the_learner_finds_per_voter_lists_and_names_the_emoji_field(): + schema = learned() + post_spec = schema["feeds"][0]["post"] + assert post_spec["votes"]["list_path"] == "$.votes" and post_spec["votes"]["author"]["fields"]["username"] == ["$.user.username"] + assert post_spec["votes"]["fields"]["value"] == ["$.value"] + assert post_spec["reactions"]["fields"]["emoji"] == ["$.emoji"] + assert schema["feeds"][0]["comments"]["votes"]["list_path"] == "$.votes" + + +def test_aggregate_counts_alone_learn_nothing(): + samples = [{"id": number, "votes": number * 3, "likes": 5, "reactions": {"fire": 2}} for number in range(1, 5)] + assert learn_engagement(samples) == {} + assert learn_engagement([{"id": 1, "votes": [{"user": {"username": "a"}}]}]) == {} + assert learn_engagement([{"id": number, "votes": [{"score": 1}, {"score": 2}]} for number in range(3)]) == {} + + +def test_the_collector_turns_voter_lists_into_records_with_users(): + schema = learned() + result = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep).run()) + records = result["records"] + assert len(records["votes"]) == 4 * 3 + 4 * 2 and len(records["reactions"]) == 4 * 2 + assert {item["value"] for item in records["votes"]} == {1, -1} + assert {item["emoji"] for item in records["reactions"]} == {"\U0001f525", "\U0001f44d"} + voters = {item["voter_uid"] for item in records["votes"]} + assert voters <= {user["origin_uid"] for user in records["users"]} + for kind in ("votes", "reactions"): + for row in records[kind]: + RECORD_MODELS[kind].model_validate(row) + assert len({item["origin_uid"] for item in records["votes"]}) == len(records["votes"]) + assert {item["target_kind"] for item in records["votes"]} == {"post", "comment"} + + +def test_attachments_are_off_by_default_and_bounded_when_on(): + schema = learned() + off = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep).run()) + assert off["records"]["attachments"] == [] + on = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True}).run()) + files = on["records"]["attachments"] + assert len(files) == 8 and {item["filename"].rsplit(".", 1)[-1] for item in files} == {"png", "pdf"} + assert base64.b64decode(next(item for item in files if item["filename"] == "p1.png")["content_b64"]) == PNG + for row in files: + RECORD_MODELS["attachments"].model_validate(row) + capped = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True, "max_attachments": 3}).run()) + assert len(capped["records"]["attachments"]) == 3 + one_each = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True, "max_attachments_per_post": 1}).run()) + assert len(one_each["records"]["attachments"]) == 4 + + +def test_missing_oversize_and_wrong_type_files_become_notices_not_failures(): + source = routes(count=2) + source["https://eng.test/img/p1.png"] = (404, "gone") + source["https://cdn.test/doc1.pdf"] = FakeBinary(b"x" * 500, "application/pdf") + source["https://eng.test/img/p2.png"] = FakeBinary(b"", "text/html") + schema = learned() + result = run_async(Collector(FakeFetcher(source), schema, sleep=no_sleep, options={"attachments": True, "max_attachment_bytes": 100}).run()) + reasons = sorted(notice["reason"] for notice in result["notices"]) + assert reasons.count("attachment_unreachable") == 2 and reasons.count("attachment_too_large") == 1 + assert result["ok"] and [item["filename"] for item in result["records"]["attachments"]] == ["doc2.pdf"] + + +def test_a_generated_collector_collects_the_same_engagement(tmp_path): + schema = learned() + module = load(CodeGenerator(schema).write(tmp_path)) + generated = run_async(module.collect_all(FakeFetcher(routes()), sleep=no_sleep, options={"attachments": True})) + direct = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True}).run()) + for kind in ("votes", "reactions", "attachments"): + assert len(generated["records"][kind]) == len(direct["records"][kind]) > 0 + + +def test_an_old_generated_file_is_regenerated_keeping_its_hand_edited_schema(tmp_path): + from devplacepy.services.community.files import CommunityFiles + from devplacepy.services.community.generator import CodeGenerator + + schema = learned() + files = CommunityFiles(tmp_path) + path = files.write("old-site", schema) + edited = path.read_text().replace(f"# generator-version: {CodeGenerator.VERSION}", "# generator-version: 1").replace('"max_posts": 150', '"max_posts": 7') + path.write_text(edited) + module = files.load("old-site", schema) + assert module.SCHEMA["limits"]["max_posts"] == 7 + assert f"# generator-version: {CodeGenerator.VERSION}" in path.read_text() + + +@pytest.fixture +def world(): + init_db() + previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins", "community_alias_enabled")} + set_setting("sync_enabled", "1") + set_setting("sync_origins", "") + for table in ("communities", "community_runs"): + get_table(table).delete() + get_table("sync_links").delete(origin="eng.test") + get_table("sync_identities").delete(origin="eng.test") + yield + for key, value in previous.items(): + set_setting(key, value) + + +def make_row(**options): + runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes()), sleep=no_sleep) + store.create("eng-test", "https://eng.test/api/posts", "tester", name="eng") + run_async(runner.discover("https://eng.test/api/posts", "tester", relearn_slug="eng-test")) + row = store.get("eng-test") + if options: + store.update(row["uid"], sync_options=SyncOptions.build(**options).to_dict()) + return runner, store.get("eng-test") + + +def test_a_live_run_syncs_votes_reactions_and_attachments(world): + runner, row = make_row(collect_attachments=True) + report = run_async(runner.run(row, dry_run=False, trigger="test")) + assert report.status in ("ok", "partial"), report.errors + assert report.pushed["votes"].get("created") == 20 and report.pushed["reactions"].get("created") == 8 + assert report.pushed["attachments"].get("created") == 8 + post_link = get_table("sync_links").find_one(origin="eng.test", kind="posts") + stars = get_table("posts").find_one(uid=post_link["local_uid"])["stars"] + assert stars == 1 + again = run_async(runner.run(store.get("eng-test"), dry_run=False, trigger="test")) + assert again.pushed.get("votes", {}).get("created", 0) == 0 + + +def test_without_the_option_no_file_is_downloaded_or_pushed(world): + runner, row = make_row() + report = run_async(runner.run(row, dry_run=False, trigger="test")) + assert report.pushed["attachments"] == {} + + +def test_voters_are_never_stored_under_pseudonyms_so_votes_wait_while_aliasing_is_on(world): + set_setting("community_alias_enabled", "1") + runner, row = make_row(alias=True, collect_attachments=True) + report = run_async(runner.run(row, dry_run=False, trigger="test")) + assert report.pushed["votes"] == {} and report.pushed["reactions"] == {} + assert report.pushed["skipped"]["alias_engagement"] == 20 + 8 + assert report.pushed["attachments"].get("created") == 8 + owners = {get_table("attachments").find_one(uid=link["local_uid"])["user_uid"] for link in get_table("sync_links").find(origin="eng.test", kind="attachments")} + assert owners and all(get_table("users").find_one(uid=uid)["username"] != "writer" for uid in owners) diff --git a/tests/unit/services/community/fakes.py b/tests/unit/services/community/fakes.py index 7c6b16fa..9424d34b 100644 --- a/tests/unit/services/community/fakes.py +++ b/tests/unit/services/community/fakes.py @@ -4,15 +4,29 @@ import json from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +class FakeBinary: + def __init__(self, data: bytes, content_type: str = "image/png"): + self.data = data + self.content_type = content_type + + class FakeResponse: def __init__(self, status: int, body): self.status = status - self.body = body if isinstance(body, str) else json.dumps(body) + self.raw = b"" self.headers = {} + if isinstance(body, FakeBinary): + self.raw, self.body = body.data, "" + self.headers = {"content-type": body.content_type} + else: + self.body = body if isinstance(body, str) else json.dumps(body) async def text(self): return self.body + async def read(self): + return self.raw or self.body.encode() + async def json(self): return json.loads(self.body) @@ -23,6 +37,8 @@ class FakeFetcher: self.requests: list[str] = [] self.session = None self.posts: list[tuple] = [] + self.last_headers = None + self.get_headers: list = [] async def __aenter__(self): return self @@ -40,6 +56,7 @@ class FakeFetcher: async def get(self, url, params=None, timeout=None, **kwargs): key = self.key(url, params) self.requests.append(key) + self.get_headers.append(kwargs.get("headers")) route = self.routes.get(key) if route is None: return FakeResponse(404, "not found") @@ -49,10 +66,12 @@ class FakeFetcher: return FakeResponse(status, body) async def post(self, url, data=None, timeout=None, **kwargs): - self.posts.append((url, dict(data or {}))) + body = dict(data or kwargs.get("json") or {}) + self.posts.append((url, body)) + self.last_headers = kwargs.get("headers") route = self.routes.get("post:" + url) if callable(route): - route = route(dict(data or {})) + route = route(body) status, body = route if isinstance(route, tuple) else (200, route or "") return FakeResponse(status, body) diff --git a/tests/unit/services/community/ledger.py b/tests/unit/services/community/ledger.py index 56a154e2..3296600f 100644 --- a/tests/unit/services/community/ledger.py +++ b/tests/unit/services/community/ledger.py @@ -63,9 +63,9 @@ def test_pusher_records_created_rejected_and_held_records_with_reasons(world): by_origin = {item.origin_uid: item for item in ledger.items} assert by_origin["u1"].outcome == "created" and by_origin["u1"].label == "ledgerone" and by_origin["u1"].local_uid assert by_origin["p1"].outcome == "created" and by_origin["p1"].label == "A good post" - assert by_origin["p2"].outcome == "rejected" and by_origin["p2"].reason == "unknown_author" and by_origin["p2"].label == "Orphan post" + assert by_origin["p2"].outcome == "deferred" and by_origin["p2"].reason == "unknown_author" and by_origin["p2"].label == "Orphan post" assert by_origin["p3"].outcome == "unchanged" and by_origin["p3"].reason == "tombstone_for_unknown" - assert tally_of(report.counts, ledger) == {"created": 2, "unchanged": 1, "rejected": 1} + assert tally_of(report.counts, ledger) == {"created": 2, "unchanged": 1, "deferred": 1} def test_repeating_the_same_batch_lists_nothing_that_is_merely_unchanged(world): diff --git a/tests/unit/services/community/login.py b/tests/unit/services/community/login.py new file mode 100644 index 00000000..8d94a703 --- /dev/null +++ b/tests/unit/services/community/login.py @@ -0,0 +1,238 @@ +# retoor + +import base64 + +import pytest + +from devplacepy.services.community import runtime +from devplacepy.services.community.discoverer import LoginLearner +from devplacepy.services.community.extract import detect_captcha, find_otp_form, parse_html, totp +from devplacepy.services.community.runtime import ( + CaptchaRequired, + Collector, + LoginFailed, + SecondFactorRequired, + code_challenge, +) +from tests.conftest import run_async +from tests.unit.services.community.fakes import FakeFetcher + + +async def no_sleep(_seconds): + return None + + +RFC_SECRET = base64.b32encode(b"12345678901234567890").decode() + + +@pytest.mark.parametrize( + "moment,expected", + [(59, "94287082"), (1111111109, "07081804"), (1111111111, "14050471"), (1234567890, "89005924"), (2000000000, "69279037"), (20000000000, "65353130")], +) +def test_totp_matches_the_rfc_6238_vectors(moment, expected): + assert totp(RFC_SECRET, moment, digits=8) == expected + + +def test_totp_default_is_six_digits_zero_padded_and_tolerates_formatting(): + code = totp(RFC_SECRET.lower(), 59) + assert len(code) == 6 and code.isdigit() + spaced = " ".join(RFC_SECRET[index : index + 4] for index in range(0, len(RFC_SECRET), 4)) + assert totp(spaced, 59) == code + assert totp(RFC_SECRET.rstrip("="), 59) == code + assert totp(RFC_SECRET, 59 + 30) != code + assert totp(RFC_SECRET, 59) == totp(RFC_SECRET, 30) + + +OTP_PAGE = """
+
""" +LOGIN_PAGE = """
+
""" +HOME_OPEN = "

Welcome back to the forum, you are signed in

" + + +def test_otp_forms_and_captcha_markers_are_recognised(): + form = find_otp_form(parse_html(OTP_PAGE), "https://x.test/login") + assert form == {"action": "https://x.test/2fa", "method": "POST", "field": "otp_code", "fields": {"csrf": "t2"}} + assert find_otp_form(parse_html(LOGIN_PAGE), "https://x.test/") is None + named = parse_html('
') + assert find_otp_form(named, "https://x.test/")["field"] == "verification_code" + hidden_token = parse_html('
') + assert find_otp_form(hidden_token, "https://x.test/") is None + assert detect_captcha('
') and detect_captcha("
") + assert not detect_captcha("

no puzzle here

") + + +def form_schema(): + return { + "community": "club", + "origin": "club.test", + "base_url": "https://club.test/", + "feeds": [], + "limits": {"delay_seconds": 0, "retries": 1}, + "login": { + "mode": "form", "url": "https://club.test/login", "action": "https://club.test/login", "method": "POST", + "fields": {"csrf": "t1"}, "username_field": "user", "password_field": "pass", "identity": "username", + }, + } + + +def club(state, challenge=True, good_code="123456"): + def login(data): + if data.get("user") == "bob" and data.get("pass") == "pw": + if challenge: + return (200, OTP_PAGE) + state["in"] = True + return (200, "ok") + return (200, LOGIN_PAGE) + + def second(data): + state["codes"] = state.get("codes", []) + [data.get("otp_code")] + state["in"] = data.get("otp_code") == good_code and data.get("csrf") == "t2" + return (200, "ok" if state["in"] else OTP_PAGE) + + return { + "https://club.test/login": LOGIN_PAGE, + "https://club.test/": lambda: HOME_OPEN if state.get("in") else LOGIN_PAGE, + "post:https://club.test/login": login, + "post:https://club.test/2fa": second, + } + + +def test_a_form_login_with_a_second_factor_submits_the_current_code(monkeypatch): + monkeypatch.setattr(runtime, "totp", lambda secret: "123456") + state = {} + collector = Collector(FakeFetcher(club(state)), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw", "totp": RFC_SECRET}) + assert run_async(collector.login()) is True + assert state["in"] and state["codes"] == ["123456"] + + +def test_a_second_factor_without_a_secret_says_what_is_missing(): + collector = Collector(FakeFetcher(club({})), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw"}) + with pytest.raises(SecondFactorRequired, match="authenticator secret"): + run_async(collector.login()) + + +def test_a_rejected_code_is_reported_not_retried(monkeypatch): + monkeypatch.setattr(runtime, "totp", lambda secret: "000000") + state = {} + collector = Collector(FakeFetcher(club(state)), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw", "totp": RFC_SECRET}) + with pytest.raises(SecondFactorRequired, match="rejected the one-time code"): + run_async(collector.login()) + assert state["codes"] == ["000000"] + + +def test_a_login_without_a_second_factor_is_unchanged(): + state = {} + collector = Collector(FakeFetcher(club(state, challenge=False)), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw"}) + assert run_async(collector.login()) is True and state["in"] + + +def test_wrong_password_still_fails_as_before(): + collector = Collector(FakeFetcher(club({})), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "nope"}) + with pytest.raises(LoginFailed, match="rejected the stored credentials"): + run_async(collector.login()) + + +def test_a_captcha_on_the_login_page_is_never_attempted(): + routes = club({}) + routes["https://club.test/login"] = LOGIN_PAGE.replace("
not json")}) + assert run_async(LoginLearner(html_only).find_json("https://api.test/", {"username": "bob", "password": "x"})) is None diff --git a/tests/unit/services/community/pagination.py b/tests/unit/services/community/pagination.py new file mode 100644 index 00000000..24c8ecb0 --- /dev/null +++ b/tests/unit/services/community/pagination.py @@ -0,0 +1,176 @@ +# retoor + +import pytest + +from devplacepy.services.community.extract import ( + find_next_page, + page_number, + parse_html, + with_page_number, +) +from devplacepy.services.community.generator import CodeGenerator +from devplacepy.services.community.runtime import Collector +from tests.conftest import run_async +from tests.unit.services.community.fakes import FakeFetcher, detail_page, html_community +from tests.unit.services.community.generator import learn, load, no_sleep, post_comment_count + +BASE = "https://board.test/t/1" + + +def page(markup: str): + return parse_html(f"
text
{markup}") + + +@pytest.mark.parametrize( + "markup,expected", + [ + ('', f"{BASE}?p=2"), + ('', f"{BASE}?p=2"), + ('', f"{BASE}?p=2"), + ('', f"{BASE}?p=2"), + ('
›
', f"{BASE}?p=2"), + ('', f"{BASE}?p=2"), + ('', f"{BASE}?page=2"), + ('arrow', f"{BASE}?p=2"), + ], +) +def test_next_page_is_found_in_every_common_markup(markup, expected): + assert find_next_page(page(markup), BASE) == expected + + +@pytest.mark.parametrize( + "markup", + [ + '

More from the author

', + '', + 'Next', + '', + 'elsewhere 2', + "", + ], +) +def test_no_next_page_is_invented(markup): + assert find_next_page(page(markup), BASE) == "" + + +def test_hints_override_the_heuristics(): + root = page('
c
') + assert find_next_page(root, BASE, {"style": "none"}) == "" + assert find_next_page(root, BASE, {"style": "next_link", "next": ".more::attr(href)"}) == f"{BASE}?custom=2" + assert find_next_page(root, BASE, {"style": "next_link", "next": ".absent::attr(href)"}) == "" + assert find_next_page(root, f"{BASE}?page=3", {"style": "page_param", "param": "page"}) == f"{BASE}?page=4" + assert find_next_page(root, BASE, {"style": "page_param"}) == f"{BASE}?page=2" + + +def test_page_numbers_round_trip(): + assert page_number("https://x.test/t?page=4") == 4 + assert page_number("https://x.test/t/page/7") == 7 + assert page_number("https://x.test/t?x=1&p=12") == 12 + assert page_number("https://x.test/t") is None + assert with_page_number("https://x.test/t?a=1&page=2", 3) == "https://x.test/t?a=1&page=3" + assert with_page_number("https://x.test/t/page/2", 3) == "https://x.test/t/page/3" + assert with_page_number("https://x.test/t", 2, "p") == "https://x.test/t?p=2" + + +def board(next_markup: str, second_next: str = "") -> dict: + routes = html_community() + routes[BASE] = detail_page("thread", 1).replace("", f"{next_markup}") + routes[f"{BASE}?cp=2"] = detail_page("thread", 1, comments=4).replace("", f"{second_next}") + return routes + + +@pytest.mark.parametrize( + "markup", + [ + '', + '
»
', + '', + ], +) +def test_collector_and_generated_file_follow_every_style(markup, tmp_path): + routes = board(markup) + schema = learn(html_community(), "https://board.test/") + result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run()) + first = next(post["origin_uid"] for post in result["records"]["posts"] if post["title"].startswith("Thread 1 ")) + assert post_comment_count(result, first) == 5 + module = load(CodeGenerator(schema).write(tmp_path)) + generated = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep)) + assert len(generated["records"]["comments"]) == len(result["records"]["comments"]) + + +def test_a_page_that_adds_no_new_comment_ends_the_walk(): + routes = board('', '') + routes[f"{BASE}?cp=3"] = routes[f"{BASE}?cp=2"].replace("cp=3", "cp=4") + routes[f"{BASE}?cp=4"] = routes[f"{BASE}?cp=2"] + schema = learn(html_community(), "https://board.test/") + fetcher = FakeFetcher(routes) + run_async(Collector(fetcher, schema, sleep=no_sleep).run()) + assert f"{BASE}?cp=3" in fetcher.requests and f"{BASE}?cp=4" not in fetcher.requests + + +def test_a_declared_no_pagination_hint_stops_after_the_first_page(): + routes = board('') + schema = learn(html_community(), "https://board.test/") + for feed in schema["feeds"]: + if (feed.get("comments") or {}).get("mode") == "page": + feed["comments"]["pagination"] = {"style": "none"} + fetcher = FakeFetcher(routes) + run_async(Collector(fetcher, schema, sleep=no_sleep).run()) + assert f"{BASE}?cp=2" not in fetcher.requests + + +def test_the_learner_records_a_selector_hint_when_one_matches(): + schema = learn(board(''), "https://board.test/") + modes = [feed["comments"] for feed in schema["feeds"] if (feed.get("comments") or {}).get("mode") == "page"] + assert modes and modes[0].get("pagination") == {"style": "next_link", "next": "a[rel=next]::attr(href)"} + + +def json_comment_schema(pagination): + comments = { + "mode": "url", + "url_template": "https://api.test/posts/{id}/comments", + "list_path": "$.items", + "fields": {"id": ["$.id"], "content": ["$.body"], "created_at": ["$.at"]}, + "author": {"fields": {"username": ["$.who"]}}, + } + if pagination: + comments["pagination"] = pagination + return { + "community": "api", + "origin": "api.test", + "limits": {"delay_seconds": 0, "retries": 1, "max_comment_pages": 3}, + "feeds": [ + { + "id": "feed1", + "url": "https://api.test/posts", + "type": "json_api", + "list_path": "$.items", + "post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"]}}, + "author": {"fields": {"username": ["$.who"]}}, + "comments": comments, + } + ], + } + + +def json_routes(): + entry = lambda n: {"id": f"c{n}", "body": f"comment number {n}", "at": "2026-10-01T10:00:00Z", "who": "alice"} + return { + "https://api.test/posts": {"items": [{"id": "p1", "title": "A post", "body": "a post body long enough", "at": "2026-10-01T10:00:00Z", "who": "alice"}]}, + "https://api.test/posts/p1/comments": {"items": [entry(1), entry(2)], "next": "/posts/p1/comments?page=2"}, + "https://api.test/posts/p1/comments?page=2": {"items": [entry(3)], "next": "/posts/p1/comments?page=3"}, + "https://api.test/posts/p1/comments?page=3": {"items": [entry(3)], "next": "/posts/p1/comments?page=4"}, + } + + +def test_json_comments_follow_a_declared_next_key_and_stop_without_news(): + fetcher = FakeFetcher(json_routes()) + result = run_async(Collector(fetcher, json_comment_schema({"style": "json_next", "path": "$.next"}), sleep=no_sleep).run()) + assert sorted(comment["content"] for comment in result["records"]["comments"]) == ["comment number 1", "comment number 2", "comment number 3"] + assert "https://api.test/posts/p1/comments?page=4" not in fetcher.requests + + +def test_json_comments_without_a_hint_read_only_the_first_page(): + fetcher = FakeFetcher(json_routes()) + result = run_async(Collector(fetcher, json_comment_schema(None), sleep=no_sleep).run()) + assert len(result["records"]["comments"]) == 2 diff --git a/tests/unit/services/community/schemacheck.py b/tests/unit/services/community/schemacheck.py new file mode 100644 index 00000000..40915c9e --- /dev/null +++ b/tests/unit/services/community/schemacheck.py @@ -0,0 +1,89 @@ +# retoor + +import copy + +import pytest + +from devplacepy.services.community.discoverer import CommunityDiscoverer +from devplacepy.services.community.schemacheck import sample_limits, validate_schema +from tests.conftest import run_async +from tests.unit.services.community.fakes import FakeFetcher, html_community, json_community + + +@pytest.fixture(scope="module") +def json_schema(): + return run_async(CommunityDiscoverer(FakeFetcher(json_community())).discover("https://forum.test/")).schema + + +@pytest.fixture(scope="module") +def html_schema(): + return run_async(CommunityDiscoverer(FakeFetcher(html_community())).discover("https://board.test/")).schema + + +def test_learned_schemas_are_valid_by_construction(json_schema, html_schema): + assert validate_schema(json_schema) == [] + assert validate_schema(html_schema) == [] + + +@pytest.mark.parametrize("bad", [None, [], "text", 4]) +def test_the_top_level_must_be_an_object(bad): + assert validate_schema(bad) == ["the schema must be a JSON object"] + + +def mutated(schema, change): + copied = copy.deepcopy(schema) + change(copied) + return validate_schema(copied) + + +def test_every_structural_mistake_is_named_with_its_path(json_schema, html_schema): + assert any("origin" in item for item in mutated(json_schema, lambda s: s.update(origin="Not A Host!"))) + assert any("community" in item for item in mutated(json_schema, lambda s: s.update(community=""))) + assert any("feeds" in item for item in mutated(json_schema, lambda s: s.update(feeds=[]))) + assert any("feeds: at most" in item for item in mutated(json_schema, lambda s: s.update(feeds=[copy.deepcopy(s["feeds"][0]) | {"id": f"f{n}"} for n in range(9)]))) + assert any("used twice" in item for item in mutated(json_schema, lambda s: s.update(feeds=[s["feeds"][0], copy.deepcopy(s["feeds"][0])]))) + assert any("feeds[0].url" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(url="ftp://x"))) + assert any("feeds[0].type" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(type="soap"))) + assert any("post.fields.id" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["post"]["fields"].pop("id"))) + assert any("content or title" in item for item in mutated(json_schema, lambda s: [s["feeds"][0]["post"]["fields"].pop(k) for k in ("content", "title")])) + assert any("$" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["post"]["fields"].update(id=["id"]))) + assert any("item_selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0].pop("item_selector"))) + assert any("unsupported" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["post"]["fields"].update(title=["h2::bogus"]))) + assert any("selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["post"]["fields"].update(title=["h2[unclosed"]))) + + +def test_comment_and_pagination_specs_are_checked(json_schema, html_schema): + assert any("comments.mode" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["comments"].update(mode="telepathy"))) + assert any("list_path" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["comments"].pop("list_path"))) + assert any("url_template" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(comments={"mode": "url", "fields": {"id": ["$.id"], "content": ["$.b"]}, "url_template": "https://x/y"}))) + assert any("item_selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["comments"].pop("item_selector"))) + assert any("pagination.style" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "psychic"}))) + assert any("param" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "page"}))) + assert any("per_page" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "offset", "param": "o"}))) + assert mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "none"})) == [] + assert any("pagination.next" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["comments"].update(pagination={"style": "next_link", "next": "a[::attr(href)"}))) + assert mutated(html_schema, lambda s: s["feeds"][0]["comments"].update(pagination={"style": "page_param", "param": "cp"})) == [] + + +def test_limits_have_ceilings_and_a_politeness_floor(json_schema): + assert any("max_posts" in item for item in mutated(json_schema, lambda s: s["limits"].update(max_posts=100000))) + assert any("delay_seconds" in item and "polite" in item for item in mutated(json_schema, lambda s: s["limits"].update(delay_seconds=0.01))) + assert any("unknown limit" in item for item in mutated(json_schema, lambda s: s["limits"].update(turbo=1))) + assert any("positive" in item for item in mutated(json_schema, lambda s: s["limits"].update(retries=-1))) + assert any("positive" in item for item in mutated(json_schema, lambda s: s["limits"].update(retries=True))) + assert mutated(json_schema, lambda s: s["limits"].update(delay_seconds=2.5, max_posts=10)) == [] + + +def test_login_specs(json_schema): + assert any("login.mode" in item for item in mutated(json_schema, lambda s: s.update(login={"mode": "magic"}))) + assert any("username_field" in item for item in mutated(json_schema, lambda s: s.update(login={"mode": "json", "url": "https://x/l"}))) + assert mutated(json_schema, lambda s: s.update(login={"mode": "json", "url": "https://x/l", "username_field": "u", "password_field": "p", "token_path": "$.t"})) == [] + + +def test_the_sample_run_is_small_and_polite(json_schema): + limits = sample_limits(json_schema) + assert limits["max_pages"] == 1 and limits["max_comment_pages"] == 1 and limits["max_posts"] <= 12 + assert limits["delay_seconds"] >= 1.0 + fast = copy.deepcopy(json_schema) + fast["limits"]["delay_seconds"] = 5.0 + assert sample_limits(fast)["delay_seconds"] == 5.0 diff --git a/tests/unit/services/community/structured.py b/tests/unit/services/community/structured.py new file mode 100644 index 00000000..b6a2e230 --- /dev/null +++ b/tests/unit/services/community/structured.py @@ -0,0 +1,100 @@ +# retoor + +import json + +import pytest + +from devplacepy.services.community.discoverer import CommunityDiscoverer +from devplacepy.services.community.extract import extract_json_ld_posts, parse_html +from devplacepy.services.community.generator import CodeGenerator +from devplacepy.services.community.runtime import Collector +from devplacepy.services.community.schemacheck import validate_schema +from devplacepy.services.sync.records import RECORD_MODELS +from tests.conftest import run_async +from tests.unit.services.community.fakes import FakeFetcher +from tests.unit.services.community.generator import load, no_sleep + + +def thread(number: int, comments: int = 2) -> dict: + return { + "@type": "DiscussionForumPosting", + "@id": f"https://ld.test/t/{number}", + "headline": f"Thread number {number} about parsing", + "text": f"This is the body of thread {number}, written long enough to sync as a post.", + "author": {"@type": "Person", "name": f"writer{number % 2}", "url": f"https://ld.test/u/{number % 2}"}, + "datePublished": f"2026-10-{number:02d}T10:00:00Z", + "dateModified": f"2026-10-{number:02d}T12:00:00Z", + "url": f"https://ld.test/t/{number}", + "comment": [ + { + "@type": "Comment", + "@id": f"https://ld.test/t/{number}#c{index}", + "text": f"Comment {index} on thread {number}", + "author": {"@type": "Person", "name": f"replier{index}"}, + "dateCreated": f"2026-10-{number:02d}T11:0{index}:00Z", + } + for index in range(comments) + ], + } + + +def page(blobs: list) -> str: + scripts = "".join(f'' for blob in blobs) + return f"LD Forum{scripts}
Loading
" + + +def ld_site(blobs=None) -> dict: + blobs = blobs or [{"@context": "https://schema.org", "@graph": [thread(number) for number in range(1, 7)]}] + return {"https://ld.test/": page(blobs), "https://ld.test/about": "

About: 100 members.

"} + + +def test_the_extractor_normalizes_graphs_lists_and_nested_comments(): + root = parse_html(page([{"@graph": [thread(1), {"@type": "WebSite", "name": "x"}]}, thread(2), [thread(3)], {"@type": "ItemList", "itemListElement": [{"item": thread(4)}]}])) + posts = extract_json_ld_posts(root) + assert [item["id"] for item in posts] == [f"https://ld.test/t/{n}" for n in (1, 2, 3, 4)] + first = posts[0] + assert first["title"] == "Thread number 1 about parsing" and first["author"] == {"username": "writer1", "id": "https://ld.test/u/1"} + assert first["created_at"].startswith("2026-10-01") and len(first["comments"]) == 2 + assert first["comments"][0]["content"] == "Comment 0 on thread 1" and first["comments"][0]["author"] == {"username": "replier0"} + + +def test_garbage_and_non_posts_are_ignored(): + root = parse_html('') + assert extract_json_ld_posts(root) == [] + nameless = parse_html(page([{"@type": "Article", "headline": "No identity at all"}])) + assert extract_json_ld_posts(nameless) == [] + + +def test_a_site_with_only_structured_data_is_learned_end_to_end(): + result = run_async(CommunityDiscoverer(FakeFetcher(ld_site())).discover("https://ld.test/")) + assert result.status == "ready", result.error + feed = result.schema["feeds"][0] + assert feed["type"] == "json_ld" and feed["post"]["fields"]["id"] and feed["comments"]["mode"] == "inline" + assert validate_schema(result.schema) == [] + records = result.sample["records"] + assert len(records["posts"]) == 6 and len(records["comments"]) == 6 and records["users"] + full = run_async(Collector(FakeFetcher(ld_site()), result.schema, sleep=no_sleep).run()) + assert len(full["records"]["comments"]) == 12 + for kind, rows in records.items(): + for row in rows: + RECORD_MODELS[kind].model_validate(row) + assert {user["username"] for user in records["users"]} >= {"writer0", "writer1", "replier0"} + + +def test_the_collector_and_the_generated_file_agree_on_structured_data(tmp_path): + schema = run_async(CommunityDiscoverer(FakeFetcher(ld_site())).discover("https://ld.test/")).schema + direct = run_async(Collector(FakeFetcher(ld_site()), schema, sleep=no_sleep).run()) + module = load(CodeGenerator(schema).write(tmp_path)) + generated = run_async(module.collect_all(FakeFetcher(ld_site()), sleep=no_sleep)) + assert {kind: len(rows) for kind, rows in direct["records"].items()} == {kind: len(rows) for kind, rows in generated["records"].items()} + + +def test_a_single_structured_post_is_not_a_feed(): + result = run_async(CommunityDiscoverer(FakeFetcher(ld_site([thread(1)]))).discover("https://ld.test/")) + assert result.status != "ready" + + +@pytest.mark.parametrize("count", [2, 5]) +def test_few_posts_still_make_a_feed(count): + result = run_async(CommunityDiscoverer(FakeFetcher(ld_site([{"@graph": [thread(number, 0) for number in range(1, count + 1)]}]))).discover("https://ld.test/")) + assert result.status == "ready", result.error diff --git a/tests/unit/services/community/submitter.py b/tests/unit/services/community/submitter.py new file mode 100644 index 00000000..9f705857 --- /dev/null +++ b/tests/unit/services/community/submitter.py @@ -0,0 +1,60 @@ +# retoor + +import pytest + +from devplacepy.services.community.extract import split_submitter +from devplacepy.services.community.runtime import Collector +from tests.conftest import run_async +from tests.unit.services.community.dedupe import entry +from tests.unit.services.community.fakes import FakeFetcher +from tests.unit.services.community.generator import no_sleep + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("lore.kernel.org via thang", "thang"), + ("jakearchibald.com by jaffathecake", "jaffathecake"), + ("decomp.dev via FedericoSchonborn", "FedericoSchonborn"), + ("via rberger 2026-10-15 12:36:38", "rberger"), + (" via spikewall", "spikewall"), + ("plainname", "plainname"), + ("two words", "two words"), + ("", ""), + ("site.org VIA Someone", "Someone"), + ], +) +def test_the_submitter_is_the_person_behind_by_or_via(raw, expected): + assert split_submitter(raw) == expected + + +def schema(): + return { + "community": "lobsters", + "origin": "lobsters.test", + "limits": {"delay_seconds": 0, "retries": 1}, + "feeds": [ + { + "id": "front", + "url": "https://lobsters.test/front", + "type": "json_api", + "list_path": "$.items", + "post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"], "url": ["$.link"]}}, + "author": {"fields": {"username": ["$.who"]}}, + "comments": {"mode": "none"}, + } + ], + } + + +def test_one_person_sharing_from_many_sites_is_one_user_with_one_identity(): + items = [ + {**entry("S1", "https://decomp.dev/a", "First"), "who": "decomp.dev via FedericoSchonborn"}, + {**entry("S2", "https://costa.jacobpalm.dk/b", "Second"), "who": "costa.jacobpalm.dk via FedericoSchonborn"}, + {**entry("S3", "https://abner.page/c", "Third"), "who": "abner.page by abnercoimbre"}, + ] + result = run_async(Collector(FakeFetcher({"https://lobsters.test/front": {"items": items}}), schema(), None, sleep=no_sleep).run(None)) + users = result["records"]["users"] + assert sorted(user["username"] for user in users) == ["FedericoSchonborn", "abnercoimbre"] + posts = {post["title"]: post["author_uid"] for post in result["records"]["posts"]} + assert posts["First"] == posts["Second"] != posts["Third"] diff --git a/tests/unit/services/devii/actions/catalog.py b/tests/unit/services/devii/actions/catalog.py index d532cf55..eaa2c6ff 100644 --- a/tests/unit/services/devii/actions/catalog.py +++ b/tests/unit/services/devii/actions/catalog.py @@ -95,6 +95,7 @@ def test_game_actions_registered_with_correct_auth(): auth = { "game_plant", "game_harvest", + "game_plot_stop", "game_buy_plot", "game_upgrade_ci", "game_water", diff --git a/tests/unit/services/game/store.py b/tests/unit/services/game/store.py index 09eb2df5..2121e310 100644 --- a/tests/unit/services/game/store.py +++ b/tests/unit/services/game/store.py @@ -1178,3 +1178,111 @@ def test_claim_grant_partial_treasury(local_db): result = store.claim_grant(user) assert result["amount"] == 300 assert store.treasury_balance() == 0 + + +# --- per-plot auto-replant pause --------------------------------------------------------- + + +def _plot(user, slot=0): + return store.get_plots(store.get_farm(user["uid"])["uid"])[slot] + + +def test_autoreplant_replants_by_default(local_db): + user = _reset("unit_a") + _set(user, mastery_autoreplant=1) + store.plant(user, 0, "shell") + _warp(user) + store.harvest(user, 0) + plot = _plot(user) + assert plot["crop_key"] == "shell" + assert not plot.get("stop_after_harvest") + + +def test_plot_stop_skips_one_replant_then_plant_resets_flag(local_db): + user = _reset("unit_a") + _set(user, mastery_autoreplant=1) + store.plant(user, 0, "shell") + assert store.set_plot_stop(user, 0, True)["stop"] is True + _warp(user) + store.harvest(user, 0) + assert not _plot(user).get("crop_key") + store.plant(user, 0, "python") + plot = _plot(user) + assert plot["crop_key"] == "python" + assert not plot.get("stop_after_harvest") + _warp(user) + store.harvest(user, 0) + assert _plot(user)["crop_key"] == "python" + + +def test_plot_stop_can_be_cancelled(local_db): + user = _reset("unit_a") + _set(user, mastery_autoreplant=1) + store.plant(user, 0, "shell") + store.set_plot_stop(user, 0, True) + store.set_plot_stop(user, 0, False) + _warp(user) + store.harvest(user, 0) + assert _plot(user)["crop_key"] == "shell" + + +def test_plot_stop_requires_continuous_delivery(local_db): + user = _reset("unit_a") + store.plant(user, 0, "shell") + with pytest.raises(GameError): + store.set_plot_stop(user, 0, True) + + +def test_plot_stop_rejects_empty_plot(local_db): + user = _reset("unit_a") + _set(user, mastery_autoreplant=1) + with pytest.raises(GameError): + store.set_plot_stop(user, 0, True) + + +def test_plot_stop_honoured_by_auto_harvest(local_db): + user = _reset("unit_a") + _set(user, mastery_autoreplant=1, legacy_autoharvest=1) + store.plant(user, 0, "shell") + store.plant(user, 1, "shell") + store.set_plot_stop(user, 0, True) + _warp(user) + store.serialize_farm(store.get_farm(user["uid"]), viewer=user, owner=user) + assert not _plot(user, 0).get("crop_key") + assert _plot(user, 1)["crop_key"] == "shell" + + +def test_autoreplant_skips_locked_crop(local_db): + user = _reset("unit_a") + _set(user, mastery_autoreplant=1) + store.plant(user, 0, "shell") + _warp(user) + original = economy.crop_lock_reason + economy.crop_lock_reason = lambda *args, **kwargs: ("level", "is locked") + try: + store.harvest(user, 0) + finally: + economy.crop_lock_reason = original + assert not _plot(user).get("crop_key") + + +def test_plot_stop_serialized_for_owner_only(local_db): + owner = _reset("unit_a") + visitor = _reset("unit_b") + _set(owner, mastery_autoreplant=1) + store.plant(owner, 0, "shell") + store.set_plot_stop(owner, 0, True) + farm = store.get_farm(owner["uid"]) + own_view = store.serialize_farm(farm, viewer=owner, owner=owner)["plots"][0] + assert own_view["can_stop_replant"] and own_view["stop_after_harvest"] + other_view = store.serialize_farm(farm, viewer=visitor, owner=owner)["plots"][0] + assert not other_view["can_stop_replant"] and not other_view["stop_after_harvest"] + + +def test_prestige_resets_plot_stop(local_db): + user = _reset("unit_a", coins=10**9) + _set(user, mastery_autoreplant=1, xp=10**7, level=economy.PRESTIGE_MIN_LEVEL) + store.plant(user, 0, "shell") + store.set_plot_stop(user, 0, True) + store.prestige(user) + assert not _plot(user).get("stop_after_harvest") diff --git a/tests/unit/services/jobs/deepsearch/crawl.py b/tests/unit/services/jobs/deepsearch/crawl.py index 949b9d11..a6234e99 100644 --- a/tests/unit/services/jobs/deepsearch/crawl.py +++ b/tests/unit/services/jobs/deepsearch/crawl.py @@ -135,3 +135,89 @@ def test_crawl_degrades_to_httpx_when_the_playwright_driver_cannot_start(monkeyp ) assert len(outcome.pages) == 1 assert outcome.pages[0].source == "httpx" + + +REDDIT_URL = "https://www.reddit.com/r/python/comments/abc/post/" +REDDIT_HTML = "A thread

" + "Real thread text from a donor. " * 30 + "

" +REDDIT_CANDIDATE = {"url": REDDIT_URL, "title": "t", "description": "", "content": "snippet " * 30} + + +def _patch_node_client(monkeypatch, handler): + import httpx + + real = crawl_module.stealth.stealth_async_client + + def factory(**kwargs): + kwargs.pop("transport", None) + return real(transport=httpx.MockTransport(handler), **kwargs) + + monkeypatch.setattr(crawl_module.stealth, "stealth_async_client", factory) + + +def test_node_servable_is_exactly_the_node_allowlist(): + assert crawl_module.node_servable(REDDIT_URL) + assert crawl_module.node_servable("https://redd.it/abc") + assert not crawl_module.node_servable("https://x.com/a/status/1") + assert not crawl_module.node_servable("https://evilreddit.com/") + assert not crawl_module.node_servable("https://reddit.com:22/") + + +def test_a_hostile_reddit_page_is_fetched_through_the_nodes_when_a_key_is_present(monkeypatch): + seen = [] + + def handler(request): + import httpx + + seen.append((request.url.path, request.headers.get("authorization"), request.content.decode())) + return httpx.Response(200, json={"url": REDDIT_URL, "status": 200, "via": "tunnel", "text": REDDIT_HTML}) + + _patch_node_client(monkeypatch, handler) + page = run_async(crawl_module._resolve_candidate(REDDIT_CANDIDATE, 1, None, "key123")) + assert page.source == "node-tunnel" and page.title == "A thread" and "Real thread text" in page.text + assert seen[0][0] == "/nodes/internal/fetch" and seen[0][1] == "Bearer key123" and "reddit.com" in seen[0][2] + + +def test_without_a_key_or_when_the_nodes_refuse_the_snippet_is_used(monkeypatch): + def handler(request): + import httpx + + return httpx.Response(503, json={"error": {"reason": "no_nodes"}}) + + _patch_node_client(monkeypatch, handler) + assert run_async(crawl_module._resolve_candidate(REDDIT_CANDIDATE, 1, None, "")).source == "search" + assert run_async(crawl_module._resolve_candidate(REDDIT_CANDIDATE, 1, None, "key")).source == "search" + + +def test_a_blocked_donor_answer_falls_back_to_the_snippet(monkeypatch): + def handler(request): + import httpx + + return httpx.Response(200, json={"url": REDDIT_URL, "status": 403, "via": "fetch", "text": "blocked " * 100}) + + _patch_node_client(monkeypatch, handler) + assert run_async(crawl_module._resolve_candidate(REDDIT_CANDIDATE, 1, None, "key")).source == "search" + + +def test_other_hostile_hosts_never_reach_the_nodes(monkeypatch): + calls = [] + + def handler(request): + import httpx + + calls.append(request.url) + return httpx.Response(200, json={}) + + _patch_node_client(monkeypatch, handler) + candidate = {"url": "https://x.com/a/status/1", "title": "t", "description": "", "content": "snippet " * 30} + assert run_async(crawl_module._resolve_candidate(candidate, 1, None, "key")).source == "search" + assert calls == [] + + +def test_a_thin_node_page_is_rejected(monkeypatch): + def handler(request): + import httpx + + return httpx.Response(200, json={"url": REDDIT_URL, "status": 200, "via": "fetch", "text": "tiny"}) + + _patch_node_client(monkeypatch, handler) + assert run_async(crawl_module.node_page(REDDIT_URL, 1, "key")) is None diff --git a/tests/unit/services/jobs/deepsearch/worker.py b/tests/unit/services/jobs/deepsearch/worker.py index a9f9c7aa..e312bd16 100644 --- a/tests/unit/services/jobs/deepsearch/worker.py +++ b/tests/unit/services/jobs/deepsearch/worker.py @@ -21,7 +21,7 @@ def _patch_pipeline(monkeypatch, pages): return [{"url": page.url, "title": page.title, "description": ""} for page in pages] async def fake_crawl( - candidates, max_pages, emit, is_cached, should_stop, query="", depth=1, seen_hashes=None + candidates, max_pages, emit, is_cached, should_stop, query="", depth=1, seen_hashes=None, api_key="" ): outcome = CrawlOutcome(seen_hashes=seen_hashes if seen_hashes is not None else set()) for page in pages[:max_pages]: @@ -176,7 +176,7 @@ def test_worker_run_performs_refinement_round_when_budget_remains(monkeypatch): return [{"url": refined_page.url, "title": refined_page.title, "description": ""}] async def fake_crawl_refinement( - candidates, max_pages, emit, is_cached, should_stop, query="", depth=1, seen_hashes=None + candidates, max_pages, emit, is_cached, should_stop, query="", depth=1, seen_hashes=None, api_key="" ): outcome = CrawlOutcome(seen_hashes=seen_hashes if seen_hashes is not None else set()) outcome.pages.append(refined_page) diff --git a/tests/unit/services/nodes/addressclass.py b/tests/unit/services/nodes/addressclass.py new file mode 100644 index 00000000..6a5032e4 --- /dev/null +++ b/tests/unit/services/nodes/addressclass.py @@ -0,0 +1,56 @@ +# retoor + +import pytest + +from devplacepy.services.nodes.addressclass import ( + CLASS_DATACENTER, + CLASS_MOBILE, + CLASS_RESIDENTIAL, + CLASS_UNKNOWN, + DATACENTER_ASNS, + ELIGIBLE_CLASSES, + classify, +) +from devplacepy.services.nodes.spec import ASN_CLASSES + + +def test_the_classes_are_exactly_the_specified_ones(): + assert set(ELIGIBLE_CLASSES) | {CLASS_DATACENTER, CLASS_UNKNOWN} == set(ASN_CLASSES) + + +@pytest.mark.parametrize( + "asn,org,expected", + [ + (24940, "Hetzner Online GmbH", CLASS_DATACENTER), + (16509, "", CLASS_DATACENTER), + (64500, "Acme Web Hosting Ltd", CLASS_DATACENTER), + (64500, "Fast Cloud Services", CLASS_DATACENTER), + (64500, "Some VPN Provider", CLASS_DATACENTER), + (64500, "Data Center Holdings", CLASS_DATACENTER), + (21928, "T-Mobile USA, Inc.", CLASS_MOBILE), + (64500, "Example Wireless", CLASS_MOBILE), + (3320, "Deutsche Telekom AG", CLASS_RESIDENTIAL), + (1136, "KPN B.V.", CLASS_RESIDENTIAL), + (0, "", CLASS_UNKNOWN), + (None, None, CLASS_UNKNOWN), + ("", "", CLASS_UNKNOWN), + ], +) +def test_classification_table(asn, org, expected): + assert classify(asn, org) == expected + + +def test_keywords_match_whole_words_only(): + assert classify(64500, "Hostingly Residential Broadband") == CLASS_RESIDENTIAL + assert classify(64500, "Cloudless Telecom") == CLASS_RESIDENTIAL + assert classify(64500, "Wirelesstown ISP") == CLASS_RESIDENTIAL + + +def test_a_known_asn_wins_over_a_residential_name(): + assert 14061 in DATACENTER_ASNS + assert classify(14061, "Friendly Home Internet") == CLASS_DATACENTER + + +def test_garbage_asn_input_is_unknown_or_name_driven(): + assert classify("not-a-number", "") == CLASS_UNKNOWN + assert classify("not-a-number", "Plain ISP") == CLASS_RESIDENTIAL diff --git a/tests/unit/services/nodes/cache.py b/tests/unit/services/nodes/cache.py index 4f202162..a7bc2940 100644 --- a/tests/unit/services/nodes/cache.py +++ b/tests/unit/services/nodes/cache.py @@ -107,3 +107,73 @@ def test_clear_empties_everything(): cache.put("GET", "u", 200, {}, b"x", False) cache.clear() assert len(cache) == 0 and cache.size_bytes == 0 + + +def test_age_moves_the_expiry_forward(): + cache, clock = make(ttl_seconds=100) + cache.put("GET", "https://r/x", 200, {}, b"x", False, age_seconds=90) + clock.now = 105 + assert cache.get("GET", "https://r/x") is not None + clock.now = 111 + assert cache.get("GET", "https://r/x") is None + + +def shared(local_db, **kwargs): + from devplacepy.database import nodes as store + from devplacepy.services.nodes.cache import SharedResultCache + + store.cache_clear() + clock = Clock() + return SharedResultCache(clock=clock, **kwargs), clock, store + + +def test_a_second_process_sees_what_the_first_stored(local_db): + first, _, store = shared(local_db, ttl_seconds=60) + assert first.put("GET", "https://r/a", 200, {"content-type": "text/plain"}, b"payload", False) + from devplacepy.services.nodes.cache import SharedResultCache + + second = SharedResultCache(ttl_seconds=60) + hit = second.get("GET", "https://r/a") + assert hit is not None and hit.body == b"payload" and hit.headers == {"content-type": "text/plain"} + assert (second.hits, second.misses) == (1, 0) + again = second.get("GET", "https://r/a") + assert again is not None and second.hits == 2 + store.cache_clear() + + +def test_the_shared_layer_honours_ttl_truncation_and_server_errors(local_db): + cache, _, store = shared(local_db, ttl_seconds=60) + assert not cache.put("GET", "https://r/b", 500, {}, b"x", False) + assert not cache.put("GET", "https://r/b", 200, {}, b"x", True) + assert store.cache_stats()["entries"] == 0 + cache.put("GET", "https://r/c", 200, {}, b"x", False) + from datetime import datetime, timedelta, timezone + + assert store.cache_get("k-not-there") is None + key = cache_key("GET", "https://r/c") + assert store.cache_get(key) is not None + assert store.cache_get(key, datetime.now(timezone.utc) + timedelta(seconds=120)) is None + store.cache_clear() + + +def test_clear_empties_both_layers(local_db): + cache, _, store = shared(local_db, ttl_seconds=60) + cache.put("GET", "https://r/d", 200, {}, b"x", False) + cache.clear() + assert len(cache) == 0 and store.cache_stats()["entries"] == 0 + + +def test_the_shared_layer_prunes_expired_rows_and_evicts_oldest_over_the_cap(local_db): + from datetime import datetime, timedelta, timezone + + cache, _, store = shared(local_db, ttl_seconds=60) + now = datetime.now(timezone.utc) + for index in range(4): + store.cache_put(f"k{index}", "GET", f"https://r/{index}", 200, "{}", "AAAA", 1000, 60, now + timedelta(seconds=index)) + store.cache_put("old", "GET", "https://r/old", 200, "{}", "AAAA", 10, 60, now - timedelta(seconds=300)) + removed = store.cache_prune(2500) + assert removed["expired"] == 1 and removed["evicted"] == 2 and removed["bytes"] == 2000 + left = store.cache_stats() + assert left["entries"] == 2 + assert store.cache_get("k0") is None and store.cache_get("k3") is not None + store.cache_clear() diff --git a/tests/unit/services/nodes/circuit.py b/tests/unit/services/nodes/circuit.py index 106c2110..823119fc 100644 --- a/tests/unit/services/nodes/circuit.py +++ b/tests/unit/services/nodes/circuit.py @@ -95,3 +95,57 @@ def test_a_zero_threshold_still_needs_one_failure(): assert breaker.allows("x") breaker.failure("x") assert not breaker.allows("x") + + +def test_snapshot_lists_open_circuits_with_time_left(): + breaker, clock = make(failures=2, open_seconds=60) + breaker.failure(("destination", "reddit.com")) + breaker.failure(("destination", "reddit.com")) + clock.now = 20 + rows = breaker.snapshot() + assert rows[0]["key"] == "destination:reddit.com" and rows[0]["state"] == "open" + assert 39 <= rows[0]["seconds_left"] <= 40 and rows[0]["failures"] == 2 + + +def test_dirty_flag_follows_every_change(): + breaker, _ = make(failures=2) + assert not breaker.dirty + breaker.failure("x") + assert breaker.dirty + breaker.dirty = False + breaker.success("x") + assert breaker.dirty + breaker.dirty = False + breaker.success("never-seen") + assert not breaker.dirty + + +def test_a_restart_keeps_an_open_circuit_open_until_its_time_is_up(): + wall = {"now": 1_000_000.0} + clock = Clock() + first = CircuitBreaker(2, 60, clock, wall=lambda: wall["now"]) + first.failure(("destination", "reddit.com")) + first.failure(("destination", "reddit.com")) + clock.now = 10 + wall["now"] += 10 + saved = first.snapshot() + second_clock = Clock() + second_clock.now = 5000 + second = CircuitBreaker(2, 60, second_clock, wall=lambda: wall["now"] + 5) + assert second.restore(saved) == 1 + assert second.state(("destination", "reddit.com")) == OPEN + second_clock.now += 60 + assert second.state(("destination", "reddit.com")) == HALF_OPEN + + +def test_restore_ignores_closed_ancient_and_malformed_rows(): + wall = {"now": 1_000_000.0} + breaker = CircuitBreaker(3, 60, Clock(), wall=lambda: wall["now"]) + rows = [ + {"key": "destination:a", "failures": 1, "opened_at": wall["now"]}, + {"key": "destination:b", "failures": 3, "opened_at": wall["now"] - 10_000}, + {"key": "nocolon", "failures": 9, "opened_at": wall["now"]}, + {"key": "destination:c", "failures": 9, "opened_at": 0}, + ] + assert breaker.restore(rows) == 0 + assert breaker.open_count() == 0 diff --git a/tests/unit/services/nodes/dispatcher.py b/tests/unit/services/nodes/dispatcher.py index eaec12b3..52acf2e6 100644 --- a/tests/unit/services/nodes/dispatcher.py +++ b/tests/unit/services/nodes/dispatcher.py @@ -19,6 +19,9 @@ from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler from devplacepy.services.nodes.session import NodeSession from devplacepy.services.nodes.spec import SPEC from devplacepy.utils import generate_uid +import asyncio +from concurrent.futures import ThreadPoolExecutor + from tests.conftest import run_async from tests.unit.services.nodes.scheduler import Clock, node, settings @@ -26,6 +29,11 @@ LIMITS = SPEC.limits TARGET_URL = "https://www.reddit.com/r/python.json" +def run_in_thread(coro): + with ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, coro).result() + + class Rig: def __init__(self): self.clock = Clock() @@ -38,12 +46,13 @@ class Rig: self.registry, self.scheduler, self.broker, self.breaker, UsageLedger(), self.counters, clock=self.clock, ) + self.dispatcher.redundancy_percent = staticmethod(lambda: 0) def connect(self, **overrides): owner = generate_uid() device, _ = store.create_device(owner, "d", "native") session = NodeSession( - node().websocket, {"uid": device["uid"], "user_uid": owner, "kind": "native"} + node().websocket, {"uid": device["uid"], "user_uid": owner, "kind": "native", "asn_class": "residential"} ) session.hello = {"modes": ["fetch"]} session.settings = settings(**overrides) @@ -413,3 +422,210 @@ def test_the_owner_exclusion_does_not_saturate_the_destination(rig): other.host_started.clear() rig.tick() assert second in mine.inflight + + +def test_the_claim_records_the_donor_address_class(rig): + session = rig.connect() + session.asn_class = "mobile" + uid = rig.enqueue() + rig.tick() + assert store.get_request(uid)["device_asn_class"] == "mobile" + + +def test_outcomes_are_observed_per_class(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + rig.answer(session, uid) + other = rig.enqueue(url="https://www.reddit.com/r/two.json") + rig.clock.now += 100 + rig.tick() + rig.answer(session, other, status=403) + third = rig.enqueue(url="https://www.reddit.com/r/three.json") + rig.clock.now += 100 + rig.tick() + rig.answer(session, third, status=404) + seen = rig.counters.outcomes() + assert seen["outcome:reddit:residential:ok"] == 2 and seen["outcome:reddit:residential:blocked"] == 1 + + +def test_a_node_failure_is_observed_but_a_donor_limit_is_not(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + run_async(session._handle({"type": "job_error", "job_id": uid, "code": "fetch_failed"})) + assert rig.counters.outcomes().get("outcome:reddit:residential:failed") == 1 + rig.clock.now += 100 + session.paused_until = 0 + second = rig.enqueue(url="https://www.reddit.com/r/two.json") + rig.tick() + run_async(session._handle({"type": "job_error", "job_id": second, "code": "outside_hours"})) + assert rig.counters.outcomes().get("outcome:reddit:residential:failed") == 1 + + +def test_on_battery_pauses_the_donor_without_blaming_it(rig): + session = rig.connect() + uid = rig.enqueue() + rig.tick() + run_async(session._handle({"type": "job_error", "job_id": uid, "code": "on_battery"})) + assert session.paused_until > rig.clock() + assert not rig.breaker.failures or rig.breaker.allows(("node", session.device_uid)) + assert store.get_request(uid)["error"] == "on_battery" + + +class SampleAll: + def __init__(self, rig): + rig.dispatcher.redundancy_percent = staticmethod(lambda: 100) + + +def mismatch_flags(device_uid=None): + return [ + flag + for flag in store.list_flags() + if flag["kind"] == "redundancy_mismatch" and (device_uid is None or flag["device_uid"] == device_uid) + ] + + +def sent_to(session): + return [frame["job_id"] for frame in session.websocket.sent if frame["type"] == "job"] + + +def test_a_sampled_fetch_result_is_cross_checked_on_a_different_donor(rig): + SampleAll(rig) + first, second = rig.connect(), rig.connect() + uid = rig.enqueue() + rig.tick() + holder = first if uid in sent_to(first) else second + other = second if holder is first else first + rig.answer(holder, uid, headers={"content-type": "application/json"}, body_b64=base64.b64encode(b'{"kind":"Listing","data":{}}').decode()) + shadows = list(get_table("node_requests").find(shadow_of=uid)) + assert len(shadows) == 1 and shadows[0]["exclude_device"] == holder.device_uid + assert shadows[0]["caller_kind"] == "shadow" + rig.clock.now += 100 + assert rig.tick() == 1 + assert sent_to(other) and shadows[0]["uid"] in sent_to(other) + rig.answer(other, shadows[0]["uid"], headers={"content-type": "application/json"}, body_b64=base64.b64encode(b'{"kind":"Listing","data":{}}').decode()) + counts = rig.counters.snapshot() + assert counts["redundancy_checked"] == 1 and counts["redundancy_matched"] == 1 + assert not mismatch_flags(other.device_uid) + + +def test_a_forged_second_opinion_raises_a_flag_and_no_ban(rig): + SampleAll(rig) + first, second = rig.connect(), rig.connect() + uid = rig.enqueue() + rig.tick() + holder = first if uid in sent_to(first) else second + other = second if holder is first else first + rig.answer(holder, uid, headers={"content-type": "application/json"}, body_b64=base64.b64encode(b'{"kind":"Listing","data":{}}').decode()) + shadow = list(get_table("node_requests").find(shadow_of=uid))[0]["uid"] + rig.clock.now += 100 + rig.tick() + rig.answer(other, shadow, headers={"content-type": "application/json"}, body_b64=base64.b64encode(b'{"buy":"now"}').decode()) + assert rig.counters.snapshot()["redundancy_mismatches"] == 1 + flags = mismatch_flags(other.device_uid) + assert len(flags) == 1 + assert effective_status_of(other) == "active" + + +def effective_status_of(session): + return store.effective_status(store.get_device(session.device_uid)) + + +def test_a_shadow_never_spends_the_caller_quota_and_is_not_cross_checked_again(rig): + SampleAll(rig) + first, second = rig.connect(), rig.connect() + uid = rig.enqueue() + rig.tick() + holder = first if uid in sent_to(first) else second + other = second if holder is first else first + rig.answer(holder, uid) + shadow = list(get_table("node_requests").find(shadow_of=uid))[0]["uid"] + rig.clock.now += 100 + rig.tick() + rig.answer(other, shadow) + assert get_table("node_requests").count(shadow_of={"!=": ""}) == 1 + + +def test_sampling_is_deterministic_and_close_to_the_requested_rate(): + uids = [generate_uid() for _ in range(10000)] + chosen = [uid for uid in uids if NodeDispatcher.sampled(uid, 5)] + assert chosen == [uid for uid in uids if NodeDispatcher.sampled(uid, 5)] + assert 350 <= len(chosen) <= 650 + assert not any(NodeDispatcher.sampled(uid, 0) for uid in uids[:500]) + assert all(NodeDispatcher.sampled(uid, 100) for uid in uids[:500]) + + +def test_the_percent_is_clamped_to_the_spec_ceiling(local_db): + from devplacepy.database import set_setting + + set_setting("nodes_redundancy_percent", "99") + try: + assert NodeDispatcher.redundancy_percent() == LIMITS.redundancy_max_percent + set_setting("nodes_redundancy_percent", "-4") + assert NodeDispatcher.redundancy_percent() == 0 + finally: + set_setting("nodes_redundancy_percent", str(LIMITS.redundancy_percent)) + + +def test_a_missing_original_is_inconclusive_not_a_flag(rig): + session = rig.connect() + rig.dispatcher._compare_shadow(session, type("J", (), {"shadow_of": "gone", "destination": "reddit.com"})(), 200, {}, b"x") + assert rig.counters.snapshot()["redundancy_inconclusive"] == 1 + assert not mismatch_flags(session.device_uid) + + +def test_a_waiter_in_the_dispatcher_process_wakes_without_polling(rig): + import asyncio + + from devplacepy.services.nodes.wake import HUB + + async def scenario(): + HUB.attach() + target = DestinationPolicy().check(TARGET_URL) + uid = rig.broker.enqueue("user", "c1", target, "any") + store.claim_request(uid, "d1") + loop = asyncio.get_running_loop() + loop.call_later(0.02, lambda: rig.broker.complete(uid, 200, {}, b"ok", False)) + started = time.monotonic() + result = await rig.broker.wait(uid, 5) + return result, time.monotonic() - started + + try: + result, elapsed = run_in_thread(scenario()) + finally: + HUB.enabled = True + assert result.ok and result.body == b"ok" + assert elapsed < LIMITS.poll_interval_ms / 1000 + + +def test_a_waiter_without_the_hub_still_finishes_by_polling(rig): + import asyncio + + from devplacepy.services.nodes.wake import HUB + + async def scenario(): + target = DestinationPolicy().check(TARGET_URL) + uid = rig.broker.enqueue("user", "c2", target, "any") + store.claim_request(uid, "d1") + asyncio.get_running_loop().call_later(0.02, lambda: rig.broker.complete(uid, 200, {}, b"ok", False)) + return await rig.broker.wait(uid, 5) + + HUB._loop = None + assert run_in_thread(scenario()).ok + + +def test_enqueue_signals_the_dispatcher(rig): + import asyncio + + from devplacepy.services.nodes.wake import HUB + + async def scenario(): + HUB.attach() + target = DestinationPolicy().check(TARGET_URL) + woke = asyncio.ensure_future(HUB.wait_queued(5)) + await asyncio.sleep(0) + rig.broker.enqueue("user", "c3", target, "any") + return await asyncio.wait_for(woke, 1) + + assert run_in_thread(scenario()) is True diff --git a/tests/unit/services/nodes/fabric.py b/tests/unit/services/nodes/fabric.py index 8f61c85e..d044bb3f 100644 --- a/tests/unit/services/nodes/fabric.py +++ b/tests/unit/services/nodes/fabric.py @@ -60,7 +60,9 @@ def online(local_db): set_setting("nodes_daily_quota_user", "1000") set_setting("nodes_daily_quota_guest", "0") heartbeat(3) + store.cache_clear() yield + store.cache_clear() set_setting("nodes_enabled", "0") get_table("service_state").delete(name="nodes") diff --git a/tests/unit/services/nodes/geo.py b/tests/unit/services/nodes/geo.py new file mode 100644 index 00000000..fd8b1362 --- /dev/null +++ b/tests/unit/services/nodes/geo.py @@ -0,0 +1,126 @@ +# retoor + +import gzip +from pathlib import Path + +import pytest + +from devplacepy.services.nodes import geo +from devplacepy.services.nodes.geo import GeoDatabase, GeoError, build, pack + +ASN_ROWS = [ + ("1.0.0.0", "1.0.0.255", 13335, "Cloudflare, Inc."), + ("8.8.8.0", "8.8.8.255", 15169, "Google LLC"), + ("20.0.0.0", "20.0.0.255", 64500, "Home Fiber BV"), + ("2001:db8::", "2001:db8::ffff", 64501, "Mobile Wireless NV"), +] +COUNTRY_ROWS = [ + ("1.0.0.0", "1.0.0.255", "AU"), + ("8.8.8.0", "8.8.8.255", "US"), + ("20.0.0.0", "20.0.0.255", "NL"), + ("2001:db8::", "2001:db8::ffff", "ZZ"), +] + + +def lines(rows): + return [",".join(f'"{part}"' if "," in str(part) else str(part) for part in row) for row in rows] + + +def make(tmp_path, monkeypatch, checks=()): + monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=1, min_country_rows=1, checks=checks)) + target = tmp_path / "geo.db" + built = build(lines(ASN_ROWS), lines(COUNTRY_ROWS), target, "2026-10") + geo.install(built, target) + return GeoDatabase(target) + + +def test_pack_normalises_v4_mapped_and_rejects_garbage(): + assert pack("::ffff:8.8.8.8") == pack("8.8.8.8") + assert pack("[2001:db8::1]")[0] == 6 + with pytest.raises(ValueError): + pack("nope") + + +def test_lookup_hits_first_last_and_middle_of_a_range(tmp_path, monkeypatch): + database = make(tmp_path, monkeypatch) + for address in ("1.0.0.0", "1.0.0.77", "1.0.0.255"): + found = database.lookup(address) + assert (found.asn, found.country, found.asn_class) == (13335, "AU", "datacenter") + + +def test_lookup_between_and_outside_ranges_is_unknown(tmp_path, monkeypatch): + database = make(tmp_path, monkeypatch) + for address in ("1.0.1.0", "0.255.255.255", "9.9.9.9", "255.255.255.255"): + assert database.lookup(address).asn_class == "unknown" + + +def test_ipv6_and_zz_country(tmp_path, monkeypatch): + database = make(tmp_path, monkeypatch) + found = database.lookup("2001:db8::1234") + assert found.asn == 64501 and found.country == "" and found.asn_class == "mobile" + assert database.lookup("2001:db9::1").asn == 0 + + +def test_classification_of_a_residential_range(tmp_path, monkeypatch): + assert make(tmp_path, monkeypatch).lookup("20.0.0.9").asn_class == "residential" + + +def test_missing_file_and_bad_input_degrade_to_unknown(tmp_path): + database = GeoDatabase(tmp_path / "absent.db") + assert not database.available() + assert database.lookup("8.8.8.8").asn_class == "unknown" + assert database.status()["available"] is False + assert GeoDatabase(tmp_path / "absent.db").lookup("").asn_class == "unknown" + + +def test_a_too_small_dataset_is_refused_and_leaves_no_file(tmp_path, monkeypatch): + monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=10, min_country_rows=10, checks=())) + target = tmp_path / "geo.db" + with pytest.raises(GeoError): + build(lines(ASN_ROWS), lines(COUNTRY_ROWS), target, "2026-10") + assert not target.exists() and not target.with_suffix(".tmp").exists() + + +def test_a_failed_sanity_check_refuses_and_keeps_the_current_database(tmp_path, monkeypatch): + current = make(tmp_path, monkeypatch) + before = current.path.read_bytes() + monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=1, min_country_rows=1, checks=(("8.8.8.8", 99999),))) + with pytest.raises(GeoError): + build(lines(ASN_ROWS), lines(COUNTRY_ROWS), tmp_path / "geo.db", "2026-11") + assert current.path.read_bytes() == before + + +def test_the_row_cap_is_enforced(tmp_path, monkeypatch): + monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(max_rows=2, min_asn_rows=1, min_country_rows=1, checks=())) + with pytest.raises(GeoError): + build(lines(ASN_ROWS), lines(COUNTRY_ROWS), tmp_path / "geo.db", "2026-10") + + +def test_malformed_rows_are_skipped_not_fatal(tmp_path, monkeypatch): + monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=1, min_country_rows=1, checks=())) + asn = lines(ASN_ROWS) + ["garbage", "a,b,c,d", "1.2.3.4,1.2.3.5,notanumber,x"] + built = build(asn, lines(COUNTRY_ROWS), tmp_path / "geo.db", "2026-10") + assert built["asn_rows"] == len(ASN_ROWS) + + +def test_status_reports_metadata_and_attribution(tmp_path, monkeypatch): + status = make(tmp_path, monkeypatch).status() + assert status["available"] and status["month"] == "2026-10" and "DB-IP" in status["attribution"] + + +def test_months_walk_back_over_a_year_boundary(): + from datetime import datetime, timezone + + assert geo.months(datetime(2026, 1, 15, tzinfo=timezone.utc)) == ["2026-01", "2025-12", "2025-11"] + + +def test_build_from_files_reads_gzip(tmp_path, monkeypatch): + monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=1, min_country_rows=1, checks=())) + asn_file, country_file = tmp_path / "a.csv.gz", tmp_path / "c.csv.gz" + with gzip.open(asn_file, "wt") as handle: + handle.write("\n".join(lines(ASN_ROWS))) + with gzip.open(country_file, "wt") as handle: + handle.write("\n".join(lines(COUNTRY_ROWS))) + built = geo.build_from_files(asn_file, country_file, tmp_path / "geo.db", "2026-10") + geo.install(built, tmp_path / "geo.db") + assert GeoDatabase(tmp_path / "geo.db").lookup("8.8.8.8").asn == 15169 diff --git a/tests/unit/services/nodes/registry.py b/tests/unit/services/nodes/registry.py new file mode 100644 index 00000000..43da6bc7 --- /dev/null +++ b/tests/unit/services/nodes/registry.py @@ -0,0 +1,55 @@ +# retoor + +import pytest + +from devplacepy.services.nodes.registry import ( + COUNTER_NAMES, + NodeCounters, + outcome_name, +) + + +def test_inc_counts_both_the_live_view_and_the_pending_delta(): + counters = NodeCounters() + counters.inc("kicked") + counters.inc("kicked") + assert counters.snapshot()["kicked"] == 2 + assert counters.take_pending() == {"kicked": 2} + assert counters.take_pending() == {} + assert counters.snapshot()["kicked"] == 2 + + +def test_unknown_names_are_rejected(): + counters = NodeCounters() + with pytest.raises(KeyError): + counters.inc("nope") + + +def test_restore_pending_adds_back_what_a_failed_flush_took(): + counters = NodeCounters() + counters.inc("jobs_done") + taken = counters.take_pending() + counters.inc("jobs_done") + counters.restore_pending(taken) + assert counters.take_pending() == {"jobs_done": 2} + + +def test_outcomes_are_validated_and_bounded_in_shape(): + counters = NodeCounters() + counters.observe("reddit", "residential", "ok") + counters.observe("reddit", "residential", "ok") + counters.observe("reddit", "", "blocked") + assert counters.outcomes() == { + "outcome:reddit:residential:ok": 2, + "outcome:reddit:unknown:blocked": 1, + } + for adapter, asn_class, kind in (("Re:dit", "x", "ok"), ("reddit", "a b", "ok"), ("reddit", "x", "weird"), ("x" * 40, "x", "ok")): + with pytest.raises(KeyError): + outcome_name(adapter, asn_class, kind) + + +def test_every_declared_counter_can_be_incremented(): + counters = NodeCounters() + for name in COUNTER_NAMES: + counters.inc(name) + assert all(value == 1 for value in counters.snapshot().values()) diff --git a/tests/unit/services/nodes/scheduler.py b/tests/unit/services/nodes/scheduler.py index 10acfd38..c228e350 100644 --- a/tests/unit/services/nodes/scheduler.py +++ b/tests/unit/services/nodes/scheduler.py @@ -43,7 +43,7 @@ def settings(**overrides): def node(uid="d1", user="u1", modes=("fetch",), **overrides): - session = NodeSession(FakeSocket(), {"uid": uid, "user_uid": user, "kind": "native"}) + session = NodeSession(FakeSocket(), {"uid": uid, "user_uid": user, "kind": "native", "asn_class": "residential"}) session.hello = {"modes": list(modes)} session.settings = settings(**overrides) session.roll_day() @@ -281,3 +281,48 @@ def test_choose_applies_the_owner_exclusion_to_a_shared_candidate_list(world): assert world.choose(found, "bob") is mine assert world.choose([mine], "alice") is None assert world.choose([], "") is None + + +def test_a_datacenter_donor_is_not_scheduled(world): + session = add(world) + session.asn_class = "datacenter" + assert world.pick(TARGET) is None + + +def test_an_unknown_class_is_refused_unless_the_site_allows_it(world, local_db): + from devplacepy.database import set_setting + + session = add(world) + session.asn_class = "unknown" + assert world.pick(TARGET) is None + set_setting("nodes_allow_unknown_class", "1") + try: + assert world.pick(TARGET) is session + session.asn_class = "datacenter" + assert world.pick(TARGET) is None + finally: + set_setting("nodes_allow_unknown_class", "0") + + +def test_mobile_and_residential_are_eligible(world): + for index, asn_class in enumerate(("mobile", "residential")): + session = add(world, uid=f"c{index}", user=f"cu{index}") + session.asn_class = asn_class + assert len(world.candidates(TARGET)) == 2 + + +def test_an_admin_override_beats_the_class(world): + session = add(world) + session.asn_class = "datacenter" + session.class_override = "allow" + assert world.pick(TARGET) is session + session.class_override = "" + assert world.pick(TARGET) is None + + +def test_class_refusals_are_counted(world): + session = add(world) + session.asn_class = "datacenter" + before = world.class_refusals + world.candidates(TARGET) + assert world.class_refusals > before diff --git a/tests/unit/services/nodes/service.py b/tests/unit/services/nodes/service.py index 4b5c2c3b..e3da1af6 100644 --- a/tests/unit/services/nodes/service.py +++ b/tests/unit/services/nodes/service.py @@ -176,3 +176,78 @@ def test_the_dispatch_loop_starts_once_and_stops_with_the_service(service): assert service._dispatch_task is None and first.cancelled() run_async(scenario()) + + +def test_a_timed_suspension_tells_the_node_when_it_ends(service): + uid = member() + device, socket = connected(service, uid) + suspend_device(device["uid"], "test", 30) + reconcile(service) + frame = socket.sent[-1] + assert frame["reason"] == "suspended" and frame["until"].startswith("20") + + +def test_other_kicks_carry_no_until(service): + _, socket = connected(service, member()) + set_setting("nodes_enabled", "0") + reconcile(service) + assert "until" not in socket.sent[-1] + + +def test_persist_state_touches_flushes_and_saves_circuits(service): + from devplacepy.database import nodes as store + + uid = member() + device, _ = connected(service, uid) + get_table("node_devices").update({"uid": device["uid"], "last_seen_at": "2000-01-01T00:00:00+00:00"}, ["uid"]) + store.flush_counters({}) + service.counters.inc("kicked") + service.counters.observe("reddit", "residential", "ok") + service.breaker.failure(("destination", "reddit.com")) + service.persist_state(force=True) + assert store.get_device(device["uid"])["last_seen_at"] > "2020" + lifetime = store.lifetime_counters() + assert lifetime["kicked"] >= 1 and lifetime["outcome:reddit:residential:ok"] >= 1 + assert service.counters.take_pending() == {} + assert not service.breaker.dirty + + +def test_a_failed_flush_keeps_the_deltas(service, monkeypatch): + from devplacepy.services.nodes import service as module + + service.counters.inc("kicked") + + def boom(deltas): + raise RuntimeError("disk full") + + monkeypatch.setattr(module, "flush_counters", boom) + with pytest.raises(RuntimeError): + service.persist_state(force=True) + assert service.counters.take_pending() == {"kicked": 1} + + +def test_persist_state_respects_the_intervals(service, monkeypatch): + from devplacepy.services.nodes import service as module + + connected(service, member()) + calls = [] + monkeypatch.setattr(module, "touch_devices", lambda uids: calls.append(list(uids))) + service.persist_state(force=True) + service.persist_state() + assert len(calls) == 1 + + +def test_open_circuits_survive_a_restart(service): + from devplacepy.database.nodes import load_circuits + + for _ in range(service.breaker.failures): + service.breaker.failure(("destination", "reddit.com")) + service.persist_state() + assert [row["key"] for row in load_circuits()] == ["destination:reddit.com"] + fresh = NodesService() + run_async(fresh.on_enable()) + assert fresh.breaker.open_count() == 1 + fresh.breaker.success(("destination", "reddit.com")) + fresh.persist_state() + assert load_circuits() == [] + run_async(fresh.on_disable()) diff --git a/tests/unit/services/nodes/shape.py b/tests/unit/services/nodes/shape.py new file mode 100644 index 00000000..956fbcc3 --- /dev/null +++ b/tests/unit/services/nodes/shape.py @@ -0,0 +1,70 @@ +# retoor + +import json + +from devplacepy.services.nodes.shape import INCONCLUSIVE, MATCH, MISMATCH, compare, shape_of + +JSON = {"content-type": "application/json; charset=utf-8"} +HTML = {"content-type": "text/html"} + + +def listing(children): + return json.dumps({"kind": "Listing", "data": {"after": None, "children": children}}).encode() + + +def test_two_listings_with_different_content_match(): + first = shape_of(200, JSON, listing([{"id": 1}])) + second = shape_of(200, JSON, listing([{"id": 2}, {"id": 3}])) + assert compare(first, second) == MATCH + + +def test_a_forged_json_shape_mismatches(): + first = shape_of(200, JSON, listing([])) + forged = shape_of(200, JSON, json.dumps({"message": "buy now"}).encode()) + assert compare(first, forged) == MISMATCH + + +def test_html_titles_decide(): + page = b" Python : r/python " + b"x" * 600 + b"" + same = b"Python : r/python" + b"y" * 700 + b"" + other = b"Totally different" + b"x" * 600 + b"" + assert compare(shape_of(200, HTML, page), shape_of(200, HTML, same)) == MATCH + assert compare(shape_of(200, HTML, page), shape_of(200, HTML, other)) == MISMATCH + + +def test_html_without_a_title_falls_back_to_the_first_heading(): + first = shape_of(200, HTML, b"

Hello world

") + second = shape_of(200, HTML, b"

Hello world

") + assert first.signature == "Hello world" and compare(first, second) == MATCH + + +def test_a_kind_change_mismatches_even_with_the_same_status(): + assert compare(shape_of(200, JSON, listing([])), shape_of(200, HTML, b"x")) == MISMATCH + + +def test_blocked_or_error_responses_are_never_conclusive(): + ok = shape_of(200, JSON, listing([])) + for status in (403, 429, 500, 404): + assert compare(ok, shape_of(status, HTML, b"blocked")) == INCONCLUSIVE + assert compare(shape_of(status, HTML, b""), ok) == INCONCLUSIVE + + +def test_a_wild_size_difference_mismatches_only_for_real_bodies(): + big = shape_of(200, HTML, b"t" + b"a" * 5000) + small = shape_of(200, HTML, b"t" + b"a" * 600) + tiny = shape_of(200, HTML, b"t") + assert compare(big, small) == MISMATCH + assert compare(big, tiny) == MATCH + + +def test_invalid_json_is_its_own_kind(): + broken = shape_of(200, JSON, b"{not json") + assert broken.kind == "json-invalid" + assert compare(broken, shape_of(200, JSON, listing([]))) == MISMATCH + + +def test_nested_key_sets_are_bounded_and_unknown_types_compare_by_type(): + deep = {"a": {"b": {"c": {"d": 1}}}} + assert "a.b" in shape_of(200, JSON, json.dumps(deep).encode()).signature + assert "a.b.c.d" not in shape_of(200, JSON, json.dumps(deep).encode()).signature + assert compare(shape_of(200, {"content-type": "image/png"}, b"1"), shape_of(200, {"content-type": "image/png"}, b"2")) == MATCH diff --git a/tests/unit/services/nodes/wake.py b/tests/unit/services/nodes/wake.py new file mode 100644 index 00000000..6fe3dfa1 --- /dev/null +++ b/tests/unit/services/nodes/wake.py @@ -0,0 +1,90 @@ +# retoor + +import asyncio +import threading +from concurrent.futures import ThreadPoolExecutor +import time + +from devplacepy.services.nodes.wake import WakeHub + + +def run(coro): + with ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, coro).result() + + +def test_a_disabled_or_detached_hub_just_sleeps(): + async def scenario(): + hub = WakeHub() + started = time.monotonic() + assert await hub.wait_queued(0.05) is False + assert time.monotonic() - started >= 0.04 + hub.enabled = False + hub.notify_queued() + assert hub.job_event("x") is None + + run(scenario()) + + +def test_notify_wakes_the_dispatcher_immediately(): + async def scenario(): + hub = WakeHub() + hub.attach() + asyncio.get_running_loop().call_later(0.02, hub.notify_queued) + started = time.monotonic() + assert await hub.wait_queued(5) is True + assert time.monotonic() - started < 1 + + run(scenario()) + + +def test_a_notification_before_the_wait_is_not_lost(): + async def scenario(): + hub = WakeHub() + hub.attach() + hub.notify_queued() + assert await hub.wait_queued(5) is True + assert await hub.wait_queued(0.02) is False + + run(scenario()) + + +def test_job_events_are_per_job_and_only_for_the_attached_loop(): + async def scenario(): + hub = WakeHub() + assert hub.job_event("a") is None + hub.attach() + first, second = hub.job_event("a"), hub.job_event("b") + assert first is not second and hub.job_event("a") is first + hub.job_done("a") + assert first.is_set() and not second.is_set() + hub.forget("a") + assert hub.job_event("a") is not first + + run(scenario()) + + +def test_another_loop_gets_no_event(): + hub = WakeHub() + seen = [] + + async def attach(): + hub.attach() + + run(attach()) + + async def foreign(): + seen.append(hub.job_event("z")) + + run(foreign()) + assert seen == [None] + + +def test_notifying_from_another_thread_wakes_the_loop(): + async def scenario(): + hub = WakeHub() + hub.attach() + threading.Timer(0.02, hub.notify_queued).start() + assert await hub.wait_queued(5) is True + + run(scenario()) diff --git a/tests/unit/services/skills/fetcher.py b/tests/unit/services/skills/fetcher.py index 1b17e4a2..98569d00 100644 --- a/tests/unit/services/skills/fetcher.py +++ b/tests/unit/services/skills/fetcher.py @@ -92,3 +92,135 @@ def test_decode_text(): decode_text(b"a\x00b") with pytest.raises(SkillFormatError): decode_text(b"\xff\xfe\xfa") + + +import ipaddress +import socket +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from devplacepy.net_guard import is_blocked_address +from devplacepy.services.skills.fetcher import fetch_skill +from tests.conftest import run_async + +SKILL_TEXT = ( + "---\nname: wire-skill\ndescription: Fetched over a real socket.\n---\n" + "# Wire skill\n\nRead [the guide](references/guide.md) and `references/missing.md`.\n" +) + + +class Handler(BaseHTTPRequestHandler): + routes: dict = {} + + def do_GET(self): + entry = self.routes.get(self.path.split("?", 1)[0]) + if entry is None: + self.send_response(404) + self.end_headers() + return + status, headers, body = entry + self.send_response(status) + for name, value in headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + return None + + +def global_address() -> str | None: + probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + probe.connect(("8.8.8.8", 80)) + address = probe.getsockname()[0] + except OSError: + return None + finally: + probe.close() + return None if is_blocked_address(ipaddress.ip_address(address)) else address + + +needs_global_address = pytest.mark.skipif( + global_address() is None, + reason="this host has no globally routable IPv4 address, so a local server cannot pass the SSRF guard without patching it", +) + + +@pytest.fixture +def origin(): + port_holder = {} + + def text(body, kind="text/markdown"): + return 200, {"Content-Type": kind}, body.encode() + + Handler.routes = { + "/ok/": text("skill directory", "text/html"), + "/ok/SKILL.md": text(SKILL_TEXT), + "/ok/references/guide.md": text("# Guide\n\nStep one.\n"), + "/big/SKILL.md": text("---\nname: big\ndescription: too big\n---\n" + "x" * (limits.SKILL_MD_MAX_BYTES + 10)), + "/html/SKILL.md": text("not a skill", "text/html"), + "/bounce/SKILL.md": (302, {"Location": "PLACEHOLDER"}, b""), + "/loop/SKILL.md": (302, {"Location": "/loop/SKILL.md"}, b""), + } + server = ThreadingHTTPServer(("0.0.0.0", 0), Handler) + port_holder["port"] = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield port_holder["port"] + server.shutdown() + server.server_close() + + +@needs_global_address +def test_a_skill_is_fetched_over_a_real_socket_with_its_bundled_file(origin): + address = global_address() + fetched = run_async(fetch_skill(f"http://{address}:{origin}/ok/SKILL.md")) + assert fetched.parsed.name == "wire-skill" and fetched.source_url.endswith("/ok/SKILL.md") + assert fetched.files == {"references/guide.md": "# Guide\n\nStep one.\n"} + assert [(item.path, item.reason) for item in fetched.skipped] == [("references/missing.md", "the server returned 404")] + + +@needs_global_address +def test_the_directory_address_resolves_to_the_skill_file(origin): + address = global_address() + fetched = run_async(fetch_skill(f"http://{address}:{origin}/ok/")) + assert fetched.parsed.name == "wire-skill" + + +@needs_global_address +def test_loopback_is_refused_even_though_the_server_is_right_there(origin): + with pytest.raises(SkillFetchError, match="private or local"): + run_async(fetch_skill(f"http://127.0.0.1:{origin}/ok/SKILL.md")) + + +@needs_global_address +def test_a_redirect_to_loopback_is_refused_on_the_second_hop(origin): + address = global_address() + Handler.routes["/bounce/SKILL.md"] = (302, {"Location": f"http://127.0.0.1:{origin}/ok/SKILL.md"}, b"") + with pytest.raises(SkillFetchError, match="private or local"): + run_async(fetch_skill(f"http://{address}:{origin}/bounce/SKILL.md")) + + +@needs_global_address +def test_a_redirect_loop_ends_with_a_clean_error(origin): + address = global_address() + with pytest.raises(SkillFetchError): + run_async(fetch_skill(f"http://{address}:{origin}/loop/SKILL.md")) + + +@needs_global_address +def test_an_oversize_skill_is_cut_at_the_limit_and_html_is_rejected(origin): + address = global_address() + with pytest.raises(SkillFetchError, match="byte limit"): + run_async(fetch_skill(f"http://{address}:{origin}/big/SKILL.md")) + with pytest.raises(SkillFetchError, match="HTML page"): + run_async(fetch_skill(f"http://{address}:{origin}/html/SKILL.md")) + + +@needs_global_address +def test_a_missing_skill_reports_the_status(origin): + address = global_address() + with pytest.raises(SkillFetchError, match="404"): + run_async(fetch_skill(f"http://{address}:{origin}/nothing/SKILL.md")) diff --git a/tests/unit/services/sync/handles.py b/tests/unit/services/sync/handles.py new file mode 100644 index 00000000..7df11049 --- /dev/null +++ b/tests/unit/services/sync/handles.py @@ -0,0 +1,134 @@ +# retoor + +from collections import Counter +from itertools import islice + +import pytest + +from devplacepy.database import get_table, init_db +from devplacepy.database.sync import claim_identity +from devplacepy.services.sync.batch import BatchRunner +from devplacepy.services.sync.handles import HandleBuilder +from devplacepy.services.sync.records import SyncBatchForm +from devplacepy.services.sync.spec import HANDLES +from devplacepy.utils import generate_uid + +ORIGIN = "handles.example.org" +OWN_PERSON = { + "abner_page_by_abnercoimbre": "abnercoimbre", + "jakearchibald_com_by_jaffathecak": "jaffathecak", + "spritely_institute_via_untrusem": "untrusem", + "libreboot_org_via_runxiyu": "runxiyu", + "lumi_codeberg_page_via_stilic": "stilic", + "decomp_dev_via_FedericoSchonborn": "federicoschonborn", + "via_rberger_2026-10-15_12_36_38_": "rberger", + "via_veqq_2026-10-14_23_47_30_cac": "veqq", + "costa_jacobpalm_dk_via_FedericoS": "federicos", + "entropicthoughts_com_via_lcapald": "lcapald", + "github_com_via_gmem": "gmem", + "try_haxy_net_via_krig": "krig", + "youtu_be_via_jeffective": "jeffective", +} +DOMAIN_ONLY = { + "costa_jacobpalm_dk": "jacobpalm", + "techcommunity_microsoft_com": "microsoft", + "gtoal_com": "gtoal", + "javidjamae": "javidjamae", + "john_smith": "john_smith", +} + + +@pytest.mark.parametrize("raw,expected", sorted((OWN_PERSON | DOMAIN_ONLY).items())) +def test_the_person_is_found_behind_the_site_and_the_stamp(raw, expected): + assert HandleBuilder.person(raw) == expected + + +@pytest.mark.parametrize("raw", sorted(OWN_PERSON)) +def test_no_candidate_keeps_the_site_the_stamp_or_the_relation_word(raw): + for candidate in islice(HandleBuilder(ORIGIN, raw).candidates(raw), HANDLES.rerolls): + lowered = candidate.lower() + assert "_via_" not in lowered and "_by_" not in lowered + assert "2026" not in lowered and not lowered.startswith("via_") + + +def test_every_candidate_is_a_valid_handle_across_the_input_domain(): + raws = ["", "a", "ab", "x" * 80, "UPPER_case", "ünï-cödé", "__", "-a-", "via", "by_via", "9999", "john.smith", "a_b_c_d_e_f_g_h"] + raws += [f"user{index}_by_someone{index}" for index in range(300)] + for index, raw in enumerate(raws): + seen = list(islice(HandleBuilder(ORIGIN, f"u{index}").candidates(raw), 60)) + assert seen + for candidate in seen: + assert HANDLES.min_length <= len(candidate) <= HANDLES.max_length, (raw, candidate) + assert HandleBuilder.valid(candidate), (raw, candidate) + + +def test_the_same_record_always_gets_the_same_candidates(): + first = list(islice(HandleBuilder(ORIGIN, "u1").candidates("someone_via_other"), 40)) + second = list(islice(HandleBuilder(ORIGIN, "u1").candidates("someone_via_other"), 40)) + assert first == second + + +def test_the_mix_follows_the_spec_weights(): + counts = Counter() + total = 4000 + for index in range(total): + first = next(iter(HandleBuilder(ORIGIN, f"id{index}").candidates("kernelhacker"))) + counts["plain" if first == "kernelhacker" else "digits" if any(char.isdigit() for char in first) else "word"] += 1 + plain = counts["plain"] / total * 100 + assert abs(plain - HANDLES.plain_weight) < 5 + assert counts["digits"] / total * 100 > 25 + + +def test_candidates_differ_between_records_with_the_same_name(): + firsts = {next(iter(HandleBuilder(ORIGIN, f"id{index}").candidates("same"))) for index in range(200)} + assert len(firsts) > 40 + + +@pytest.fixture +def clean(local_db): + init_db() + yield + for table in ("sync_links", "sync_identities"): + get_table(table).delete(origin=ORIGIN) + + +def push(records, humanize): + return BatchRunner("users", humanize=humanize).run(SyncBatchForm(origin=ORIGIN, records=records)) + + +def stored_name(origin_uid): + link = get_table("sync_links").find_one(origin=ORIGIN, kind="users", origin_uid=origin_uid) + return get_table("users").find_one(uid=link["local_uid"])["username"] + + +def test_a_new_user_gets_a_realistic_handle_and_an_update_keeps_it(clean): + raw = f"abner_page_by_abnercoimbre{generate_uid()[-4:]}" + record = {"origin_uid": "h1", "username": raw, "created_at": "2025-01-01T00:00:00Z"} + push([record], humanize=True) + name = stored_name("h1") + assert name != raw and "_by_" not in name and name.startswith("abnercoimbre") + push([{**record, "bio": "changed bio", "updated_at": "2025-02-01T00:00:00Z"}], humanize=True) + assert stored_name("h1") == name + + +def test_without_the_flag_the_original_behaviour_is_untouched(clean): + raw = f"plain_sender_{generate_uid()[-6:]}" + push([{"origin_uid": "h2", "username": raw, "created_at": "2025-01-01T00:00:00Z"}], humanize=False) + assert stored_name("h2") == raw + + +def test_an_alias_clone_name_is_never_decorated(clean): + alias = f"mirakou{generate_uid()[-5:]}" + claim_identity(ORIGIN, "alice", "alice", alias, "{}", False) + push([{"origin_uid": "alice", "username": alias, "created_at": "2025-01-01T00:00:00Z"}], humanize=True) + assert stored_name("alice") == alias + + +def test_two_senders_with_one_person_get_two_distinct_free_handles(clean): + base = f"samename{generate_uid()[-5:]}" + push( + [{"origin_uid": f"s{index}", "username": f"site_{index}_via_{base}", "created_at": "2025-01-01T00:00:00Z"} for index in range(12)], + humanize=True, + ) + names = [stored_name(f"s{index}") for index in range(12)] + assert len({name.lower() for name in names}) == 12 diff --git a/tests/unit/services/sync/healing.py b/tests/unit/services/sync/healing.py new file mode 100644 index 00000000..28a7cae7 --- /dev/null +++ b/tests/unit/services/sync/healing.py @@ -0,0 +1,99 @@ +# retoor + +import pytest + +from devplacepy.database import get_table, init_db +from devplacepy.database.sync import claim_identity, claim_link +from devplacepy.services.sync.healing import AliasHealer +from devplacepy.services.sync.processors import UserProcessor +from devplacepy.utils import generate_uid + +ORIGIN = "heal.example.org" + + +@pytest.fixture(autouse=True) +def clean(local_db): + init_db() + for table in ("sync_identities", "sync_links"): + get_table(table).delete(origin=ORIGIN) + yield + for table in ("sync_identities", "sync_links"): + get_table(table).delete(origin=ORIGIN) + + +def clone(source="alice", alias="Mira"): + suffix = generate_uid()[-6:] + alias = f"{alias}{suffix}" + processor = UserProcessor(ORIGIN) + result = processor.process(0, {"origin_uid": source, "username": alias, "created_at": "2026-01-01T00:00:00Z"}) + claim_identity(ORIGIN, source.capitalize(), source, alias, "style", False) + return result.local_uid, alias + + +def synced_text(origin_uid, table, text, kind): + uid = generate_uid() + get_table(table).insert( + {"uid": uid, "user_uid": "u", "title": None, "content": text, "created_at": "2026-01-01T00:00:00+00:00", "target_uid": "x", "target_type": "post", "deleted_at": None, "deleted_by": None} + ) + claim_link(ORIGIN, kind, origin_uid, uid, "fp", "2026-01-01T00:00:00+00:00") + return uid + + +def test_nothing_changes_when_the_clone_still_has_the_alias(): + clone() + report = AliasHealer(ORIGIN).heal() + assert not report.renamed and not report.conflicts and report.rewritten == 0 + + +def test_an_administrator_rename_moves_the_identity_and_the_stored_mentions(): + uid, alias = clone() + post = synced_text("p1", "posts", f"thanks @{alias} and {alias} again, @{alias}x stays", "posts") + comment = synced_text("c1", "comments", f"@{alias.lower()} agreed", "comments") + other = synced_text("p2", "posts", "unrelated text", "posts") + renamed = f"Renamed{generate_uid()[-6:]}" + get_table("users").update({"uid": uid, "username": renamed}, ["uid"]) + report = AliasHealer(ORIGIN).heal() + assert report.renamed == {alias: renamed} and report.rewritten == 2 + row = get_table("sync_identities").find_one(origin=ORIGIN, source_key="alice") + assert row["our_name"] == renamed and row["our_key"] == renamed.lower() + assert get_table("posts").find_one(uid=post)["content"] == f"thanks @{renamed} and {renamed} again, @{alias}x stays" + assert get_table("comments").find_one(uid=comment)["content"] == f"@{renamed} agreed" + assert get_table("posts").find_one(uid=other)["content"] == "unrelated text" + assert not AliasHealer(ORIGIN).heal().renamed + + +def test_a_suffixed_clone_name_is_adopted(local_db): + uid, alias = clone("bob", "Taken") + suffixed = f"{alias}-heal" + get_table("users").update({"uid": uid, "username": suffixed}, ["uid"]) + assert AliasHealer(ORIGIN).heal().renamed == {alias: suffixed} + assert get_table("sync_identities").find_one(origin=ORIGIN, source_key="bob")["our_name"] == suffixed + + +def test_a_name_that_belongs_to_another_identity_is_a_conflict_not_a_steal(): + first_uid, first_alias = clone("alice", "Una") + _, second_alias = clone("carol", "Duo") + get_table("users").update({"uid": first_uid, "username": second_alias}, ["uid"]) + report = AliasHealer(ORIGIN).heal() + assert report.conflicts == ["alice"] and not report.renamed + assert get_table("sync_identities").find_one(origin=ORIGIN, source_key="alice")["our_name"] == first_alias + + +def test_a_purged_clone_is_ignored(): + uid, _ = clone() + get_table("users").delete(uid=uid) + assert AliasHealer(ORIGIN).heal().renamed == {} + + +def test_the_rewrite_work_per_run_is_bounded(monkeypatch): + from dataclasses import replace + + from devplacepy.services.sync import healing + from devplacepy.services.sync.spec import ALIAS + + uid, alias = clone() + for index in range(5): + synced_text(f"p{index}", "posts", f"hello @{alias}", "posts") + monkeypatch.setattr(healing, "ALIAS", replace(ALIAS, heal_rewrites_per_run=2)) + get_table("users").update({"uid": uid, "username": f"Bounded{generate_uid()[-6:]}"}, ["uid"]) + assert AliasHealer(ORIGIN).heal().rewritten == 2 diff --git a/tests/unit/services/sync/records.py b/tests/unit/services/sync/records.py index d88e0139..d7269f0b 100644 --- a/tests/unit/services/sync/records.py +++ b/tests/unit/services/sync/records.py @@ -83,3 +83,67 @@ def test_envelope_normalizes_origin_and_bounds_batch(): SyncBatchForm(origin="other.org", records=[{}] * (LIMITS.max_batch_records + 1)) with pytest.raises(ValidationError): SyncBatchForm(origin="bad origin", records=[{}]) + + +def test_new_kinds_have_models_and_a_published_schema(): + from devplacepy.services.sync.records import RECORD_MODELS + from devplacepy.services.sync.spec import KINDS + + assert set(RECORD_MODELS) == set(KINDS) + for model in RECORD_MODELS.values(): + assert model.model_json_schema()["title"] + + +def test_vote_value_and_target_kind_are_closed_sets(): + import pytest + from pydantic import ValidationError + + from devplacepy.services.sync.records import VoteRecord + + base = {"origin_uid": "v", "voter_uid": "u", "target_kind": "post", "target_uid": "p", "value": 1, "created_at": "2025-01-01T00:00:00Z"} + assert VoteRecord.model_validate(base).value == 1 + for change in ({"value": 0}, {"value": 2}, {"target_kind": "gist"}, {"voter_uid": "bad id"}): + with pytest.raises(ValidationError): + VoteRecord.model_validate({**base, **change}) + + +def test_attachment_fingerprint_ignores_the_blob_but_not_its_hash(): + import base64 + + from devplacepy.services.sync.records import AttachmentRecord + + def make(data): + return AttachmentRecord.model_validate({ + "origin_uid": "a", "author_uid": "u", "target_kind": "post", "target_uid": "p", + "filename": "dir/evil\\name.png", "content_b64": base64.b64encode(data).decode(), "created_at": "2025-01-01T00:00:00Z", + }) + + first, second = make(b"one"), make(b"two") + assert first.filename == "name.png" + assert "content_b64" not in first.content_fields() and first.content_fields()["sha256"] != second.content_fields()["sha256"] + assert first.content_bytes() == b"one" + + +def test_batch_byte_cap(monkeypatch): + import pytest + from dataclasses import replace + from pydantic import ValidationError + + from devplacepy.services.sync import records + from devplacepy.services.sync.spec import LIMITS + + monkeypatch.setattr(records, "LIMITS", replace(LIMITS, max_batch_bytes=100)) + with pytest.raises(ValidationError): + records.SyncBatchForm(origin="a.b", records=[{"x": "y" * 200}]) + + +def test_ack_form_validates_kinds_and_ids(): + import pytest + from pydantic import ValidationError + + from devplacepy.services.sync.records import SyncAckForm + + assert SyncAckForm(origin="A.B", items='[{"kind": "posts", "origin_uid": "p1"}]').items == [{"kind": "posts", "origin_uid": "p1"}] + for items in ([], [{"kind": "gists", "origin_uid": "x"}], [{"kind": "posts", "origin_uid": ""}], "not json"): + with pytest.raises(ValidationError): + SyncAckForm(origin="a.b", items=items) diff --git a/tests/unit/services/sync/selection.py b/tests/unit/services/sync/selection.py new file mode 100644 index 00000000..bc4283c6 --- /dev/null +++ b/tests/unit/services/sync/selection.py @@ -0,0 +1,125 @@ +# retoor + +import json + +import pytest + +from devplacepy.database import get_table, init_db +from devplacepy.database.sync import claim_link, load_candidates +from devplacepy.services.sync.options import SyncOptions +from devplacepy.services.sync.selection import Selector +from devplacepy.services.sync.spec import LIMITS, SKIP_MIN_COMMENTS + +ORIGIN = "selection.example.org" + + +@pytest.fixture +def world(local_db): + init_db() + get_table("sync_candidates").delete() + get_table("sync_links").delete(origin=ORIGIN) + yield + get_table("sync_candidates").delete() + get_table("sync_links").delete(origin=ORIGIN) + + +def options(**extra): + return SyncOptions(min_comments=2, **extra) + + +def user(uid): + return {"origin_uid": uid, "username": f"user{uid}", "created_at": "2026-01-01T00:00:00Z"} + + +def post(uid, author="a"): + return {"origin_uid": uid, "author_uid": author, "content": f"post body {uid} long enough", "created_at": "2026-01-02T00:00:00Z"} + + +def comment(uid, on, author="b", parent=None): + record = {"origin_uid": uid, "post_uid": on, "author_uid": author, "content": f"comment {uid}", "created_at": "2026-01-03T00:00:00Z"} + if parent: + record["parent_uid"] = parent + return record + + +def run(records, persist=True, **extra): + selector = Selector(ORIGIN, options(**extra), persist=persist) + return selector, *selector.select(records) + + +def test_a_post_below_the_minimum_is_parked_with_its_comments_and_authors(world): + selector, kept, skipped = run({"users": [user("a"), user("b")], "posts": [post("p1")], "comments": [comment("c1", "p1")]}) + assert kept["posts"] == [] and kept["comments"] == [] and skipped[SKIP_MIN_COMMENTS] == 1 + assert selector.parked == 1 + stored = json.loads(load_candidates(ORIGIN)[0]["record"]) + assert stored["post"]["origin_uid"] == "p1" and [item["origin_uid"] for item in stored["comments"]] == ["c1"] + assert {item["origin_uid"] for item in stored["users"]} == {"a", "b"} + + +def test_comments_arriving_in_a_later_run_complete_the_post(world): + run({"users": [user("a"), user("b")], "posts": [post("p1")], "comments": [comment("c1", "p1")]}) + selector, kept, skipped = run({"users": [user("b")], "posts": [], "comments": [comment("c2", "p1")]}) + assert selector.revived == 1 + assert [item["origin_uid"] for item in kept["posts"]] == ["p1"] + assert [item["origin_uid"] for item in kept["comments"]] == ["c1", "c2"] + assert {item["origin_uid"] for item in kept["users"]} == {"a", "b"} + assert not skipped and load_candidates(ORIGIN) == [] + + +def test_a_repeated_comment_is_counted_once(world): + run({"users": [user("a"), user("b")], "posts": [post("p1")], "comments": [comment("c1", "p1")]}) + selector, kept, skipped = run({"users": [], "posts": [], "comments": [comment("c1", "p1")]}) + assert kept["posts"] == [] and skipped[SKIP_MIN_COMMENTS] == 1 + assert len(load_candidates(ORIGIN)) == 1 + + +def test_reply_order_is_kept_parent_before_reply(world): + run({"users": [user("a"), user("b")], "posts": [post("p1")], "comments": [comment("c1", "p1")]}) + _, kept, _ = run({"users": [], "posts": [], "comments": [comment("c2", "p1", parent="c1")]}) + assert [item["origin_uid"] for item in kept["comments"]] == ["c1", "c2"] + + +def test_a_dry_run_neither_reads_nor_writes_candidates(world): + selector, _, skipped = run({"users": [user("a")], "posts": [post("p1")], "comments": []}, persist=False) + assert skipped[SKIP_MIN_COMMENTS] == 1 and selector.parked == 0 and load_candidates(ORIGIN) == [] + + +def test_a_known_post_is_never_parked_or_dropped(world): + claim_link(ORIGIN, "posts", "p9", "local-9", "x", "2026-01-01T00:00:00+00:00") + selector, kept, skipped = run({"users": [user("a")], "posts": [post("p9")], "comments": []}) + assert [item["origin_uid"] for item in kept["posts"]] == ["p9"] and not skipped and selector.parked == 0 + + +def test_the_candidate_list_is_capped_per_origin(world, monkeypatch): + from dataclasses import replace + + from devplacepy.services.sync import selection + + monkeypatch.setattr(selection, "LIMITS", replace(LIMITS, candidate_max_per_origin=2)) + selector, _, _ = run({"users": [user("a")], "posts": [post("p1"), post("p2"), post("p3")], "comments": []}) + assert selector.parked == 2 and len(load_candidates(ORIGIN)) == 2 + + +def test_an_oversize_candidate_is_dropped_not_stored(world, monkeypatch): + from dataclasses import replace + + from devplacepy.services.sync import selection + + monkeypatch.setattr(selection, "LIMITS", replace(LIMITS, candidate_max_bytes=50)) + selector, _, skipped = run({"users": [user("a")], "posts": [post("p1")], "comments": []}) + assert skipped[SKIP_MIN_COMMENTS] == 1 and selector.parked == 0 + + +def test_expired_candidates_are_not_revived(world): + run({"users": [user("a")], "posts": [post("p1")], "comments": []}) + get_table("sync_candidates").update({"uid": load_candidates(ORIGIN)[0]["uid"], "expires_at": "2000-01-01T00:00:00+00:00"}, ["uid"]) + selector, _, _ = run({"users": [], "posts": [], "comments": []}) + assert selector.revived == 0 and load_candidates(ORIGIN) == [] + + +def test_other_filters_stay_final_and_clear_the_candidate(world): + run({"users": [user("a")], "posts": [post("p1")], "comments": []}) + selector, kept, skipped = run({"users": [user("a")], "posts": [], "comments": []}, topics=("devlog",)) + from devplacepy.services.sync.spec import SKIP_TOPIC + + assert kept["posts"] == [] and skipped[SKIP_TOPIC] == 1 and load_candidates(ORIGIN) == []