@@ -73,6 +73,10 @@ devplace zips prune # delete expired zip archives + job rows
|
||||
devplace zips clear # delete every zip archive + job row
|
||||
devplace forks prune # delete expired completed fork job rows (forked projects persist)
|
||||
devplace forks clear # delete every fork job row (forked projects persist)
|
||||
devplace community discover <query> [--run] [--relearn] [--dry-run] # discover, learn and generate collectors for an external community
|
||||
devplace community run <slug> [--dry-run] # run a community's collectors through the sync API
|
||||
devplace community login <slug> [--login-user U] [--login-email E] [--login-password P | --clear] # store or remove the account used to log in
|
||||
devplace community list # list communities
|
||||
devplace messaging prune-tickets # delete expired WebSocket auth tickets (ws_tickets)
|
||||
devplace seo prune # delete expired SEO audit reports + job rows
|
||||
devplace seo clear # delete every SEO audit report + job row
|
||||
@@ -163,6 +167,8 @@ Nested `CLAUDE.md` files (loaded automatically by Claude Code only when a file i
|
||||
| `devplacepy/services/quiz/CLAUDE.md` | Quizzes (the terminal publish lock, attempt atomicity, answer-key withholding, AI free-text grading, the best-attempt scoreboard) |
|
||||
| `devplacepy/services/skills/CLAUDE.md` | Agent Skills: strict `SKILL.md` parser, SSRF-guarded fetcher, the bounded prompt index, per-user privacy, soft delete and account-deletion cascade, why a skill is data and never code |
|
||||
| `devplacepy/services/nodes/CLAUDE.md` | Community node network: the protocol specification (single source of truth), the no-open-proxy rule, the two transports and their trust levels, the decisions taken, and the gaps per phase |
|
||||
| `devplacepy/services/community/CLAUDE.md` | Community discovery: rsearch + feed detection, one-time structure learning, generated collectors, dry run, the failure floor, admin-only visibility |
|
||||
| `devplacepy/services/sync/CLAUDE.md` | Community sync: the origin link table, idempotent batch upserts, normalization and rejection rules, what is deliberately not imported, the race that was measured |
|
||||
| `devplacepy/services/CLAUDE.md` | Background task queue, AI correction/modifier, presence, live view relay, `BaseService`/`ServiceManager` |
|
||||
| `devplacepy/database/CLAUDE.md` | Dataset rules, indexing conventions, soft delete, tables, site settings |
|
||||
| `devplacepy/utils/CLAUDE.md` | Notifications and gamification (XP/levels/badges/leaderboard) |
|
||||
@@ -206,6 +212,7 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror
|
||||
| `/xmlrpc` | xmlrpc.py - see `services/xmlrpc/CLAUDE.md` |
|
||||
| `/api` | devrant/ package - see `routers/devrant/CLAUDE.md` |
|
||||
| `/dbapi` | dbapi/ package, **primary-administrator-only, strictly READ-ONLY** - see `services/dbapi/CLAUDE.md` |
|
||||
| `/sync` | sync/ package, **administrator only**, off by default - see `services/sync/CLAUDE.md` |
|
||||
| `/game` | game/ package - see `services/game/CLAUDE.md` |
|
||||
| `/reports`, `/admin/moderation`, `/workspaces` | reports.py, admin/moderation.py, workspaces.py - see `services/moderation/CLAUDE.md` |
|
||||
| `/quizzes` | quizzes/ package - see `services/quiz/CLAUDE.md` |
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ FROM docker.io/library/python:3.13-slim
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl ca-certificates rclone \
|
||||
curl ca-certificates rclone git \
|
||||
libglib2.0-0 libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz0b libfontconfig1 \
|
||||
fonts-dejavu-core \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -148,6 +148,14 @@ clean:
|
||||
test-cache-clean:
|
||||
rm -rf .pytest_cache
|
||||
|
||||
# Builds the donor clients (Go binaries for five platforms and the browser
|
||||
# extension zip) into the directory the /nodes/download route serves.
|
||||
NODE_CLIENT_OUT ?= data/node_clients
|
||||
.PHONY: node-client
|
||||
node-client:
|
||||
sh nodes/client/build.sh $(NODE_CLIENT_OUT)
|
||||
sh nodes/extension/build.sh $(NODE_CLIENT_OUT)
|
||||
|
||||
# Safe but aggressive disk-space cleanup, acting on the REAL data/devplace.db
|
||||
# and data/ tree (never the test database) - see CLAUDE.md "devplace system
|
||||
# prune". prune-dry-run reports what would be removed without deleting.
|
||||
|
||||
@@ -65,7 +65,8 @@ devplacepy/
|
||||
| `/feed` | Post feed with topic/tab filtering and free-text `search` (title, content, and author username) in the left panel (public). Each page interleaves authors so no two consecutive posts share an author. |
|
||||
| `/topics` | Crawlable per-topic category pages (public): `/topics` hub links every topic with a live post count, `/topics/{topic}` lists that topic's posts with its own canonical URL, title, and breadcrumbs |
|
||||
| `/skills` | Per-user Agent Skills: list, detail, install from a URL, enable/disable, delete. Private to the owner, `noindex` |
|
||||
| `/nodes` | Community node network: `GET /nodes/spec.json`, `GET/POST /nodes/devices` (+ `/rotate`, `/delete`), `POST /nodes/ticket`, `WS /nodes/ws`. Server side only, off by default, no client yet (see `plan.md`) |
|
||||
| `/nodes` | Community node network: `GET /nodes/spec.json`, `GET/POST /nodes/devices` (+ `/rotate`, `/delete`), `POST /nodes/ticket`, `WS /nodes/ws`. Off by default; also `GET /nodes/me`, `POST /nodes/settings`, `GET /nodes/social/search`, `POST /nodes/internal/fetch`, `GET /nodes/stats`, `GET /nodes/leaderboard`, `GET /nodes/client-version`, `GET /nodes/download/{artifact}` (see `plan.md`) |
|
||||
| `/sync` | Community sync: `GET /sync/spec.json`, `POST /sync/{users,posts,comments}`. Administrator only, off by default (`sync_enabled`), idempotent batch upserts keyed on the origin community's ids |
|
||||
| `/news` | Developer news listing, detail page with comments |
|
||||
| `/posts` | Post detail, creation |
|
||||
| `/gists` | Code gist listing, detail, creation, and editing; left panel offers language filtering and free-text `search` (title, description, and author username), public read. A plain Markdown gist has a **View rendered / View raw** toggle beside its Copy button, switching between the raw source and the same rendered view a `Markdown Rendered` gist always shows |
|
||||
@@ -252,9 +253,19 @@ DevPlace is open in the sense that it does not editorialise technical opinion. I
|
||||
|
||||
Members can install [Agent Skills](https://agentskills.io) from a URL at `/skills` so Devii can use them. A skill is a `SKILL.md` (YAML frontmatter plus markdown instructions) with optional text files under `scripts/`, `references/` and `assets/`. Skills are private to their owner, count against a per-user cap, and only the name and a one-line description of enabled skills are added to Devii's system prompt; Devii loads the full text on demand. A skill is untrusted third-party text: it can never grant Devii a tool, bypass a confirmation or run a script. Deleting a skill is a soft delete and needs confirmation when Devii does it. Documented in the API docs under *Skills* and in `devplacepy/services/skills/CLAUDE.md`.
|
||||
|
||||
## Community nodes (server side built, no client yet)
|
||||
## Community sync
|
||||
|
||||
Members will be able to opt in to donate a little of their connection to the DevPlace community, so search can reach sites that block datacenter addresses. **Today the server side exists:** the specification (`GET /nodes/spec.json` and the [Node network protocol](/docs/nodes-protocol.html) docs page, both generated from `devplacepy/services/nodes/spec.py`), device registration (`/nodes/devices`), the socket ticket (`POST /nodes/ticket`), the node socket (`WS /nodes/ws`), the consent gate and the administrator kill switch (`nodes_enabled`, off by default). There is no client and no search integration yet, so nothing donates and nothing is relayed.
|
||||
Users, posts and comments can be copied in from another community. The sending side posts batches of up to 50 records to `POST /sync/users`, `/sync/posts` and `/sync/comments` (administrator API key, `sync_enabled` set to `1`). Each record carries the id it has in the origin community (`origin_uid`); the server keeps a `sync_links` row from `(origin, kind, origin_uid)` to the local row, so a repeated record is an update or a no-op and never a duplicate. Records are validated and normalized with Pydantic (`devplacepy/services/sync/records.py`) and every record gets its own result, so one bad record never fails the batch. Votes, stars, XP and levels cannot be verified across communities and are not imported; they are listed under `ignored` in the result. Synced content sends no notifications, awards no XP, runs no AI correction and passes the content filter. `sync_origins` optionally restricts which origins may send. Limits, statuses and the record schemas are published at `GET /sync/spec.json`. There is no export endpoint and no scheduled sync service yet; the batch API is the building block for it. Usage is written up in `prompt.md`.
|
||||
|
||||
### Community discovery
|
||||
|
||||
Administrators can add an external community by name or address at `/admin/communities` (or `devplace community discover`). The system searches (rsearch), detects RSS, Atom, JSON and HTML feeds, learns the structure once, records the community's description, rules and size for the admin detail page, generates standalone collector code under `data/communities/<slug>/`, and then runs it on a schedule through the sync API above. Members never see these communities. Login-protected communities take a stored account; JavaScript pages are rendered through stealthii; the internal AI model helps with HTML structure it cannot read otherwise. Administrators can manage everything through the admin pages, `devplace community ...` and Devii. Dry run (`--dry-run`, the admin button, and the default of the `Community sync` service) validates and logs everything without writing. Design and limits: `add_community.md` and `devplacepy/services/community/CLAUDE.md`. Per community, administrators can optionally (all off by default) sample which posts are accepted, filter by topic and minimum comments, jitter the schedule, translate content, and store authors under permanent pseudonyms with every mention rewritten (`community_sync_additional.md`); this is administrator-only and invisible to members.
|
||||
|
||||
## Community nodes (built, off by default)
|
||||
|
||||
Members will be able to opt in to donate a little of their connection to the DevPlace community, so search can reach sites that block datacenter addresses. **Today the server side exists:** the specification (`GET /nodes/spec.json` and the [Node network protocol](/docs/nodes-protocol.html) docs page, both generated from `devplacepy/services/nodes/spec.py`), device registration (`/nodes/devices`), the socket ticket (`POST /nodes/ticket`), the node socket (`WS /nodes/ws`), the consent gate and the administrator kill switch (`nodes_enabled`, off by default). The server queues, schedules and meters jobs and relays them either as a `tunnel` (raw TCP, TLS stays end to end between DevPlace and the site, so the donor cannot read or forge the page) or as a `fetch` (extension donors, results verified and labelled). Donor clients are a Go binary (`nodes/client/`) and a browser extension (`nodes/extension/`); `make node-client` builds them into `data/node_clients/`, served at `/nodes/client-version` and `/nodes/download/{artifact}`. Social search (`GET /nodes/social/search`, Devii `social_search`, optional DeepSearch source) reads Hacker News, Stack Overflow, GitHub, dev.to and Mastodon directly and Reddit through donors when `nodes_social_via_nodes` is on. Whether Reddit works from residential donors is not measured yet. Administrators manage it at `/admin/nodes` (pool and queue summary, devices by status, kind, address class and country, open flags, timed device suspension, the kill switch and the caps), with the same operations available to Devii for administrators.
|
||||
|
||||
Members manage their donation on the owner-only **Donate** tab of their profile (`/profile/{username}?tab=donate`, noindex): the explanation, the `bandwidth_sharing` consent, limits and active hours, devices with a one-time secret and install lines, and a live "Donated by you" card. The user guide is [Donate your connection](/docs/nodes.html).
|
||||
|
||||
What is fixed already, because the specification enforces it: nobody is enrolled by default; a node is not an open proxy and only ever contacts a short published allowlist of websites, on ports 80 and 443, never a private address; the refusal is enforced on the donor's machine as well as on the server; every limit is a published hard cap. The implementation plan, the measured and unmeasured claims, and the legal gate are in `plan.md`.
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
|
||||
from devplacepy.cli._shared import _audit_cli
|
||||
|
||||
|
||||
def _prepare():
|
||||
from devplacepy.database import init_db
|
||||
|
||||
init_db()
|
||||
|
||||
|
||||
def _credentials(args) -> dict:
|
||||
return {
|
||||
"username": args.login_user or "",
|
||||
"email": args.login_email or "",
|
||||
"password": args.login_password or os.environ.get("COMMUNITY_PASSWORD", ""),
|
||||
}
|
||||
|
||||
|
||||
def _print_report(report):
|
||||
mode = "DRY RUN" if report.dry_run else "LIVE"
|
||||
print(f"\n[{mode}] status: {report.status}")
|
||||
print(f"collected: {report.collected}")
|
||||
for kind, counts in report.pushed.items():
|
||||
print(f" {kind}: {counts or 'nothing'}")
|
||||
for error in report.errors[:10]:
|
||||
print(f" problem: {error}")
|
||||
|
||||
|
||||
def cmd_community_discover(args):
|
||||
from devplacepy.services.community.logbook import LogBook
|
||||
from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner
|
||||
from devplacepy.services.community.spec import STATUS_READY
|
||||
from devplacepy.services.community.store import store
|
||||
|
||||
from devplacepy.services.community.discoverer import slug_for_query
|
||||
|
||||
_prepare()
|
||||
runner = CommunityRunner()
|
||||
existing = store.get(slug_for_query(args.query))
|
||||
if existing and not args.relearn and not args.dry_run:
|
||||
print(f"Community '{existing['slug']}' already exists ({existing['status']}); pass --relearn to learn it again")
|
||||
if args.run and existing["status"] == STATUS_READY:
|
||||
with LogBook(echo=True):
|
||||
_print_report(asyncio.run(runner.run(existing, dry_run=False, trigger=TRIGGER_CLI)))
|
||||
return
|
||||
with LogBook(echo=True):
|
||||
result, row, _ = asyncio.run(
|
||||
runner.discover(
|
||||
args.query,
|
||||
"cli",
|
||||
dry_run=args.dry_run,
|
||||
relearn_slug=existing["slug"] if existing else None,
|
||||
credentials=_credentials(args),
|
||||
)
|
||||
)
|
||||
print(f"\n{result.slug or '-'}: {result.status}" + (f" ({result.error})" if result.error else ""))
|
||||
if result.status == STATUS_READY:
|
||||
for feed in result.schema["feeds"]:
|
||||
print(f" feed {feed['id']}: {feed['type']} {feed['url']} comments={feed['comments']['mode']} pagination={feed['pagination']['style']}")
|
||||
print(f" sample: { {kind: len(rows) for kind, rows in result.sample['records'].items()} }")
|
||||
if args.dry_run:
|
||||
print("dry run: nothing was saved")
|
||||
else:
|
||||
_audit_cli("cli.community.discover", f"CLI discovered community {result.slug}: {result.status}", target_label=result.slug, metadata={"status": result.status})
|
||||
if result.status != STATUS_READY and not args.dry_run and result.status != "login_required":
|
||||
sys.exit(1)
|
||||
if args.run and row and result.status == STATUS_READY and not args.dry_run:
|
||||
with LogBook(echo=True):
|
||||
_print_report(asyncio.run(runner.run(store.get(row["uid"]), dry_run=False, trigger=TRIGGER_CLI)))
|
||||
|
||||
|
||||
def cmd_community_run(args):
|
||||
from devplacepy.services.community.logbook import LogBook
|
||||
from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner
|
||||
from devplacepy.services.community.store import store
|
||||
|
||||
_prepare()
|
||||
row = store.get(args.slug)
|
||||
if row is None:
|
||||
print(f"Unknown community '{args.slug}'")
|
||||
sys.exit(1)
|
||||
with LogBook(echo=True):
|
||||
report = asyncio.run(CommunityRunner().run(row, dry_run=args.dry_run, trigger=TRIGGER_CLI))
|
||||
_print_report(report)
|
||||
if not args.dry_run:
|
||||
_audit_cli("cli.community.run", f"CLI ran community {row['slug']}: {report.status}", target_label=row["slug"], metadata={"status": report.status, "collected": report.collected})
|
||||
if report.status == "failed":
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_community_login(args):
|
||||
from devplacepy.services.community.store import store
|
||||
|
||||
_prepare()
|
||||
row = store.get(args.slug)
|
||||
if row is None:
|
||||
print(f"Unknown community '{args.slug}'")
|
||||
sys.exit(1)
|
||||
if args.clear:
|
||||
store.update(row["uid"], login_username="", login_email="", login_password="")
|
||||
print(f"Removed the stored account of {row['slug']}")
|
||||
else:
|
||||
secrets = _credentials(args)
|
||||
if not secrets["password"] or not (secrets["username"] or secrets["email"]):
|
||||
print("Give --login-user or --login-email, and --login-password (or COMMUNITY_PASSWORD)")
|
||||
sys.exit(1)
|
||||
store.update(row["uid"], login_username=secrets["username"], login_email=secrets["email"], login_password=secrets["password"])
|
||||
print(f"Stored the account for {row['slug']}; run 'community discover \"{row['query']}\" --relearn' to learn behind the login")
|
||||
_audit_cli("cli.community.login", f"CLI changed the stored account of community {row['slug']}", target_label=row["slug"])
|
||||
|
||||
|
||||
def cmd_community_list(args):
|
||||
from devplacepy.services.community.store import store
|
||||
|
||||
_prepare()
|
||||
rows = store.list()
|
||||
if not rows:
|
||||
print("No communities")
|
||||
return
|
||||
for row in rows:
|
||||
print(
|
||||
f"{row['slug']:<28} {row['status']:<15} runs={row.get('run_count') or 0:<4} "
|
||||
f"last={row.get('last_run_at') or '-'} next={row.get('next_run_at') or '-'}"
|
||||
)
|
||||
|
||||
|
||||
def add_login_arguments(parser):
|
||||
parser.add_argument("--login-user", help="username of an account on the community")
|
||||
parser.add_argument("--login-email", help="email of an account on the community")
|
||||
parser.add_argument("--login-password", help="password (or set COMMUNITY_PASSWORD)")
|
||||
|
||||
|
||||
def register_community(subparsers):
|
||||
community = subparsers.add_parser("community", help="External community discovery and sync")
|
||||
sub = community.add_subparsers(title="action", dest="action")
|
||||
discover = sub.add_parser("discover", help="Discover, learn and generate collectors for a community")
|
||||
discover.add_argument("query", help="community name, address or search phrase")
|
||||
discover.add_argument("--run", action="store_true", help="also run the collectors once, live")
|
||||
discover.add_argument("--relearn", action="store_true", help="learn again even if the community exists")
|
||||
discover.add_argument("--dry-run", action="store_true", help="learn and report, save nothing")
|
||||
add_login_arguments(discover)
|
||||
discover.set_defaults(func=cmd_community_discover)
|
||||
run = sub.add_parser("run", help="Run the generated collectors of a community")
|
||||
run.add_argument("slug")
|
||||
run.add_argument("--dry-run", action="store_true", help="collect and validate, write nothing, keep the watermark")
|
||||
run.set_defaults(func=cmd_community_run)
|
||||
login = sub.add_parser("login", help="Store or remove the account used to log in to a community")
|
||||
login.add_argument("slug")
|
||||
login.add_argument("--clear", action="store_true", help="remove the stored account")
|
||||
add_login_arguments(login)
|
||||
login.set_defaults(func=cmd_community_login)
|
||||
listing = sub.add_parser("list", help="List communities")
|
||||
listing.set_defaults(func=cmd_community_list)
|
||||
@@ -18,6 +18,8 @@ from devplacepy.cli.quiz import register_quiz
|
||||
from devplacepy.cli.gateway import register_gateway
|
||||
from devplacepy.cli.messaging import register_messaging
|
||||
from devplacepy.cli.system import register_system
|
||||
from devplacepy.cli.community import register_community
|
||||
from devplacepy.cli.nodes import register_nodes
|
||||
|
||||
|
||||
def build_parser():
|
||||
@@ -40,6 +42,8 @@ def build_parser():
|
||||
register_messaging(sub)
|
||||
register_accounts(sub)
|
||||
register_system(sub)
|
||||
register_community(sub)
|
||||
register_nodes(sub)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import sys
|
||||
|
||||
from devplacepy.cli._shared import _audit_cli
|
||||
|
||||
LIST_LIMIT = 200
|
||||
SUSPEND_DEFAULT_MINUTES = 60
|
||||
SUSPEND_MAX_MINUTES = 7 * 24 * 60
|
||||
|
||||
|
||||
def _prepare():
|
||||
from devplacepy.database import init_db
|
||||
|
||||
init_db()
|
||||
|
||||
|
||||
def cmd_nodes_list(args):
|
||||
from devplacepy.database import get_table
|
||||
from devplacepy.database.nodes import effective_status
|
||||
|
||||
_prepare()
|
||||
rows = list(get_table("node_devices").find(deleted_at=None, order_by=["-created_at"], _limit=LIST_LIMIT))
|
||||
if not rows:
|
||||
print("No devices")
|
||||
return
|
||||
for row in rows:
|
||||
print(
|
||||
f"{row['uid']} {effective_status(row):<10} {row.get('kind') or '-':<9} "
|
||||
f"{row.get('platform') or '-':<14} user={row['user_uid']} last={row.get('last_seen_at') or '-'}"
|
||||
)
|
||||
|
||||
|
||||
def cmd_nodes_stats(args):
|
||||
from devplacepy.database.nodes import list_flags, public_stats
|
||||
|
||||
_prepare()
|
||||
stats = public_stats()
|
||||
print(f"donors: {stats['donors']} requests: {stats['requests']} bytes: {stats['bytes']}")
|
||||
print(f"open flags: {len(list_flags())}")
|
||||
|
||||
|
||||
def cmd_nodes_flags(args):
|
||||
from devplacepy.database.nodes import list_flags
|
||||
|
||||
_prepare()
|
||||
flags = list_flags()
|
||||
if not flags:
|
||||
print("No open flags")
|
||||
return
|
||||
for flag in flags:
|
||||
print(f"{flag['uid']} {flag.get('kind') or '-':<22} device={flag.get('device_uid') or '-'} {flag.get('detail') or ''}")
|
||||
|
||||
|
||||
def cmd_nodes_suspend(args):
|
||||
from devplacepy.database.nodes import suspend_device
|
||||
|
||||
_prepare()
|
||||
minutes = max(1, min(args.minutes, SUSPEND_MAX_MINUTES))
|
||||
if suspend_device(args.device_uid, args.reason, minutes):
|
||||
print(f"Suspended {args.device_uid} for {minutes} minutes")
|
||||
_audit_cli(
|
||||
"cli.nodes.suspend",
|
||||
f"CLI suspended device {args.device_uid}",
|
||||
target_type="node_device",
|
||||
target_uid=args.device_uid,
|
||||
metadata={"minutes": minutes, "reason": args.reason},
|
||||
)
|
||||
return
|
||||
print(f"No active device {args.device_uid}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_nodes_unsuspend(args):
|
||||
from devplacepy.database.nodes import unsuspend_device
|
||||
|
||||
_prepare()
|
||||
if unsuspend_device(args.device_uid):
|
||||
print(f"Lifted the suspension of {args.device_uid}")
|
||||
_audit_cli(
|
||||
"cli.nodes.unsuspend",
|
||||
f"CLI lifted the suspension of device {args.device_uid}",
|
||||
target_type="node_device",
|
||||
target_uid=args.device_uid,
|
||||
)
|
||||
return
|
||||
print(f"{args.device_uid} is not suspended")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_nodes_prune(args):
|
||||
from devplacepy.services.nodes.ledger import UsageLedger
|
||||
|
||||
_prepare()
|
||||
counts = UsageLedger().prune(dry_run=args.dry_run)
|
||||
label = "would remove" if args.dry_run else "removed"
|
||||
for name, count in counts.items():
|
||||
print(f"{label} {count} {name}")
|
||||
|
||||
|
||||
def cmd_nodes_clear(args):
|
||||
from devplacepy.database.nodes import clear_ephemeral
|
||||
|
||||
_prepare()
|
||||
counts = clear_ephemeral(dry_run=not args.force)
|
||||
label = "removed" if args.force else "would remove (pass --force)"
|
||||
for name, count in counts.items():
|
||||
print(f"{label} {count} {name}")
|
||||
if args.force:
|
||||
_audit_cli("cli.nodes.clear","CLI cleared the node job queue and event log", metadata=counts)
|
||||
|
||||
|
||||
def register_nodes(subparsers):
|
||||
nodes = subparsers.add_parser("nodes", help="Community donor node network")
|
||||
sub = nodes.add_subparsers(title="action", dest="action")
|
||||
sub.add_parser("list", help="List devices").set_defaults(func=cmd_nodes_list)
|
||||
sub.add_parser("stats", help="Aggregate donation totals and open flags").set_defaults(func=cmd_nodes_stats)
|
||||
sub.add_parser("flags", help="List open flags that need a human").set_defaults(func=cmd_nodes_flags)
|
||||
suspend = sub.add_parser("suspend", help="Suspend one device for a while")
|
||||
suspend.add_argument("device_uid")
|
||||
suspend.add_argument("--minutes", type=int, default=SUSPEND_DEFAULT_MINUTES)
|
||||
suspend.add_argument("--reason", default="suspended from the command line")
|
||||
suspend.set_defaults(func=cmd_nodes_suspend)
|
||||
unsuspend = sub.add_parser("unsuspend", help="Lift the suspension of a device")
|
||||
unsuspend.add_argument("device_uid")
|
||||
unsuspend.set_defaults(func=cmd_nodes_unsuspend)
|
||||
prune = sub.add_parser("prune", help="Prune usage buckets, events and finished requests past their retention")
|
||||
prune.add_argument("--dry-run", action="store_true", help="report what would be removed")
|
||||
prune.set_defaults(func=cmd_nodes_prune)
|
||||
clear = sub.add_parser("clear", help="Empty the job queue and the event log (never usage or donor totals)")
|
||||
clear.add_argument("--force", action="store_true", help="actually delete; without it only counts are shown")
|
||||
clear.set_defaults(func=cmd_nodes_clear)
|
||||
@@ -25,6 +25,7 @@ ZIP_STAGING_DIR = DATA_DIR / "zip_staging"
|
||||
FORK_STAGING_DIR = DATA_DIR / "fork_staging"
|
||||
BACKUPS_DIR = DATA_DIR / "backups"
|
||||
BACKUP_STAGING_DIR = DATA_DIR / "backup_staging"
|
||||
NODE_CLIENTS_DIR = DATA_DIR / "node_clients"
|
||||
|
||||
RCLONE_BIN = environ.get("DEVPLACE_RCLONE_BIN", "rclone")
|
||||
RCLONE_CONFIG_FILE = environ.get(
|
||||
@@ -45,6 +46,7 @@ ISSLOP_RUNS_DIR = ISSLOP_DIR / "runs"
|
||||
ISSLOP_MEDIA_DIR = ISSLOP_DIR / "media"
|
||||
KEYS_DIR = DATA_DIR / "keys"
|
||||
BOT_DIR = DATA_DIR / "bot"
|
||||
COMMUNITIES_DIR = DATA_DIR / "communities"
|
||||
LOCKS_DIR = DATA_DIR / "locks"
|
||||
|
||||
DEVII_TASKS_DB = DATA_DIR / "devii_tasks.db"
|
||||
@@ -152,6 +154,7 @@ DATA_PATHS: dict[str, Path] = {
|
||||
"fork_staging": FORK_STAGING_DIR,
|
||||
"backups": BACKUPS_DIR,
|
||||
"backup_staging": BACKUP_STAGING_DIR,
|
||||
"node_clients": NODE_CLIENTS_DIR,
|
||||
"seo_reports": SEO_REPORTS_DIR,
|
||||
"planning_reports": PLANNING_REPORTS_DIR,
|
||||
"dbapi": DBAPI_DIR,
|
||||
@@ -163,6 +166,7 @@ DATA_PATHS: dict[str, Path] = {
|
||||
"isslop_media": ISSLOP_MEDIA_DIR,
|
||||
"keys": KEYS_DIR,
|
||||
"bot": BOT_DIR,
|
||||
"communities": COMMUNITIES_DIR,
|
||||
"locks": LOCKS_DIR,
|
||||
}
|
||||
|
||||
|
||||
@@ -223,6 +223,7 @@ Site settings are seeded on startup (`site_settings` table):
|
||||
| `terms_version` / `privacy_version` / `guidelines_version` | `"1"` | Bumping `terms_version` forces re-acceptance before the next write. **Every reader uses `get_setting(key, "1") or "1"`** - an empty stored value must read as the default or the gate 403s every write |
|
||||
| `ai_third_party_provider` | `""` | Named in the consent copy and the privacy policy |
|
||||
| `extra_head` | `""` | Raw HTML emitted verbatim into every page `<head>` by `templating.extra_head_tag()`; site-wide trusted-admin input, not sanitized |
|
||||
| `sync_enabled` / `sync_origins` | `"0"` / `""` | Community sync kill switch (`routers/sync/_shared.py`) and an optional comma separated allowlist of sending origins; empty allows any origin |
|
||||
|
||||
Besides the site/news/upload keys above, the **Operational** group is admin-editable at `/admin/settings`: `site_url` (public origin for absolute links incl. container ingress; resolved by `seo.public_base_url()` = setting -> `DEVPLACE_SITE_URL` env -> request origin), `rate_limit_per_minute`, `rate_limit_window_seconds`, `news_service_interval`, `session_max_age_days`, `session_remember_days`, `registration_open`, `maintenance_mode`, `maintenance_message`, `docs_search_mode` (`agent`|`bm25`, default `agent` - picks the `/docs/search.html` surface: the in-page Devii chat or the classic BM25 list; a viewer over their daily AI limit, or a guest when Devii is disabled, auto-falls-back to BM25 via `routers/docs/views.py` `_agent_search_state`), `outbound_proxy_url` (empty by default - when set, every `stealth.stealth_async_client`/`stealth_sync_client` call across the whole app routes through it via `stealth.configured_proxy_url()`; validated as `http(s)://`/`socks5(h)://` with a host in `AdminSettingsForm`; falls back to `DEVPLACE_OUTBOUND_PROXY_URL` when unset - see the "Outbound HTTP" note in the root `CLAUDE.md`). The **Custom Code** key `extra_head` and the `ios_app_url` badge link are the keys in the settings handler's `CLEARABLE_SETTINGS` set, so saving an empty textarea/field removes them (the default loop skips empty values).
|
||||
|
||||
|
||||
@@ -49,10 +49,12 @@ UNREPORTABLE_TABLES: dict[str, str] = {
|
||||
"bookmarks": "private to the owner",
|
||||
"notes": "private to the owner",
|
||||
"skills": "private to the owner, never shown to others",
|
||||
"communities": "administrator-only configuration, never shown to members",
|
||||
"node_devices": "private to the owner, the label is never shown to others",
|
||||
"node_settings": "private to the owner",
|
||||
"node_usage_daily": "derived donation counters, carry no authored content",
|
||||
"node_donors": "derived donation totals, carry no authored content",
|
||||
"node_caller_daily": "derived daily request counters, carry no authored content",
|
||||
"node_flags": "operational flags for administrators, carry no authored content",
|
||||
"follows": "relationship rows, carry no authored content",
|
||||
"poll_votes": "private ballots",
|
||||
|
||||
@@ -621,9 +621,20 @@ def prune_usage_daily(*, dry_run: bool = False, moment: datetime | None = None)
|
||||
)
|
||||
)
|
||||
)["c"]
|
||||
if not dry_run and count:
|
||||
_execute("DELETE FROM node_usage_daily WHERE day < :cutoff", {"cutoff": cutoff})
|
||||
return count
|
||||
callers = next(
|
||||
iter(
|
||||
db.query(
|
||||
"SELECT COUNT(*) AS c FROM node_caller_daily WHERE day < :cutoff",
|
||||
cutoff=cutoff,
|
||||
)
|
||||
)
|
||||
)["c"]
|
||||
if not dry_run:
|
||||
if count:
|
||||
_execute("DELETE FROM node_usage_daily WHERE day < :cutoff", {"cutoff": cutoff})
|
||||
if callers:
|
||||
_execute("DELETE FROM node_caller_daily WHERE day < :cutoff", {"cutoff": cutoff})
|
||||
return count + callers
|
||||
|
||||
|
||||
def prune_events(*, dry_run: bool = False, moment: datetime | None = None) -> int:
|
||||
@@ -669,3 +680,312 @@ def prune_requests(*, dry_run: bool = False, moment: datetime | None = None) ->
|
||||
if not dry_run and count:
|
||||
_execute("DELETE FROM node_requests WHERE created_at < :cutoff", {"cutoff": cutoff})
|
||||
return count
|
||||
|
||||
|
||||
def clear_ephemeral(*, dry_run: bool = True) -> dict:
|
||||
counts = {}
|
||||
for table in ("node_requests", "node_events"):
|
||||
counts[table] = next(iter(db.query(f"SELECT COUNT(*) AS c FROM {table}")))["c"]
|
||||
if not dry_run and counts[table]:
|
||||
_execute(f"DELETE FROM {table}", {})
|
||||
return counts
|
||||
|
||||
|
||||
class QueueFull(NodeLimitError):
|
||||
pass
|
||||
|
||||
|
||||
REQUEST_OPEN = ("queued", "assigned")
|
||||
REQUEST_COLUMNS_PUBLIC = (
|
||||
"uid",
|
||||
"status",
|
||||
"result_status",
|
||||
"result_bytes",
|
||||
"result_headers",
|
||||
"truncated",
|
||||
"error",
|
||||
)
|
||||
|
||||
|
||||
def enqueue_request(
|
||||
caller_kind: str,
|
||||
caller_id: str,
|
||||
url: str,
|
||||
method: str,
|
||||
adapter: str,
|
||||
mode: str,
|
||||
) -> str:
|
||||
uid = _uid()
|
||||
created = _execute(
|
||||
"INSERT INTO node_requests (uid, caller_kind, caller_id, url, method, adapter, "
|
||||
"mode, status, device_uid, result_status, result_bytes, result_body, "
|
||||
"result_headers, truncated, error, created_at, assigned_at, finished_at) "
|
||||
"SELECT :uid, :kind, :caller, :url, :method, :adapter, :mode, 'queued', '', 0, 0, "
|
||||
"'', '', 0, '', :now, '', '' "
|
||||
"WHERE (SELECT COUNT(*) FROM node_requests WHERE status IN ('queued','assigned')) "
|
||||
"< :max_depth",
|
||||
{
|
||||
"uid": uid,
|
||||
"kind": caller_kind,
|
||||
"caller": caller_id,
|
||||
"url": url,
|
||||
"method": method,
|
||||
"adapter": adapter,
|
||||
"mode": mode,
|
||||
"now": _now_iso(),
|
||||
"max_depth": LIMITS.max_queue_depth,
|
||||
},
|
||||
)
|
||||
if created != 1:
|
||||
raise QueueFull("The node queue is full")
|
||||
return uid
|
||||
|
||||
|
||||
def get_request(request_uid: str) -> dict | None:
|
||||
return get_table("node_requests").find_one(uid=request_uid)
|
||||
|
||||
|
||||
def list_queued(limit: int) -> list[dict]:
|
||||
return [
|
||||
dict(row)
|
||||
for row in db.query(
|
||||
"SELECT * FROM node_requests WHERE status = 'queued' "
|
||||
"ORDER BY created_at, id LIMIT :n",
|
||||
n=int(limit),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def count_open_requests() -> int:
|
||||
return next(
|
||||
iter(
|
||||
db.query(
|
||||
"SELECT COUNT(*) AS c FROM node_requests WHERE status IN ('queued','assigned')"
|
||||
)
|
||||
)
|
||||
)["c"]
|
||||
|
||||
|
||||
def claim_request(request_uid: str, device_uid: str) -> bool:
|
||||
return (
|
||||
_execute(
|
||||
"UPDATE node_requests SET status = 'assigned', device_uid = :device, "
|
||||
"assigned_at = :now WHERE uid = :uid AND status = 'queued'",
|
||||
{"device": device_uid, "now": _now_iso(), "uid": request_uid},
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def finish_request(
|
||||
request_uid: str,
|
||||
*,
|
||||
result_status: int,
|
||||
result_headers: str,
|
||||
result_body: str,
|
||||
result_bytes: int,
|
||||
truncated: bool,
|
||||
) -> bool:
|
||||
return (
|
||||
_execute(
|
||||
"UPDATE node_requests SET status = 'done', result_status = :status, "
|
||||
"result_headers = :headers, result_body = :body, result_bytes = :bytes, "
|
||||
"truncated = :truncated, finished_at = :now "
|
||||
"WHERE uid = :uid AND status = 'assigned'",
|
||||
{
|
||||
"status": int(result_status),
|
||||
"headers": result_headers,
|
||||
"body": result_body,
|
||||
"bytes": int(result_bytes),
|
||||
"truncated": 1 if truncated else 0,
|
||||
"now": _now_iso(),
|
||||
"uid": request_uid,
|
||||
},
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def set_request_transport(request_uid: str, transport: str) -> bool:
|
||||
return (
|
||||
_execute(
|
||||
"UPDATE node_requests SET mode = :mode WHERE uid = :uid AND status = 'assigned'",
|
||||
{"mode": transport, "uid": request_uid},
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def fail_request(request_uid: str, error: str, *, expired: bool = False) -> bool:
|
||||
return (
|
||||
_execute(
|
||||
"UPDATE node_requests SET status = :status, error = :error, finished_at = :now "
|
||||
"WHERE uid = :uid AND status IN ('queued','assigned')",
|
||||
{
|
||||
"status": "expired" if expired else "failed",
|
||||
"error": (error or "")[:EVENT_DETAIL_MAX],
|
||||
"now": _now_iso(),
|
||||
"uid": request_uid,
|
||||
},
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def expire_requests(moment: datetime | None = None) -> dict[str, int]:
|
||||
now = moment or datetime.now(timezone.utc)
|
||||
queued_cutoff = (now - timedelta(seconds=LIMITS.queue_max_wait_seconds)).isoformat()
|
||||
assigned_cutoff = (
|
||||
now - timedelta(seconds=LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds)
|
||||
).isoformat()
|
||||
finished = now.isoformat()
|
||||
queued = _execute(
|
||||
"UPDATE node_requests SET status = 'expired', error = 'no_node', finished_at = :now "
|
||||
"WHERE status = 'queued' AND created_at < :cutoff",
|
||||
{"now": finished, "cutoff": queued_cutoff},
|
||||
)
|
||||
assigned = _execute(
|
||||
"UPDATE node_requests SET status = 'failed', error = 'timeout', finished_at = :now "
|
||||
"WHERE status = 'assigned' AND assigned_at < :cutoff",
|
||||
{"now": finished, "cutoff": assigned_cutoff},
|
||||
)
|
||||
return {"queued": queued, "assigned": assigned}
|
||||
|
||||
|
||||
def reserve_caller_request(
|
||||
caller_kind: str, caller_id: str, limit: int, *, moment: datetime | None = None
|
||||
) -> bool:
|
||||
if limit <= 0 or not caller_id:
|
||||
return False
|
||||
now = (moment or datetime.now(timezone.utc)).isoformat()
|
||||
day = _day(moment)
|
||||
with db:
|
||||
db.executable.execute(
|
||||
text(
|
||||
"INSERT OR IGNORE INTO node_caller_daily (uid, caller_kind, caller_id, day, "
|
||||
"requests, created_at, updated_at) VALUES (:uid, :kind, :caller, :day, 0, :now, :now)"
|
||||
),
|
||||
{"uid": _uid(), "kind": caller_kind, "caller": caller_id, "day": day, "now": now},
|
||||
)
|
||||
counted = db.executable.execute(
|
||||
text(
|
||||
"UPDATE node_caller_daily SET requests = COALESCE(requests, 0) + 1, "
|
||||
"updated_at = :now WHERE caller_kind = :kind AND caller_id = :caller "
|
||||
"AND day = :day AND deleted_at IS NULL AND COALESCE(requests, 0) < :limit"
|
||||
),
|
||||
{"now": now, "kind": caller_kind, "caller": caller_id, "day": day, "limit": int(limit)},
|
||||
).rowcount
|
||||
return counted == 1
|
||||
|
||||
|
||||
def caller_requests_today(caller_kind: str, caller_id: str) -> int:
|
||||
row = get_table("node_caller_daily").find_one(
|
||||
caller_kind=caller_kind, caller_id=caller_id, day=_day(), deleted_at=None
|
||||
)
|
||||
return int((row or {}).get("requests") or 0)
|
||||
|
||||
|
||||
def get_settings_by_users(user_uids) -> dict[str, dict]:
|
||||
found: dict[str, dict] = {}
|
||||
for batch in _batches(user_uids):
|
||||
placeholders, params = _in_clause(batch)
|
||||
for row in db.query(
|
||||
f"SELECT * FROM node_settings WHERE user_uid IN ({placeholders}) "
|
||||
"AND deleted_at IS NULL",
|
||||
**params,
|
||||
):
|
||||
found[row["user_uid"]] = normalise_settings(dict(row))
|
||||
return found
|
||||
|
||||
|
||||
def donation_days(user_uid: str) -> int:
|
||||
rows = db.query(
|
||||
"SELECT COUNT(DISTINCT day) AS days FROM node_usage_daily "
|
||||
"WHERE user_uid = :user AND COALESCE(requests_ok, 0) > 0 AND deleted_at IS NULL",
|
||||
user=user_uid,
|
||||
)
|
||||
return int(next(iter(rows))["days"] or 0)
|
||||
|
||||
|
||||
def public_stats() -> dict:
|
||||
row = next(
|
||||
iter(
|
||||
db.query(
|
||||
"SELECT COALESCE(SUM(requests_ok), 0) AS requests, "
|
||||
"COALESCE(SUM(bytes_total), 0) AS bytes, COUNT(*) AS donors "
|
||||
"FROM node_donors WHERE deleted_at IS NULL AND COALESCE(requests_ok, 0) > 0"
|
||||
)
|
||||
)
|
||||
)
|
||||
return {
|
||||
"requests": int(row["requests"] or 0),
|
||||
"bytes": int(row["bytes"] or 0),
|
||||
"donors": int(row["donors"] or 0),
|
||||
}
|
||||
|
||||
|
||||
def top_donors(limit: int = 10) -> list[dict]:
|
||||
limit = max(1, min(int(limit), 50))
|
||||
rows = db.query(
|
||||
"SELECT d.user_uid AS user_uid, d.requests_ok AS requests_ok, d.bytes_total AS bytes_total "
|
||||
"FROM node_donors d JOIN node_settings s ON s.user_uid = d.user_uid "
|
||||
"WHERE d.deleted_at IS NULL AND s.deleted_at IS NULL AND COALESCE(s.public_display, 0) = 1 "
|
||||
"AND COALESCE(d.requests_ok, 0) > 0 ORDER BY d.bytes_total DESC LIMIT :limit",
|
||||
limit=limit,
|
||||
)
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
ADMIN_LIST_LIMIT = 100
|
||||
ADMIN_BREAKDOWN_LIMIT = 50
|
||||
DEVICE_BREAKDOWN_COLUMNS = ("status", "kind", "asn_class", "country")
|
||||
REQUEST_STATUSES = ("queued", "assigned", "done", "failed", "expired")
|
||||
|
||||
|
||||
def device_breakdown(column: str) -> dict[str, int]:
|
||||
if column not in DEVICE_BREAKDOWN_COLUMNS:
|
||||
raise ValueError("Unknown device breakdown column")
|
||||
rows = db.query(
|
||||
f"SELECT COALESCE(NULLIF({column}, ''), 'unknown') AS bucket, COUNT(*) AS total "
|
||||
"FROM node_devices WHERE deleted_at IS NULL "
|
||||
"GROUP BY bucket ORDER BY total DESC, bucket LIMIT :n",
|
||||
n=ADMIN_BREAKDOWN_LIMIT,
|
||||
)
|
||||
return {row["bucket"]: int(row["total"]) for row in rows}
|
||||
|
||||
|
||||
def request_status_counts() -> dict[str, int]:
|
||||
counts = {status: 0 for status in REQUEST_STATUSES}
|
||||
for row in db.query("SELECT status, COUNT(*) AS total FROM node_requests GROUP BY status"):
|
||||
counts[row["status"]] = int(row["total"])
|
||||
return counts
|
||||
|
||||
|
||||
def usage_day_totals(moment: datetime | None = None) -> dict[str, int]:
|
||||
row = next(
|
||||
iter(
|
||||
db.query(
|
||||
"SELECT COALESCE(SUM(requests_ok), 0) AS requests_ok, "
|
||||
"COALESCE(SUM(requests_failed), 0) AS requests_failed, "
|
||||
"COALESCE(SUM(bytes_up), 0) AS bytes_up, "
|
||||
"COALESCE(SUM(bytes_down), 0) AS bytes_down, "
|
||||
"COALESCE(SUM(cache_hits_served), 0) AS cache_hits_served, "
|
||||
"COALESCE(SUM(integrity_failures), 0) AS integrity_failures "
|
||||
"FROM node_usage_daily WHERE day = :day AND deleted_at IS NULL",
|
||||
day=_day(moment),
|
||||
)
|
||||
)
|
||||
)
|
||||
return {key: int(value or 0) for key, value in dict(row).items()}
|
||||
|
||||
|
||||
def recent_devices(limit: int = ADMIN_LIST_LIMIT) -> list[dict]:
|
||||
limit = max(1, min(int(limit), ADMIN_LIST_LIMIT))
|
||||
return [
|
||||
dict(row)
|
||||
for row in db.query(
|
||||
"SELECT * FROM node_devices WHERE deleted_at IS NULL "
|
||||
"ORDER BY COALESCE(NULLIF(last_seen_at, ''), created_at) DESC, id DESC LIMIT :n",
|
||||
n=limit,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -1854,6 +1854,8 @@ def init_db():
|
||||
_ensure_moderation_tables()
|
||||
_ensure_skills_table()
|
||||
_ensure_nodes_tables()
|
||||
_ensure_sync_tables()
|
||||
_ensure_community_tables()
|
||||
|
||||
for table in db.tables:
|
||||
_uid_index(db, table)
|
||||
@@ -2014,6 +2016,8 @@ def init_db():
|
||||
"nodes_daily_quota_guest": str(NODES_SPEC.limits.default_daily_requests_guest),
|
||||
"nodes_cache_ttl_seconds": str(NODES_SPEC.limits.cache_ttl_seconds),
|
||||
"nodes_social_via_nodes": "0",
|
||||
"sync_enabled": "0",
|
||||
"sync_origins": "",
|
||||
}
|
||||
for key, value in operational_defaults.items():
|
||||
existing = db["site_settings"].find_one(key=key)
|
||||
@@ -2285,11 +2289,24 @@ NODES_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
|
||||
("result_status", 0),
|
||||
("result_bytes", 0),
|
||||
("result_body", ""),
|
||||
("result_headers", ""),
|
||||
("truncated", 0),
|
||||
("error", ""),
|
||||
("created_at", ""),
|
||||
("assigned_at", ""),
|
||||
("finished_at", ""),
|
||||
),
|
||||
"node_caller_daily": (
|
||||
("uid", ""),
|
||||
("caller_kind", ""),
|
||||
("caller_id", ""),
|
||||
("day", ""),
|
||||
("requests", 0),
|
||||
("created_at", ""),
|
||||
("updated_at", ""),
|
||||
("deleted_at", ""),
|
||||
("deleted_by", ""),
|
||||
),
|
||||
"node_events": (
|
||||
("uid", ""),
|
||||
("device_uid", ""),
|
||||
@@ -2343,6 +2360,14 @@ def _ensure_nodes_tables() -> None:
|
||||
where="status IN ('queued','assigned')",
|
||||
)
|
||||
_index(db, "node_requests", "idx_node_requests_created", ["created_at"])
|
||||
_index(
|
||||
db,
|
||||
"node_caller_daily",
|
||||
"idx_node_caller_day",
|
||||
["caller_kind", "caller_id", "day"],
|
||||
unique=True,
|
||||
)
|
||||
_index(db, "node_caller_daily", "idx_node_caller_day_only", ["day"])
|
||||
_index(db, "node_events", "idx_node_events_created", ["created_at"])
|
||||
_index(db, "node_events", "idx_node_events_device", ["device_uid", "created_at"])
|
||||
_index(
|
||||
@@ -2354,6 +2379,120 @@ def _ensure_nodes_tables() -> None:
|
||||
)
|
||||
|
||||
|
||||
SYNC_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
|
||||
"sync_links": (
|
||||
("uid", ""),
|
||||
("origin", ""),
|
||||
("kind", ""),
|
||||
("origin_uid", ""),
|
||||
("local_uid", ""),
|
||||
("fingerprint", ""),
|
||||
("origin_updated_at", ""),
|
||||
("first_synced_at", ""),
|
||||
("synced_at", ""),
|
||||
),
|
||||
"sync_identities": (
|
||||
("uid", ""),
|
||||
("origin", ""),
|
||||
("source_name", ""),
|
||||
("source_key", ""),
|
||||
("our_name", ""),
|
||||
("our_key", ""),
|
||||
("style_fingerprint", ""),
|
||||
("style_samples", ""),
|
||||
("ai_generated", 0),
|
||||
("created_at", ""),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _ensure_sync_tables() -> None:
|
||||
for table_name, columns in SYNC_COLUMNS.items():
|
||||
table = get_table(table_name)
|
||||
for column, example in columns:
|
||||
if not table.has_column(column):
|
||||
table.create_column_by_example(column, example)
|
||||
_index(
|
||||
db,
|
||||
"sync_links",
|
||||
"idx_sync_links_origin",
|
||||
["origin", "kind", "origin_uid"],
|
||||
unique=True,
|
||||
)
|
||||
_index(db, "sync_links", "idx_sync_links_local", ["kind", "local_uid"])
|
||||
_index(
|
||||
db,
|
||||
"sync_identities",
|
||||
"idx_sync_identities_source",
|
||||
["origin", "source_key"],
|
||||
unique=True,
|
||||
)
|
||||
_index(db, "sync_identities", "idx_sync_identities_our", ["our_key"], unique=True)
|
||||
|
||||
|
||||
COMMUNITY_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
|
||||
"communities": (
|
||||
("uid", ""),
|
||||
("slug", ""),
|
||||
("name", ""),
|
||||
("origin", ""),
|
||||
("url", ""),
|
||||
("query", ""),
|
||||
("status", ""),
|
||||
("description", ""),
|
||||
("rules", ""),
|
||||
("language", ""),
|
||||
("favicon", ""),
|
||||
("user_count", 0),
|
||||
("schema", ""),
|
||||
("watermark", ""),
|
||||
("enabled", 1),
|
||||
("interval_seconds", 0),
|
||||
("next_run_at", ""),
|
||||
("last_run_at", ""),
|
||||
("last_status", ""),
|
||||
("last_error", ""),
|
||||
("run_count", 0),
|
||||
("failure_count", 0),
|
||||
("totals", ""),
|
||||
("learned_at", ""),
|
||||
("created_at", ""),
|
||||
("created_by", ""),
|
||||
("login_username", ""),
|
||||
("login_email", ""),
|
||||
("login_password", ""),
|
||||
("sync_options", ""),
|
||||
("claimed_at", ""),
|
||||
("deleted_at", ""),
|
||||
("deleted_by", ""),
|
||||
),
|
||||
"community_runs": (
|
||||
("uid", ""),
|
||||
("community_uid", ""),
|
||||
("started_at", ""),
|
||||
("finished_at", ""),
|
||||
("status", ""),
|
||||
("dry_run", 0),
|
||||
("trigger", ""),
|
||||
("collected", ""),
|
||||
("pushed", ""),
|
||||
("errors", ""),
|
||||
("log", ""),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _ensure_community_tables() -> None:
|
||||
for table_name, columns in COMMUNITY_COLUMNS.items():
|
||||
table = get_table(table_name)
|
||||
for column, example in columns:
|
||||
if not table.has_column(column):
|
||||
table.create_column_by_example(column, example)
|
||||
_index(db, "communities", "idx_communities_slug", ["slug"], unique=True, where="deleted_at IS NULL")
|
||||
_index(db, "communities", "idx_communities_due", ["status", "enabled", "next_run_at"])
|
||||
_index(db, "community_runs", "idx_community_runs_community", ["community_uid", "started_at"])
|
||||
|
||||
|
||||
def _ensure_moderation_tables() -> None:
|
||||
for table_name, columns in MODERATION_COLUMNS.items():
|
||||
table = get_table(table_name)
|
||||
|
||||
@@ -18,11 +18,13 @@ SOFT_DELETE_TABLES = [
|
||||
"bookmarks",
|
||||
"notes",
|
||||
"skills",
|
||||
"communities",
|
||||
"node_devices",
|
||||
"node_settings",
|
||||
"node_usage_daily",
|
||||
"node_donors",
|
||||
"node_flags",
|
||||
"node_caller_daily",
|
||||
"follows",
|
||||
"poll_votes",
|
||||
"polls",
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from devplacepy.utils.text import generate_uid
|
||||
|
||||
from .core import _now_iso, db, get_table
|
||||
|
||||
|
||||
def get_link(origin: str, kind: str, origin_uid: str) -> dict | None:
|
||||
return get_table("sync_links").find_one(
|
||||
origin=origin, kind=kind, origin_uid=origin_uid
|
||||
)
|
||||
|
||||
|
||||
def get_links_by_origin_uids(
|
||||
origin: str, kind: str, origin_uids: list[str]
|
||||
) -> dict[str, dict]:
|
||||
uids = [uid for uid in dict.fromkeys(origin_uids) if uid]
|
||||
if not uids:
|
||||
return {}
|
||||
table = get_table("sync_links")
|
||||
column = table.table.columns.origin_uid
|
||||
rows = table.find(column.in_(uids), origin=origin, kind=kind)
|
||||
return {row["origin_uid"]: row for row in rows}
|
||||
|
||||
|
||||
def claim_link(
|
||||
origin: str,
|
||||
kind: str,
|
||||
origin_uid: str,
|
||||
local_uid: str,
|
||||
fingerprint: str,
|
||||
origin_updated_at: str,
|
||||
) -> bool:
|
||||
now = _now_iso()
|
||||
with db:
|
||||
result = db.executable.execute(
|
||||
text(
|
||||
"INSERT OR IGNORE INTO sync_links (uid, origin, kind, origin_uid, "
|
||||
"local_uid, fingerprint, origin_updated_at, first_synced_at, synced_at) "
|
||||
"VALUES (:uid, :origin, :kind, :origin_uid, :local_uid, :fingerprint, "
|
||||
":origin_updated_at, :now, :now)"
|
||||
),
|
||||
{
|
||||
"uid": generate_uid(),
|
||||
"origin": origin,
|
||||
"kind": kind,
|
||||
"origin_uid": origin_uid,
|
||||
"local_uid": local_uid,
|
||||
"fingerprint": fingerprint,
|
||||
"origin_updated_at": origin_updated_at,
|
||||
"now": now,
|
||||
},
|
||||
)
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
def release_link(origin: str, kind: str, origin_uid: str, local_uid: str) -> None:
|
||||
get_table("sync_links").delete(
|
||||
origin=origin, kind=kind, origin_uid=origin_uid, local_uid=local_uid
|
||||
)
|
||||
|
||||
|
||||
def advance_link(link_uid: str, fingerprint: str, origin_updated_at: str) -> None:
|
||||
get_table("sync_links").update(
|
||||
{
|
||||
"uid": link_uid,
|
||||
"fingerprint": fingerprint,
|
||||
"origin_updated_at": origin_updated_at,
|
||||
"synced_at": _now_iso(),
|
||||
},
|
||||
["uid"],
|
||||
)
|
||||
|
||||
|
||||
def username_taken(username: str) -> bool:
|
||||
return get_table("users").find_one(username=username) is not None
|
||||
|
||||
|
||||
def count_links(origin: str) -> int:
|
||||
return get_table("sync_links").count(origin=origin)
|
||||
|
||||
|
||||
def identity_map(origin: str) -> dict[str, dict]:
|
||||
return {row["source_key"]: row for row in get_table("sync_identities").find(origin=origin)}
|
||||
|
||||
|
||||
def identity_by_source(origin: str, source_key: str) -> dict | None:
|
||||
return get_table("sync_identities").find_one(origin=origin, source_key=source_key)
|
||||
|
||||
|
||||
def identity_name_taken(our_key: str) -> bool:
|
||||
return get_table("sync_identities").find_one(our_key=our_key) is not None
|
||||
|
||||
|
||||
def claim_identity(
|
||||
origin: str,
|
||||
source_name: str,
|
||||
source_key: str,
|
||||
our_name: str,
|
||||
style_fingerprint: str,
|
||||
ai_generated: bool,
|
||||
) -> bool:
|
||||
with db:
|
||||
result = db.executable.execute(
|
||||
text(
|
||||
"INSERT OR IGNORE INTO sync_identities (uid, origin, source_name, source_key, "
|
||||
"our_name, our_key, style_fingerprint, style_samples, ai_generated, created_at) "
|
||||
"VALUES (:uid, :origin, :source_name, :source_key, :our_name, :our_key, "
|
||||
":style_fingerprint, '[]', :ai_generated, :now)"
|
||||
),
|
||||
{
|
||||
"uid": generate_uid(),
|
||||
"origin": origin,
|
||||
"source_name": source_name,
|
||||
"source_key": source_key,
|
||||
"our_name": our_name,
|
||||
"our_key": our_name.lower(),
|
||||
"style_fingerprint": style_fingerprint,
|
||||
"ai_generated": int(ai_generated),
|
||||
"now": _now_iso(),
|
||||
},
|
||||
)
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
def save_style_samples(identity_uid: str, samples_json: str) -> None:
|
||||
get_table("sync_identities").update({"uid": identity_uid, "style_samples": samples_json}, ["uid"])
|
||||
@@ -25,6 +25,8 @@ from . import (
|
||||
battles,
|
||||
skills,
|
||||
nodes,
|
||||
sync,
|
||||
community,
|
||||
)
|
||||
|
||||
ORDERED_GROUPS = [
|
||||
@@ -52,4 +54,6 @@ ORDERED_GROUPS = [
|
||||
battles.GROUP,
|
||||
skills.GROUP,
|
||||
nodes.GROUP,
|
||||
sync.GROUP,
|
||||
community.GROUP,
|
||||
]
|
||||
|
||||
@@ -545,6 +545,23 @@ four ways to sign requests.
|
||||
"Allow signups.",
|
||||
["1", "0"],
|
||||
),
|
||||
field(
|
||||
"sync_enabled",
|
||||
"form",
|
||||
"enum",
|
||||
False,
|
||||
"0",
|
||||
"Allow community sync.",
|
||||
["1", "0"],
|
||||
),
|
||||
field(
|
||||
"sync_origins",
|
||||
"form",
|
||||
"string",
|
||||
False,
|
||||
"other.example.org",
|
||||
"Comma separated origins allowed to sync; empty allows any.",
|
||||
),
|
||||
field(
|
||||
"maintenance_mode",
|
||||
"form",
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .._shared import endpoint, field
|
||||
|
||||
SLUG = field("slug", "path", "string", True, "other-example-org", "Community slug, as listed by GET /admin/communities.")
|
||||
ACCOUNT = [
|
||||
field("login_username", "form", "string", False, "bot", "Username of an account on the community, for a login-protected community."),
|
||||
field("login_email", "form", "string", False, "bot@example.com", "Email of that account, when the community logs in by email."),
|
||||
field("login_password", "form", "string", False, "", "Password of that account. Stored for the collectors and never returned by any endpoint."),
|
||||
]
|
||||
|
||||
ITEM_SAMPLE = {
|
||||
"uid": "COMMUNITY_UID",
|
||||
"slug": "other-example-org",
|
||||
"name": "Other Example",
|
||||
"origin": "other.example.org",
|
||||
"url": "https://other.example.org/",
|
||||
"status": "ready",
|
||||
"enabled": True,
|
||||
"interval_seconds": 86400,
|
||||
"run_count": 3,
|
||||
"failure_count": 0,
|
||||
"last_run_at": "2026-10-14T10:00:00+00:00",
|
||||
"next_run_at": "2026-10-15T10:00:00+00:00",
|
||||
"has_login": False,
|
||||
}
|
||||
|
||||
GROUP = {
|
||||
"slug": "community",
|
||||
"title": "Community discovery",
|
||||
"admin": True,
|
||||
"intro": """
|
||||
# Community discovery
|
||||
|
||||
Administrators add an external community by name or address. The system searches for it, detects
|
||||
its feeds (RSS, Atom, JSON or HTML lists, with JavaScript rendering when a page needs it), learns
|
||||
the structure once, records the community's description, rules and size, generates collectors and
|
||||
then runs them on a schedule, pushing users, posts and comments through the
|
||||
[community sync API](/docs/sync.html). Members never see these communities.
|
||||
|
||||
- **Statuses.** `pending`, `learning`, `ready`, `login_required`, `failed` and `failing` (five failed
|
||||
runs in a row: the community is disabled and needs a human).
|
||||
- **Login-protected communities.** Give `login_username` or `login_email` plus `login_password`; the
|
||||
system finds the login form (or, for a page that needs JavaScript, drives it in the browser), logs
|
||||
in before discovery and before every run. The password is never returned.
|
||||
- **Dry run.** `dry_run=true` on a run validates and logs everything and writes nothing.
|
||||
- **Asynchronous.** Adding, relearning and running return at once; read the result from the detail
|
||||
endpoint.
|
||||
- **AI assistance.** For HTML pages whose structure the heuristics cannot read, the internal AI model
|
||||
proposes selectors, and a proposal is only used when it matches the sampled pages.
|
||||
|
||||
Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html).
|
||||
""",
|
||||
"endpoints": [
|
||||
endpoint(
|
||||
id="community-list",
|
||||
method="GET",
|
||||
path="/admin/communities",
|
||||
title="List communities",
|
||||
summary="Every community with status, schedule and last run, plus the state of the sync switches.",
|
||||
auth="admin",
|
||||
interactive=True,
|
||||
sample_response={"items": [ITEM_SAMPLE], "sync_enabled": True, "service_enabled": True, "service_dry_run": True},
|
||||
),
|
||||
endpoint(
|
||||
id="community-add",
|
||||
method="POST",
|
||||
path="/admin/communities",
|
||||
title="Add a community",
|
||||
summary="Starts discovery in the background and redirects to the community.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
interactive=False,
|
||||
params=[field("query", "form", "string", True, "https://forum.example.com", "Community name or address."), *ACCOUNT],
|
||||
sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
|
||||
),
|
||||
endpoint(
|
||||
id="community-detail",
|
||||
method="GET",
|
||||
path="/admin/communities/{slug}",
|
||||
title="Community detail",
|
||||
summary="Metadata, learned feeds, totals and recent runs with their logs.",
|
||||
auth="admin",
|
||||
interactive=True,
|
||||
params=[SLUG],
|
||||
sample_response={
|
||||
"community": ITEM_SAMPLE,
|
||||
"rules": "Be kind.",
|
||||
"totals": {"posts_created": 120},
|
||||
"feeds": [{"id": "feed1", "type": "json_api", "url": "https://other.example.org/api/posts", "comments": "inline", "pagination": "page"}],
|
||||
"runs": [{"status": "ok", "dry_run": 0, "trigger": "service", "collected": {"users": 4, "posts": 20, "comments": 31}}],
|
||||
"busy": False,
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="community-run",
|
||||
method="POST",
|
||||
path="/admin/communities/{slug}/run",
|
||||
title="Run now",
|
||||
summary="Collects and pushes in the background. A live run needs the sync_enabled setting.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
interactive=False,
|
||||
params=[SLUG, field("dry_run", "form", "boolean", False, "false", "Validate and log, write nothing, keep the watermark.")],
|
||||
sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
|
||||
),
|
||||
endpoint(
|
||||
id="community-relearn",
|
||||
method="POST",
|
||||
path="/admin/communities/{slug}/relearn",
|
||||
title="Relearn",
|
||||
summary="Learns the structure again with the stored account. A failed relearn keeps the working schema.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
interactive=False,
|
||||
params=[SLUG],
|
||||
sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
|
||||
),
|
||||
endpoint(
|
||||
id="community-update",
|
||||
method="POST",
|
||||
path="/admin/communities/{slug}/update",
|
||||
title="Update schedule or account",
|
||||
summary="Changes the interval, the enabled flag and the stored account. An empty password keeps the stored one.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
interactive=False,
|
||||
params=[
|
||||
SLUG,
|
||||
field("interval_seconds", "form", "integer", True, "86400", "Seconds between runs, 900 to 2592000."),
|
||||
field("enabled", "form", "boolean", False, "true", "Whether the scheduled service runs this community."),
|
||||
*ACCOUNT,
|
||||
field("clear_login", "form", "boolean", False, "false", "Remove the stored account."),
|
||||
field("alias", "form", "boolean", False, "false", "Store authors under permanent pseudonyms and rewrite every mention. Refused once the origin has records under original names."),
|
||||
field("translate_to", "form", "string", False, "en", "Language code to translate collected posts and comments into; empty turns it off."),
|
||||
field("sample_percent", "form", "integer", False, "100", "Percentage of new posts accepted, 1 to 100, chosen deterministically per post."),
|
||||
field("min_comments", "form", "integer", False, "0", "Minimum comments collected in the run for a new post to be accepted."),
|
||||
field("topics", "form", "string", False, "devlog, rant", "Comma separated topic allow-list; empty accepts all."),
|
||||
field("jitter_percent", "form", "integer", False, "0", "Randomise the next run time by up to this percentage of the interval, 0 to 50."),
|
||||
],
|
||||
sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
|
||||
),
|
||||
endpoint(
|
||||
id="community-delete",
|
||||
method="POST",
|
||||
path="/admin/communities/{slug}/delete",
|
||||
title="Delete a community",
|
||||
summary="Moves the community to the trash and stops its collectors. Content already synchronised stays.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
interactive=False,
|
||||
params=[SLUG],
|
||||
sample_response={"ok": True, "redirect": "/admin/communities"},
|
||||
),
|
||||
],
|
||||
}
|
||||
@@ -154,5 +154,265 @@ page. Every endpoint follows the shared [Conventions & Errors](/docs/conventions
|
||||
"`403` with a `reason` when the device is suspended or its owner no longer passes the consent gate.",
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="nodes-me",
|
||||
method="GET",
|
||||
path="/nodes/me",
|
||||
title="Your donation status",
|
||||
summary="Whether you may donate right now (and the reason when not), your settings, your devices and what you have donated.",
|
||||
auth="user",
|
||||
interactive=True,
|
||||
sample_response={
|
||||
"consent": True,
|
||||
"can_donate": True,
|
||||
"reason": "",
|
||||
"message": "",
|
||||
"settings": {"enabled": True, "bandwidth_mb_day": 500, "max_concurrent": 2},
|
||||
"platforms": ["reddit"],
|
||||
"devices": [DEVICE_SAMPLE],
|
||||
"max_devices": 5,
|
||||
"donated": {"lifetime_requests": 12, "lifetime_bytes": 482113, "today_requests": 3},
|
||||
"limits": {"max_donor_mb_per_day": 20480, "max_concurrent_per_node": 2, "default_donor_mb_per_day": 500},
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="nodes-settings",
|
||||
method="POST",
|
||||
path="/nodes/settings",
|
||||
title="Change your donation settings",
|
||||
summary="Update bandwidth, concurrency, active hours, platforms and display. Unsent fields keep their value.",
|
||||
auth="user",
|
||||
ajax=True,
|
||||
encoding="form",
|
||||
params=[
|
||||
field("enabled", "form", "boolean", False, "true", "Switch donation on or off. Switching on needs the `bandwidth_sharing` consent and is refused when sent by the Devii assistant."),
|
||||
field("bandwidth_mb_day", "form", "integer", False, "500", "Daily bandwidth cap in megabytes."),
|
||||
field("max_concurrent", "form", "integer", False, "2", "Simultaneous requests."),
|
||||
field("active_from", "form", "string", False, "08:00", "Start of the active window, HH:MM."),
|
||||
field("active_to", "form", "string", False, "22:00", "End of the active window, HH:MM."),
|
||||
field("timezone", "form", "string", False, "Europe/Amsterdam", "IANA timezone of the window."),
|
||||
field("allowed_platforms", "form", "string", False, "reddit", "Comma separated platforms you are willing to serve."),
|
||||
field("pause_on_battery", "form", "boolean", False, "true", "Stop donating on battery power."),
|
||||
field("public_display", "form", "boolean", False, "false", "Show your name on the donor leaderboard."),
|
||||
],
|
||||
sample_response={"consent": True, "can_donate": True, "settings": {"enabled": True}},
|
||||
notes=["`403` with `reason: no_consent` when switching on without the consent."],
|
||||
),
|
||||
endpoint(
|
||||
id="nodes-stats",
|
||||
method="GET",
|
||||
path="/nodes/stats",
|
||||
title="Community donation totals",
|
||||
summary="Aggregate numbers only: requests relayed, bytes relayed and number of donors. No per-member data.",
|
||||
auth="public",
|
||||
interactive=True,
|
||||
sample_response={"requests": 1200, "bytes": 90311234, "donors": 7},
|
||||
),
|
||||
endpoint(
|
||||
id="nodes-leaderboard",
|
||||
method="GET",
|
||||
path="/nodes/leaderboard",
|
||||
title="Donor leaderboard",
|
||||
summary="The top donors who chose to be shown. Members are not listed unless they switched on public display.",
|
||||
auth="public",
|
||||
interactive=True,
|
||||
sample_response={"donors": [{"username": "alice", "requests": 400, "bytes": 31200000}]},
|
||||
),
|
||||
endpoint(
|
||||
id="nodes-social-search",
|
||||
method="GET",
|
||||
path="/nodes/social/search",
|
||||
title="Social search",
|
||||
summary="Search public developer discussion on Hacker News, Stack Overflow, GitHub, dev.to, Mastodon and, when donors are online, Reddit.",
|
||||
auth="user",
|
||||
interactive=True,
|
||||
params=[
|
||||
field("q", "query", "string", True, "sqlite wal", "What to search for, up to 200 characters."),
|
||||
field("platforms", "query", "string", False, "hackernews,github", "Comma separated subset. Empty means all."),
|
||||
field("limit", "query", "integer", False, "8", "Hits per platform, 1 to 20."),
|
||||
],
|
||||
sample_response={
|
||||
"query": "sqlite wal",
|
||||
"platforms": ["hackernews", "reddit"],
|
||||
"hits": [
|
||||
{
|
||||
"platform": "hackernews",
|
||||
"title": "SQLite WAL mode explained",
|
||||
"url": "https://news.ycombinator.com/item?id=1",
|
||||
"snippet": "",
|
||||
"author": "someone",
|
||||
"score": 120,
|
||||
"comments": 33,
|
||||
"created_at": "2026-01-01T00:00:00Z",
|
||||
"via": "direct",
|
||||
}
|
||||
],
|
||||
"unavailable": {"reddit": "no_nodes"},
|
||||
},
|
||||
notes=[
|
||||
"Platforms that could not answer are listed under `unavailable` with a reason code; results are never invented for them.",
|
||||
"Counted against a daily per-member quota (`429` when reached). Bluesky and Lobsters are not covered because they block unauthenticated server access.",
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="nodes-internal-fetch",
|
||||
method="POST",
|
||||
path="/nodes/internal/fetch",
|
||||
title="Fetch a page through the donor network",
|
||||
summary="Fetch one page from an allowlisted site through a donor node. Meant for agents and services that hold an API key.",
|
||||
auth="user",
|
||||
encoding="form",
|
||||
params=[
|
||||
field("url", "form", "string", True, "https://www.reddit.com/r/programming/.json", "Absolute URL on an allowlisted host."),
|
||||
field("cache", "form", "boolean", False, "true", "Allow a cached copy."),
|
||||
field("timeout", "form", "integer", False, "15", "Seconds to wait, 1 to 60."),
|
||||
],
|
||||
sample_response={
|
||||
"url": "https://www.reddit.com/r/programming/.json",
|
||||
"status": 200,
|
||||
"via": "fetch",
|
||||
"truncated": False,
|
||||
"elapsed_ms": 812,
|
||||
"content_type": "application/json",
|
||||
"text": "{...}",
|
||||
},
|
||||
notes=[
|
||||
"`403` with `reason: host_not_allowed` for any host that is not on the allowlist, `503` with `reason: no_nodes` when no donor is online, `429` when the daily node quota is reached.",
|
||||
"`via` says how the body arrived: `fetch` (relayed by a donor browser, treat as unverified) or `cache`.",
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="nodes-client-version",
|
||||
method="GET",
|
||||
path="/nodes/client-version",
|
||||
title="Client versions and checksums",
|
||||
summary="The protocol versions this server speaks and every downloadable client with its SHA-256.",
|
||||
auth="public",
|
||||
interactive=True,
|
||||
sample_response={
|
||||
"version": "0.1.0",
|
||||
"protocol": 1,
|
||||
"min_protocol": 1,
|
||||
"artifacts": [{"name": "devplace-node-linux-amd64", "size": 6291456, "sha256": "ab12..."}],
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="nodes-download",
|
||||
method="GET",
|
||||
path="/nodes/download/{artifact}",
|
||||
title="Download a client",
|
||||
summary="Download one client artifact listed by the client version endpoint. The SHA-256 is also sent as `X-Content-SHA256`.",
|
||||
auth="public",
|
||||
params=[
|
||||
field("artifact", "path", "string", True, "devplace-node-linux-amd64", "Artifact name."),
|
||||
],
|
||||
sample_response="(binary file)",
|
||||
),
|
||||
endpoint(
|
||||
id="admin-nodes",
|
||||
method="GET",
|
||||
path="/admin/nodes",
|
||||
title="Node network overview",
|
||||
summary="Pool, queue, device breakdowns, today's totals, open flags, recent devices, settings and the kill switch.",
|
||||
auth="admin",
|
||||
interactive=True,
|
||||
sample_response={
|
||||
"enabled": True,
|
||||
"pool": {"online": True, "connected": 12, "capacity": 2000, "accepted": 40, "kicked": 3, "jobs_done": 310, "jobs_failed": 4, "open_circuits": 0, "heartbeat": "2026-01-01T00:00:00+00:00"},
|
||||
"queue": {"depth": 2, "max_depth": 500, "queued": 1, "assigned": 1, "done": 30, "failed": 1, "expired": 0},
|
||||
"devices_by_status": {"active": 14, "suspended": 1},
|
||||
"devices_by_kind": {"native": 10, "extension": 5},
|
||||
"devices_by_asn_class": {"unknown": 15},
|
||||
"devices_by_country": {"unknown": 15},
|
||||
"usage_today": {"requests_ok": 120, "requests_failed": 3, "bytes_up": 40000, "bytes_down": 9000000, "cache_hits_served": 0, "integrity_failures": 0},
|
||||
"flags": [{"uid": "FLAG_UID", "kind": "refusal_mismatch", "detail": "", "status": "open", "device_uid": "DEVICE_UID", "device_label": "Home laptop", "user_uid": "USER_UID", "owner_username": "alice", "created_at": "2026-01-01T00:00:00+00:00"}],
|
||||
"devices": [dict(DEVICE_SAMPLE, owner_uid="USER_UID", owner_username="alice", suspend_reason="")],
|
||||
"settings": [{"key": "nodes_max_total", "label": "Maximum connected nodes", "kind": "int", "value": "2000", "default": "2000", "ceiling": 2000}],
|
||||
"limits": {"max_devices_per_user": 5, "max_connected_nodes": 2000},
|
||||
},
|
||||
notes=[
|
||||
"Per-destination circuit state lives in the memory of the worker that holds the sockets and is not readable from other workers; only the number of open circuits is reported.",
|
||||
"`usage_today` is the current UTC day. Lists are capped at 100 rows.",
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="admin-nodes-killswitch",
|
||||
method="POST",
|
||||
path="/admin/nodes/killswitch",
|
||||
title="Switch community donation on or off",
|
||||
summary="The global kill switch. Switching it off disconnects every node within one reconcile tick.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
destructive=True,
|
||||
params=[
|
||||
field("enabled", "form", "boolean", False, "false", "True to switch on, false to switch off. Omit to flip the current state."),
|
||||
],
|
||||
sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"enabled": False}},
|
||||
notes=["Audited as `node.killswitch`."],
|
||||
),
|
||||
endpoint(
|
||||
id="admin-nodes-settings",
|
||||
method="POST",
|
||||
path="/admin/nodes/settings",
|
||||
title="Update node network settings",
|
||||
summary="Change the caps and quotas. Only the fields you send are changed; values above the ceiling are clamped to it.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
params=[
|
||||
field("nodes_max_total", "form", "integer", False, "2000", "Maximum connected nodes, 0 to 2000."),
|
||||
field("nodes_daily_quota_user", "form", "integer", False, "200", "Daily node requests per member, 0 to 10000."),
|
||||
field("nodes_daily_quota_guest", "form", "integer", False, "0", "Daily node requests per guest, 0 to 1000."),
|
||||
field("nodes_cache_ttl_seconds", "form", "integer", False, "600", "Result cache lifetime in seconds, 0 to 600."),
|
||||
field("nodes_daily_social_user", "form", "integer", False, "300", "Daily social searches per member, 0 to 10000."),
|
||||
field("nodes_social_via_nodes", "form", "string", False, "0", "`1` routes social search through nodes, `0` keeps it direct."),
|
||||
],
|
||||
sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"changed": ["nodes_max_total"], "clamped": []}},
|
||||
notes=["Audited as `node.admin.settings` with the old and new values."],
|
||||
),
|
||||
endpoint(
|
||||
id="admin-nodes-suspend",
|
||||
method="POST",
|
||||
path="/admin/nodes/devices/{device_uid}/suspend",
|
||||
title="Suspend a donor device",
|
||||
summary="Suspend one device for 1 to 10080 minutes. The device is disconnected on the next tick and cannot reconnect until the period ends.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
destructive=True,
|
||||
params=[
|
||||
field("device_uid", "path", "string", True, "DEVICE_UID", "Device uid."),
|
||||
field("minutes", "form", "integer", True, "60", "Suspension length in minutes, 1 to 10080."),
|
||||
field("reason", "form", "string", False, "Repeated refusals", "Reason, up to 500 characters."),
|
||||
],
|
||||
sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "DEVICE_UID", "minutes": 60}},
|
||||
notes=["Refused (audited as `denied`) when the device owner is a more senior administrator. Audited as `node.suspend`."],
|
||||
),
|
||||
endpoint(
|
||||
id="admin-nodes-unsuspend",
|
||||
method="POST",
|
||||
path="/admin/nodes/devices/{device_uid}/unsuspend",
|
||||
title="Lift a device suspension",
|
||||
summary="Return a suspended device to active.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
params=[
|
||||
field("device_uid", "path", "string", True, "DEVICE_UID", "Device uid."),
|
||||
],
|
||||
sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "DEVICE_UID"}},
|
||||
notes=["`409` when the device is not suspended. Audited as `node.unsuspend`."],
|
||||
),
|
||||
endpoint(
|
||||
id="admin-nodes-flag-resolve",
|
||||
method="POST",
|
||||
path="/admin/nodes/flags/{flag_uid}/resolve",
|
||||
title="Resolve a node flag",
|
||||
summary="Mark an open integrity or refusal flag as handled.",
|
||||
auth="admin",
|
||||
encoding="form",
|
||||
params=[
|
||||
field("flag_uid", "path", "string", True, "FLAG_UID", "Flag uid."),
|
||||
],
|
||||
sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "FLAG_UID"}},
|
||||
notes=["`409` when the flag is already resolved. Audited as `node.flag.resolve`."],
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .._shared import endpoint, field
|
||||
|
||||
ENVELOPE_PARAMS = [
|
||||
field("origin", "body", "string", True, "other.example.org", "Stable id of the sending community, 2-63 characters of a-z, 0-9, dots and hyphens. Lower-cased by the server."),
|
||||
field("records", "body", "array", True, "[]", "1-50 records of the kind this endpoint takes. Each record is validated on its own."),
|
||||
field("dry_run", "body", "boolean", False, "false", "Validate, normalize and report what would happen without writing anything."),
|
||||
]
|
||||
|
||||
RESULT_SAMPLE = {
|
||||
"origin": "other.example.org",
|
||||
"kind": "posts",
|
||||
"dry_run": False,
|
||||
"total": 2,
|
||||
"counts": {"created": 1, "updated": 0, "unchanged": 0, "stale": 0, "rejected": 1, "failed": 0},
|
||||
"results": [
|
||||
{"index": 0, "origin_uid": "p-1", "status": "created", "reason": "", "local_uid": "LOCAL_UID", "errors": [], "ignored": ["stars"]},
|
||||
{"index": 1, "origin_uid": "p-2", "status": "rejected", "reason": "unknown_author", "local_uid": "", "errors": ["author u-9 is not synced yet, send users first"], "ignored": []},
|
||||
],
|
||||
}
|
||||
|
||||
GROUP = {
|
||||
"slug": "sync",
|
||||
"title": "Community sync",
|
||||
"admin": True,
|
||||
"intro": """
|
||||
# Community sync
|
||||
|
||||
Copy users, posts and comments from another community into this one. The API is idempotent
|
||||
and safe to repeat: every record carries the id it has in the **origin** community
|
||||
(`origin_uid`), the server remembers which local row that id maps to, and a repeated record is
|
||||
an update or a no-op, never a duplicate.
|
||||
|
||||
- **Validated and normalized here.** The sender posts raw data; the server fixes what can be
|
||||
fixed (usernames, whitespace, control characters, unknown topics, timestamps) and rejects what
|
||||
cannot, record by record. One bad record never fails the batch.
|
||||
- **Counters are not imported.** Votes, stars, XP and levels cannot be verified across
|
||||
communities, so they are ignored and listed under `ignored` in the result.
|
||||
- **No side effects.** Synced content sends no notifications, awards no XP and runs no AI
|
||||
correction. Content that matches the content filter is rejected.
|
||||
- **Order matters.** Send users first, then posts, then comments (parents before replies).
|
||||
- **Off by default.** An administrator enables it at `/admin/settings` (`sync_enabled`) and may
|
||||
restrict senders with `sync_origins` (comma separated; empty allows any origin).
|
||||
- **JSON or form.** The body is JSON. A form-encoded body is accepted too, with `records` as a
|
||||
JSON array string; Devii uses that form through the `sync_users`, `sync_posts` and
|
||||
`sync_comments` tools.
|
||||
|
||||
The accepted fields and every limit are published at `/sync/spec.json`. Every endpoint follows
|
||||
the shared [Conventions & Errors](/docs/conventions.html).
|
||||
""",
|
||||
"endpoints": [
|
||||
endpoint(
|
||||
id="sync-spec",
|
||||
method="GET",
|
||||
path="/sync/spec.json",
|
||||
title="Sync specification",
|
||||
summary="Limits, statuses, refusal reasons and the JSON schema of each record kind.",
|
||||
auth="public",
|
||||
interactive=True,
|
||||
sample_response={
|
||||
"limits": {"max_batch_records": 50},
|
||||
"kinds": ["users", "posts", "comments"],
|
||||
"statuses": ["created", "updated", "unchanged", "stale", "rejected", "failed"],
|
||||
"reasons": ["invalid", "unknown_author"],
|
||||
"records": {"users": {"title": "UserRecord", "type": "object"}},
|
||||
},
|
||||
),
|
||||
endpoint(
|
||||
id="sync-users",
|
||||
method="POST",
|
||||
path="/sync/users",
|
||||
title="Upsert users",
|
||||
summary="Create or update synced accounts. A synced account cannot sign in; it only owns the content that was synced for it.",
|
||||
auth="admin",
|
||||
encoding="json",
|
||||
interactive=False,
|
||||
params=ENVELOPE_PARAMS,
|
||||
sample_response=RESULT_SAMPLE,
|
||||
notes=[
|
||||
"Record fields: `origin_uid`, `username`, `created_at`, optional `updated_at`, `bio`, `location`, `git_link`, `website`, `avatar_seed`, `deleted`.",
|
||||
"A username that is invalid is normalized to letters, digits, `_` and `-`. A name already taken by a different account gets the origin as a suffix, so the result may differ from the one sent.",
|
||||
"`deleted: true` deactivates the account. Email, password, role, XP and level are never accepted.",
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="sync-posts",
|
||||
method="POST",
|
||||
path="/sync/posts",
|
||||
title="Upsert posts",
|
||||
summary="Create or update posts. The author must already be synced.",
|
||||
auth="admin",
|
||||
encoding="json",
|
||||
interactive=False,
|
||||
params=ENVELOPE_PARAMS,
|
||||
sample_response=RESULT_SAMPLE,
|
||||
notes=[
|
||||
"Record fields: `origin_uid`, `author_uid` (the author's `origin_uid`), `content`, `created_at`, optional `title`, `topic`, `updated_at`, `deleted`.",
|
||||
"An unknown `topic` becomes `random`. Content shorter than 10 characters is rejected.",
|
||||
],
|
||||
),
|
||||
endpoint(
|
||||
id="sync-comments",
|
||||
method="POST",
|
||||
path="/sync/comments",
|
||||
title="Upsert comments",
|
||||
summary="Create or update comments on synced posts. The author, the post and any parent comment must already be synced.",
|
||||
auth="admin",
|
||||
encoding="json",
|
||||
interactive=False,
|
||||
params=ENVELOPE_PARAMS,
|
||||
sample_response=RESULT_SAMPLE,
|
||||
notes=[
|
||||
"Record fields: `origin_uid`, `author_uid`, `post_uid` (the post's `origin_uid`), `content`, `created_at`, optional `parent_uid`, `updated_at`, `deleted`.",
|
||||
"A parent that belongs to a different post is rejected with `parent_mismatch`.",
|
||||
],
|
||||
),
|
||||
],
|
||||
}
|
||||
@@ -53,6 +53,7 @@ from devplacepy.routers import (
|
||||
auth,
|
||||
battles,
|
||||
nodes,
|
||||
sync,
|
||||
feed,
|
||||
posts,
|
||||
topics,
|
||||
@@ -128,6 +129,7 @@ from devplacepy.services.audit import AuditService
|
||||
from devplacepy.services.moderation.service import ModerationService
|
||||
from devplacepy.services.moderation.screening import ContentRefused
|
||||
from devplacepy.services.acceptance.service import AcceptanceService
|
||||
from devplacepy.services.community.service import CommunityService
|
||||
from devplacepy.services.nodes.service import NodesService
|
||||
from devplacepy.services.audit import record as audit
|
||||
from devplacepy.services.push import PushService
|
||||
@@ -298,6 +300,7 @@ async def lifespan(app: FastAPI):
|
||||
service_manager.register(AuditService())
|
||||
service_manager.register(ModerationService())
|
||||
service_manager.register(AcceptanceService())
|
||||
service_manager.register(CommunityService())
|
||||
service_manager.register(NodesService())
|
||||
service_manager.register(PushService())
|
||||
service_manager.register(TelegramService())
|
||||
@@ -529,6 +532,7 @@ app.include_router(game.router, prefix="/game")
|
||||
app.include_router(quizzes.router, prefix="/quizzes")
|
||||
app.include_router(battles.router, prefix="/battles")
|
||||
app.include_router(nodes.router, prefix="/nodes")
|
||||
app.include_router(sync.router, prefix="/sync")
|
||||
app.include_router(workspaces.router, prefix="/workspaces")
|
||||
|
||||
|
||||
|
||||
@@ -328,6 +328,37 @@ class ProjectEditForm(BaseModel):
|
||||
return normalize_website_url(value)
|
||||
|
||||
|
||||
class CommunityAddForm(BaseModel):
|
||||
query: str = Field(min_length=2, max_length=200)
|
||||
login_username: str = Field(default="", max_length=200)
|
||||
login_email: str = Field(default="", max_length=254)
|
||||
login_password: str = Field(default="", max_length=500)
|
||||
|
||||
@field_validator("query")
|
||||
@classmethod
|
||||
def trimmed(cls, value):
|
||||
return value.strip()
|
||||
|
||||
|
||||
class CommunityUpdateForm(BaseModel):
|
||||
interval_seconds: int = Field(ge=900, le=2592000)
|
||||
enabled: bool = False
|
||||
login_username: str = Field(default="", max_length=200)
|
||||
login_email: str = Field(default="", max_length=254)
|
||||
login_password: str = Field(default="", max_length=500)
|
||||
clear_login: bool = False
|
||||
alias: Optional[bool] = None
|
||||
translate_to: Optional[str] = Field(default=None, max_length=12)
|
||||
sample_percent: Optional[int] = Field(default=None, ge=1, le=100)
|
||||
min_comments: Optional[int] = Field(default=None, ge=0, le=1000)
|
||||
topics: Optional[str] = Field(default=None, max_length=500)
|
||||
jitter_percent: Optional[int] = Field(default=None, ge=0, le=50)
|
||||
|
||||
|
||||
class CommunityRunForm(BaseModel):
|
||||
dry_run: bool = False
|
||||
|
||||
|
||||
class BackupRunForm(BaseModel):
|
||||
target: Literal["database", "uploads", "keys", "full"] = "full"
|
||||
|
||||
@@ -721,6 +752,16 @@ class AdminSettingsForm(BaseModel):
|
||||
modifier_model: str = Field(default="", max_length=120)
|
||||
quiz_grading_model: str = Field(default="", max_length=120)
|
||||
extra_head: str = Field(default="", max_length=50000)
|
||||
sync_enabled: str = Field(default="", max_length=1)
|
||||
sync_origins: str = Field(default="", max_length=1000)
|
||||
|
||||
@field_validator("sync_origins")
|
||||
@classmethod
|
||||
def validate_sync_origins(cls, value):
|
||||
from devplacepy.services.sync.normalize import normalize_origin
|
||||
|
||||
items = [item for item in value.replace("\n", ",").split(",") if item.strip()]
|
||||
return ",".join(dict.fromkeys(normalize_origin(item) for item in items))
|
||||
|
||||
@field_validator("moderation_filter_mode")
|
||||
@classmethod
|
||||
@@ -1124,6 +1165,59 @@ class BanForm(BaseModel):
|
||||
reason: str = Field(default="", max_length=200)
|
||||
|
||||
|
||||
class NodeSettingsForm(BaseModel):
|
||||
enabled: Optional[bool] = None
|
||||
bandwidth_mb_day: Optional[int] = Field(default=None, ge=1, le=20 * 1024)
|
||||
max_concurrent: Optional[int] = Field(default=None, ge=1, le=16)
|
||||
active_from: Optional[str] = Field(default=None, max_length=5)
|
||||
active_to: Optional[str] = Field(default=None, max_length=5)
|
||||
timezone: Optional[str] = Field(default=None, max_length=64)
|
||||
allowed_platforms: Optional[list[str]] = Field(default=None, max_length=16)
|
||||
pause_on_battery: Optional[bool] = None
|
||||
public_display: Optional[bool] = None
|
||||
|
||||
@field_validator("allowed_platforms", mode="before")
|
||||
@classmethod
|
||||
def split_platforms(cls, value):
|
||||
if isinstance(value, str):
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
return value
|
||||
|
||||
|
||||
class NodeFetchForm(BaseModel):
|
||||
url: str = Field(max_length=2048)
|
||||
cache: bool = True
|
||||
timeout: int = Field(default=15, ge=1, le=60)
|
||||
|
||||
|
||||
NODE_SUSPEND_MIN_MINUTES = 1
|
||||
NODE_SUSPEND_MAX_MINUTES = 10080
|
||||
NODE_SETTING_FORM_MAX = 1_000_000
|
||||
|
||||
|
||||
class AdminNodeKillswitchForm(BaseModel):
|
||||
enabled: Optional[bool] = None
|
||||
|
||||
|
||||
class AdminNodeSettingsForm(BaseModel):
|
||||
nodes_max_total: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
|
||||
nodes_daily_quota_user: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
|
||||
nodes_daily_quota_guest: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
|
||||
nodes_cache_ttl_seconds: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
|
||||
nodes_social_via_nodes: Optional[Literal["0", "1"]] = None
|
||||
nodes_daily_social_user: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
|
||||
|
||||
|
||||
class AdminNodeSuspendForm(BaseModel):
|
||||
minutes: int = Field(ge=NODE_SUSPEND_MIN_MINUTES, le=NODE_SUSPEND_MAX_MINUTES)
|
||||
reason: str = Field(default="", max_length=500)
|
||||
|
||||
@field_validator("reason")
|
||||
@classmethod
|
||||
def trimmed(cls, value):
|
||||
return value.strip()
|
||||
|
||||
|
||||
class NodeDeviceForm(BaseModel):
|
||||
label: str = Field(default="", max_length=60)
|
||||
kind: str = Field(default="native", max_length=20)
|
||||
|
||||
@@ -53,7 +53,8 @@ Prefixes are wired in `main.py`:
|
||||
| (none) | push.py - push + PWA: `GET /push.json` (VAPID public key + the providers accepting registrations; `apns` includes `environment` when active), `POST /push.json` (register with any active provider; a body without `provider` is a `webpush` body; an APNs body is `token` plus optional `client_id` for device-stable upsert; created/revived rows are probed and the JSON may include `delivered`/`error`; an unknown, disabled or unconfigured provider is a 400), `DELETE /push.json` (unregister exactly the one registration named by `endpoint`/`token`/`client_id`, same identity priority as registration; idempotent, always 200 `{unregistered}`; wired into `PushManager.js`'s logout-link interceptor so a webpush subscription is dropped before the browser navigates to `/auth/logout`), `GET /service-worker.js`, `GET /manifest.json`. Provider protocol and delivery live in `devplacepy/push/` - see `devplacepy/push/CLAUDE.md` |
|
||||
| (none) | docs.py (docs/ package) - the documentation site (prose pages + API reference). See `routers/docs/CLAUDE.md` for the deep detail on this tree (`DOCS_PAGES`, audience tiers, prose rendering pipeline) |
|
||||
| `/skills` | skills.py - per-user Agent Skills: `GET ""` (list), `GET /{name}` (detail), `POST /install` (fetch a `SKILL.md` from a URL, update in place), `POST /{name}/enabled`, `POST /{name}/delete` (owner, or an administrator naming `owner`). Private to the owner, `noindex`. See `devplacepy/services/skills/CLAUDE.md` |
|
||||
| `/nodes` | nodes/ package - community node network. `spec.py` (`GET /nodes/spec.json`, public, `SPEC.describe()`), `devices.py` (`GET/POST /nodes/devices`, `POST /nodes/devices/{uid}/rotate|delete`, `require_user_api`, secret returned once, `no-store`), `ticket.py` (`POST /nodes/ticket`, authenticated by `Authorization: Bearer dpn_...`, never a session), `ws.py` (`WS /nodes/ws?ticket=`, lock-owner gated, `4013` retry). Needs its own nginx upgrade block. See `devplacepy/services/nodes/CLAUDE.md` |
|
||||
| `/nodes` | nodes/ package - community node network. `spec.py` (`GET /nodes/spec.json`, public, `SPEC.describe()`), `devices.py` (`GET/POST /nodes/devices`, `POST /nodes/devices/{uid}/rotate|delete`, `require_user_api`, secret returned once, `no-store`), `ticket.py` (`POST /nodes/ticket`, authenticated by `Authorization: Bearer dpn_...`, never a session), `ws.py` (`WS /nodes/ws?ticket=`, lock-owner gated, `4013` retry). Needs its own nginx upgrade block. The administrator side is `admin/nodes.py` (`GET /admin/nodes`, `POST /admin/nodes/{killswitch,settings,devices/{uid}/suspend,devices/{uid}/unsuspend,flags/{uid}/resolve}`). See `devplacepy/services/nodes/CLAUDE.md` |
|
||||
| `/sync` | sync/ package - community sync. `spec.py` (`GET /sync/spec.json`, public), `users.py`, `posts.py`, `comments.py` (`POST /sync/{kind}`, administrator API key, batch of up to 50 records, per-record results, `dry_run`). Shared auth, kill switch (`sync_enabled`), origin allowlist (`sync_origins`) and audit in `_shared.py`. See `devplacepy/services/sync/CLAUDE.md` |
|
||||
| `/pubsub` | pubsub.py - **database-free** publish/subscribe bus. `WS /pubsub/ws` (subscribe/unsubscribe/publish frames, `foo.*` wildcards), `POST /pubsub/publish` + `GET /pubsub/topics` (admin/internal). Lock-owner-gated WS (`4013` retry) so subscribers converge on one worker; topic authz in `services/pubsub/policy.py` (`user.{uid}.*` private, `public.*` shared, admin/internal anywhere, guests opt-in). In-process `services.pubsub.publish(topic, data)` for backends; frontend `app.pubsub`. See `devplacepy/services/pubsub/CLAUDE.md` |
|
||||
| (none) | seo.py - `/robots.txt`, `/sitemap.xml` |
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from devplacepy.routers.admin import (
|
||||
auditlog,
|
||||
backups,
|
||||
bots,
|
||||
communities,
|
||||
containers,
|
||||
workspaces,
|
||||
devii_tasks,
|
||||
@@ -16,6 +17,7 @@ from devplacepy.routers.admin import (
|
||||
media,
|
||||
moderation,
|
||||
news,
|
||||
nodes,
|
||||
notifications,
|
||||
services,
|
||||
settings,
|
||||
@@ -40,6 +42,8 @@ router.include_router(issues.router)
|
||||
router.include_router(auditlog.router)
|
||||
router.include_router(backups.router)
|
||||
router.include_router(bots.router)
|
||||
router.include_router(communities.router)
|
||||
router.include_router(nodes.router)
|
||||
router.include_router(gateway_configs.router)
|
||||
router.include_router(devii_tasks.router)
|
||||
router.include_router(game.router)
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Form, Request
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from devplacepy.database import get_setting
|
||||
from devplacepy.database.sync import count_links, identity_map
|
||||
from devplacepy.responses import action_result, json_error, respond
|
||||
from devplacepy.schemas import AdminCommunitiesOut, AdminCommunityOut
|
||||
from devplacepy.models import CommunityAddForm, CommunityRunForm, CommunityUpdateForm
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema
|
||||
from devplacepy.services.audit import record as audit
|
||||
from devplacepy.services.community.discoverer import slug_for_query
|
||||
from devplacepy.services.community.files import files
|
||||
from devplacepy.services.community.runner import TRIGGER_ADMIN, CommunityRunner
|
||||
from devplacepy.services.community.service import DRY_RUN_KEY
|
||||
from devplacepy.services.community.spec import STATUS_LEARNING
|
||||
from devplacepy.services.community.store import iso, now, store
|
||||
from devplacepy.services.sync.gate import sync_enabled
|
||||
from devplacepy.services.sync.options import SyncOptions
|
||||
from devplacepy.utils import not_found, require_admin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
background_jobs: set[asyncio.Task] = set()
|
||||
LIST_URL = "/admin/communities"
|
||||
|
||||
|
||||
def spawn(coroutine) -> None:
|
||||
task = asyncio.create_task(coroutine)
|
||||
background_jobs.add(task)
|
||||
task.add_done_callback(background_jobs.discard)
|
||||
|
||||
|
||||
def service_flags() -> dict:
|
||||
return {
|
||||
"service_enabled": (get_setting("service_community_enabled", "0") or "0") == "1",
|
||||
"service_dry_run": (get_setting(DRY_RUN_KEY, "1") or "1") == "1",
|
||||
}
|
||||
|
||||
|
||||
def public(row: dict) -> dict:
|
||||
view = {key: value for key, value in row.items() if key != "login_password"}
|
||||
view["has_login"] = bool(row.get("login_password"))
|
||||
return view
|
||||
|
||||
|
||||
def require_community(slug: str) -> dict:
|
||||
row = store.get(slug)
|
||||
if row is None:
|
||||
raise not_found("Unknown community")
|
||||
return row
|
||||
|
||||
|
||||
def credentials_of(data) -> dict:
|
||||
return {"username": data.login_username.strip(), "email": data.login_email.strip(), "password": data.login_password}
|
||||
|
||||
|
||||
def seo(request: Request, title: str, path: str, crumb: str):
|
||||
base = site_url(request)
|
||||
return base_seo_context(
|
||||
request,
|
||||
title=f"{title} - Admin",
|
||||
description="External communities synchronised into this instance.",
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Admin", "url": "/admin"},
|
||||
{"name": crumb, "url": path},
|
||||
],
|
||||
schemas=[website_schema(base)],
|
||||
robots="noindex,nofollow",
|
||||
)
|
||||
|
||||
|
||||
def feed_summary(feed: dict) -> dict:
|
||||
return {
|
||||
"id": feed["id"],
|
||||
"type": feed["type"],
|
||||
"url": feed["url"],
|
||||
"list_path": feed.get("list_path"),
|
||||
"item_selector": feed.get("item_selector"),
|
||||
"post_fields": feed["post"]["fields"],
|
||||
"author_fields": (feed.get("author") or {}).get("fields", {}),
|
||||
"comments": (feed.get("comments") or {}).get("mode", "none"),
|
||||
"comments_reason": (feed.get("comments") or {}).get("reason", ""),
|
||||
"pagination": (feed.get("pagination") or {}).get("style", "none"),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/communities", response_class=HTMLResponse)
|
||||
async def admin_communities(request: Request):
|
||||
admin = require_admin(request)
|
||||
return respond(
|
||||
request,
|
||||
"admin_communities.html",
|
||||
{
|
||||
**seo(request, "Communities", LIST_URL, "Communities"),
|
||||
"request": request,
|
||||
"user": admin,
|
||||
"items": [public(row) for row in store.list()],
|
||||
"sync_enabled": sync_enabled(),
|
||||
**service_flags(),
|
||||
"admin_section": "communities",
|
||||
},
|
||||
model=AdminCommunitiesOut,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/communities")
|
||||
async def admin_community_add(request: Request, data: Annotated[CommunityAddForm, Form()]):
|
||||
admin = require_admin(request)
|
||||
slug = slug_for_query(data.query)
|
||||
if not slug:
|
||||
return json_error(400, "Enter a community name or address")
|
||||
existing = store.get(slug)
|
||||
if existing is None:
|
||||
secrets = credentials_of(data)
|
||||
existing = store.create(
|
||||
slug,
|
||||
data.query,
|
||||
admin["uid"],
|
||||
name=data.query,
|
||||
status=STATUS_LEARNING,
|
||||
claimed_at=iso(now()),
|
||||
login_username=secrets["username"],
|
||||
login_email=secrets["email"],
|
||||
login_password=secrets["password"],
|
||||
)
|
||||
spawn(CommunityRunner().discover(data.query, admin["uid"], relearn_slug=slug))
|
||||
audit.record(
|
||||
request,
|
||||
"admin.community.add",
|
||||
user=admin,
|
||||
target_type="community",
|
||||
target_uid=existing["uid"],
|
||||
target_label=slug,
|
||||
metadata={"query": data.query},
|
||||
summary=f"{admin['username']} added community {data.query!r}",
|
||||
)
|
||||
return action_result(request, f"{LIST_URL}/{existing['slug']}")
|
||||
|
||||
|
||||
@router.get("/communities/{slug}", response_class=HTMLResponse)
|
||||
async def admin_community(request: Request, slug: str):
|
||||
admin = require_admin(request)
|
||||
row = require_community(slug)
|
||||
source = files.read(row["slug"])
|
||||
return respond(
|
||||
request,
|
||||
"admin_community.html",
|
||||
{
|
||||
**seo(request, row["name"] or slug, f"{LIST_URL}/{slug}", "Communities"),
|
||||
"request": request,
|
||||
"user": admin,
|
||||
"community": public(row),
|
||||
"rules": row.get("rules") or "",
|
||||
"favicon": row.get("favicon") or "",
|
||||
"totals": row.get("totals") or {},
|
||||
"feeds": [feed_summary(feed) for feed in (row.get("schema") or {}).get("feeds", [])],
|
||||
"runs": store.runs(row["uid"]),
|
||||
"collectors_path": str(files.collectors_path(row["slug"])) if source else "",
|
||||
"collectors_lines": len(source.splitlines()),
|
||||
"busy": store.busy(row["uid"]),
|
||||
"sync_enabled": sync_enabled(),
|
||||
"admin_section": "communities",
|
||||
},
|
||||
model=AdminCommunityOut,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/communities/{slug}/run")
|
||||
async def admin_community_run(request: Request, slug: str, data: Annotated[CommunityRunForm, Form()]):
|
||||
admin = require_admin(request)
|
||||
row = require_community(slug)
|
||||
if store.busy(row["uid"]):
|
||||
return json_error(409, "A run is already in progress for this community")
|
||||
spawn(CommunityRunner().run(row, dry_run=data.dry_run, trigger=TRIGGER_ADMIN))
|
||||
audit.record(
|
||||
request,
|
||||
"admin.community.run",
|
||||
user=admin,
|
||||
target_type="community",
|
||||
target_uid=row["uid"],
|
||||
target_label=slug,
|
||||
metadata={"dry_run": data.dry_run},
|
||||
summary=f"{admin['username']} started a {'dry run' if data.dry_run else 'run'} of community {slug}",
|
||||
)
|
||||
return action_result(request, f"{LIST_URL}/{slug}")
|
||||
|
||||
|
||||
@router.post("/communities/{slug}/relearn")
|
||||
async def admin_community_relearn(request: Request, slug: str):
|
||||
admin = require_admin(request)
|
||||
row = require_community(slug)
|
||||
if store.busy(row["uid"]):
|
||||
return json_error(409, "A run is already in progress for this community")
|
||||
spawn(CommunityRunner().discover(row["query"] or row["url"], admin["uid"], relearn_slug=slug))
|
||||
audit.record(
|
||||
request,
|
||||
"admin.community.relearn",
|
||||
user=admin,
|
||||
target_type="community",
|
||||
target_uid=row["uid"],
|
||||
target_label=slug,
|
||||
summary=f"{admin['username']} started relearning community {slug}",
|
||||
)
|
||||
return action_result(request, f"{LIST_URL}/{slug}")
|
||||
|
||||
|
||||
@router.post("/communities/{slug}/update")
|
||||
async def admin_community_update(request: Request, slug: str, data: Annotated[CommunityUpdateForm, Form()]):
|
||||
admin = require_admin(request)
|
||||
row = require_community(slug)
|
||||
changes = {"interval_seconds": data.interval_seconds, "enabled": int(data.enabled)}
|
||||
if data.clear_login:
|
||||
changes.update(login_username="", login_email="", login_password="")
|
||||
else:
|
||||
for column, value in (("login_username", data.login_username.strip()), ("login_email", data.login_email.strip()), ("login_password", data.login_password)):
|
||||
if value:
|
||||
changes[column] = value
|
||||
submitted = {name: getattr(data, name) for name in SyncOptions.__dataclass_fields__ if getattr(data, name) is not None}
|
||||
if submitted:
|
||||
try:
|
||||
options = SyncOptions.from_row(row).merged(**submitted)
|
||||
except ValueError as exc:
|
||||
return json_error(400, str(exc))
|
||||
if options.alias and not SyncOptions.from_row(row).alias and not identity_map(row["origin"]) and count_links(row["origin"]):
|
||||
return json_error(400, "This community already has records stored under their original names; aliasing cannot be switched on afterwards")
|
||||
changes["sync_options"] = options.to_dict()
|
||||
store.update(row["uid"], **changes)
|
||||
audit.record(
|
||||
request,
|
||||
"admin.community.update",
|
||||
user=admin,
|
||||
target_type="community",
|
||||
target_uid=row["uid"],
|
||||
target_label=slug,
|
||||
metadata={"interval_seconds": data.interval_seconds, "enabled": data.enabled, "sync_options": changes.get("sync_options"), "login_changed": data.clear_login or bool(data.login_password or data.login_username or data.login_email)},
|
||||
summary=f"{admin['username']} updated community {slug}",
|
||||
)
|
||||
return action_result(request, f"{LIST_URL}/{slug}")
|
||||
|
||||
|
||||
@router.post("/communities/{slug}/delete")
|
||||
async def admin_community_delete(request: Request, slug: str):
|
||||
admin = require_admin(request)
|
||||
row = require_community(slug)
|
||||
store.delete(row["uid"], admin["uid"])
|
||||
audit.record(
|
||||
request,
|
||||
"admin.community.delete",
|
||||
user=admin,
|
||||
target_type="community",
|
||||
target_uid=row["uid"],
|
||||
target_label=slug,
|
||||
summary=f"{admin['username']} deleted community {slug}",
|
||||
)
|
||||
return action_result(request, LIST_URL)
|
||||
@@ -0,0 +1,376 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from devplacepy.database import get_int_setting, get_setting, get_table, get_users_by_uids, set_setting
|
||||
from devplacepy.database.nodes import (
|
||||
ADMIN_LIST_LIMIT,
|
||||
DEVICE_BREAKDOWN_COLUMNS,
|
||||
LIMITS,
|
||||
add_event,
|
||||
count_open_requests,
|
||||
device_breakdown,
|
||||
effective_status,
|
||||
get_device,
|
||||
list_flags,
|
||||
recent_devices,
|
||||
request_status_counts,
|
||||
resolve_flag,
|
||||
suspend_device,
|
||||
unsuspend_device,
|
||||
usage_day_totals,
|
||||
)
|
||||
from devplacepy.dependencies import json_or_form
|
||||
from devplacepy.models import AdminNodeKillswitchForm, AdminNodeSettingsForm, AdminNodeSuspendForm
|
||||
from devplacepy.responses import action_result, json_error, respond
|
||||
from devplacepy.routers.admin._shared import deny_senior, is_senior_admin
|
||||
from devplacepy.routers.nodes.social import SOCIAL_DAILY_DEFAULT
|
||||
from devplacepy.schemas import (
|
||||
AdminNodeDeviceOut,
|
||||
AdminNodeFlagOut,
|
||||
AdminNodePoolOut,
|
||||
AdminNodeQueueOut,
|
||||
AdminNodeSettingOut,
|
||||
AdminNodesOut,
|
||||
)
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema
|
||||
from devplacepy.services.audit import record as audit
|
||||
from devplacepy.services.base import BaseService
|
||||
from devplacepy.utils import not_found, require_admin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
LIST_URL = "/admin/nodes"
|
||||
SERVICE_NAME = "nodes"
|
||||
KILLSWITCH_KEY = "nodes_enabled"
|
||||
QUOTA_USER_CEILING = 10_000
|
||||
QUOTA_GUEST_CEILING = 1_000
|
||||
QUOTA_SOCIAL_CEILING = 10_000
|
||||
POOL_STATS = (
|
||||
("connected", "connected"),
|
||||
("capacity", "capacity"),
|
||||
("accepted", "accepted"),
|
||||
("kicked", "kicked"),
|
||||
("jobs done", "jobs_done"),
|
||||
("jobs failed", "jobs_failed"),
|
||||
("open circuits", "open_circuits"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NodeSettingField:
|
||||
key: str
|
||||
label: str
|
||||
default: int
|
||||
ceiling: int | None
|
||||
kind: str = "int"
|
||||
|
||||
|
||||
SETTING_FIELDS: tuple[NodeSettingField, ...] = (
|
||||
NodeSettingField("nodes_max_total", "Maximum connected nodes", LIMITS.max_connected_nodes, LIMITS.max_connected_nodes),
|
||||
NodeSettingField("nodes_daily_quota_user", "Daily node requests per member", LIMITS.default_daily_requests_user, QUOTA_USER_CEILING),
|
||||
NodeSettingField("nodes_daily_quota_guest", "Daily node requests per guest", LIMITS.default_daily_requests_guest, QUOTA_GUEST_CEILING),
|
||||
NodeSettingField("nodes_cache_ttl_seconds", "Result cache lifetime (seconds)", LIMITS.cache_ttl_seconds, LIMITS.cache_ttl_seconds),
|
||||
NodeSettingField("nodes_daily_social_user", "Daily social searches per member", SOCIAL_DAILY_DEFAULT, QUOTA_SOCIAL_CEILING),
|
||||
NodeSettingField("nodes_social_via_nodes", "Route social search through nodes", 0, None, "flag"),
|
||||
)
|
||||
SETTING_FIELDS_BY_KEY = {field.key: field for field in SETTING_FIELDS}
|
||||
|
||||
|
||||
def kill_switch_on() -> bool:
|
||||
return (get_setting(KILLSWITCH_KEY, "0") or "0") == "1"
|
||||
|
||||
|
||||
def pool_view() -> AdminNodePoolOut:
|
||||
row = get_table("service_state").find_one(name=SERVICE_NAME)
|
||||
if not row or not row.get("heartbeat"):
|
||||
return AdminNodePoolOut()
|
||||
try:
|
||||
beat = datetime.fromisoformat(row["heartbeat"])
|
||||
except ValueError:
|
||||
return AdminNodePoolOut()
|
||||
if beat.tzinfo is None:
|
||||
beat = beat.replace(tzinfo=timezone.utc)
|
||||
if (datetime.now(timezone.utc) - beat).total_seconds() > BaseService.STALE_SECONDS * 2:
|
||||
return AdminNodePoolOut(heartbeat=row["heartbeat"])
|
||||
try:
|
||||
stats = json.loads(row.get("metrics") or "{}").get("stats") or []
|
||||
except ValueError:
|
||||
stats = []
|
||||
by_label = {item.get("label"): item.get("value") for item in stats}
|
||||
values = {field: int(by_label.get(label) or 0) for label, field in POOL_STATS}
|
||||
return AdminNodePoolOut(online=True, heartbeat=row["heartbeat"], **values)
|
||||
|
||||
|
||||
def queue_view() -> AdminNodeQueueOut:
|
||||
return AdminNodeQueueOut(
|
||||
depth=count_open_requests(),
|
||||
max_depth=LIMITS.max_queue_depth,
|
||||
**request_status_counts(),
|
||||
)
|
||||
|
||||
|
||||
def setting_value(field: NodeSettingField) -> str:
|
||||
if field.kind == "flag":
|
||||
return "1" if (get_setting(field.key, str(field.default)) or "0") == "1" else "0"
|
||||
return str(get_int_setting(field.key, field.default))
|
||||
|
||||
|
||||
def settings_view() -> list[AdminNodeSettingOut]:
|
||||
return [
|
||||
AdminNodeSettingOut(
|
||||
key=field.key,
|
||||
label=field.label,
|
||||
kind=field.kind,
|
||||
value=setting_value(field),
|
||||
default=str(field.default),
|
||||
ceiling=field.ceiling,
|
||||
)
|
||||
for field in SETTING_FIELDS
|
||||
]
|
||||
|
||||
|
||||
def device_views(rows: list[dict], owners: dict) -> list[AdminNodeDeviceOut]:
|
||||
return [
|
||||
AdminNodeDeviceOut(
|
||||
uid=row["uid"],
|
||||
label=row.get("label") or "",
|
||||
kind=row.get("kind") or "",
|
||||
platform=row.get("platform") or "",
|
||||
client_version=row.get("client_version") or "",
|
||||
status=effective_status(row),
|
||||
suspended_until=row.get("suspended_until") or None,
|
||||
suspend_reason=row.get("suspend_reason") or "",
|
||||
country=row.get("country") or "",
|
||||
asn_class=row.get("asn_class") or "unknown",
|
||||
last_seen_at=row.get("last_seen_at") or None,
|
||||
created_at=row.get("created_at"),
|
||||
owner_uid=row.get("user_uid") or "",
|
||||
owner_username=(owners.get(row.get("user_uid")) or {}).get("username") or "",
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def flag_views(rows: list[dict], owners: dict, devices: dict) -> list[AdminNodeFlagOut]:
|
||||
return [
|
||||
AdminNodeFlagOut(
|
||||
uid=row["uid"],
|
||||
kind=row.get("kind") or "",
|
||||
detail=row.get("detail") or "",
|
||||
status=row.get("status") or "open",
|
||||
device_uid=row.get("device_uid") or "",
|
||||
device_label=(devices.get(row.get("device_uid")) or {}).get("label") or "",
|
||||
user_uid=row.get("user_uid") or "",
|
||||
owner_username=(owners.get(row.get("user_uid")) or {}).get("username") or "",
|
||||
created_at=row.get("created_at"),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def overview() -> dict:
|
||||
devices = recent_devices(ADMIN_LIST_LIMIT)
|
||||
flags = list_flags()[:ADMIN_LIST_LIMIT]
|
||||
device_by_uid = {row["uid"]: row for row in devices}
|
||||
missing = [row["device_uid"] for row in flags if row.get("device_uid") and row["device_uid"] not in device_by_uid]
|
||||
for uid in missing:
|
||||
found = get_device(uid)
|
||||
if found:
|
||||
device_by_uid[uid] = found
|
||||
owner_uids = [row["user_uid"] for row in devices + flags if row.get("user_uid")]
|
||||
owners = get_users_by_uids(list(dict.fromkeys(owner_uids)))
|
||||
breakdowns = {f"devices_by_{column}": device_breakdown(column) for column in DEVICE_BREAKDOWN_COLUMNS}
|
||||
return {
|
||||
"enabled": kill_switch_on(),
|
||||
"pool": pool_view(),
|
||||
"queue": queue_view(),
|
||||
**breakdowns,
|
||||
"usage_today": usage_day_totals(),
|
||||
"flags": flag_views(flags, owners, device_by_uid),
|
||||
"devices": device_views(devices, owners),
|
||||
"settings": settings_view(),
|
||||
"limits": LIMITS.describe(),
|
||||
"admin_section": "nodes",
|
||||
}
|
||||
|
||||
|
||||
def seo(request: Request):
|
||||
base = site_url(request)
|
||||
return base_seo_context(
|
||||
request,
|
||||
title="Community nodes - Admin",
|
||||
description="Community donor node network: pool, queue, devices and flags.",
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Admin", "url": "/admin"},
|
||||
{"name": "Community nodes", "url": LIST_URL},
|
||||
],
|
||||
schemas=[website_schema(base)],
|
||||
robots="noindex,nofollow",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/nodes", response_class=HTMLResponse)
|
||||
async def admin_nodes(request: Request):
|
||||
admin = require_admin(request)
|
||||
return respond(
|
||||
request,
|
||||
"admin_nodes.html",
|
||||
{**seo(request), "request": request, "user": admin, **overview()},
|
||||
model=AdminNodesOut,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/nodes/killswitch")
|
||||
async def admin_nodes_killswitch(
|
||||
request: Request,
|
||||
data: Annotated[AdminNodeKillswitchForm, Depends(json_or_form(AdminNodeKillswitchForm))],
|
||||
):
|
||||
admin = require_admin(request)
|
||||
before = kill_switch_on()
|
||||
after = (not before) if data.enabled is None else data.enabled
|
||||
set_setting(KILLSWITCH_KEY, "1" if after else "0")
|
||||
audit.record(
|
||||
request,
|
||||
"node.killswitch",
|
||||
user=admin,
|
||||
target_type="setting",
|
||||
target_uid=KILLSWITCH_KEY,
|
||||
target_label=KILLSWITCH_KEY,
|
||||
old_value="1" if before else "0",
|
||||
new_value="1" if after else "0",
|
||||
summary=f"admin {admin['username']} switched community donation {'on' if after else 'off'}",
|
||||
links=[audit.setting(KILLSWITCH_KEY)],
|
||||
)
|
||||
return action_result(request, LIST_URL, data={"enabled": after})
|
||||
|
||||
|
||||
@router.post("/nodes/settings")
|
||||
async def admin_nodes_settings(
|
||||
request: Request,
|
||||
data: Annotated[AdminNodeSettingsForm, Depends(json_or_form(AdminNodeSettingsForm))],
|
||||
):
|
||||
admin = require_admin(request)
|
||||
changes: dict[str, dict] = {}
|
||||
clamped: list[str] = []
|
||||
for key, submitted in data.model_dump().items():
|
||||
if submitted is None:
|
||||
continue
|
||||
field = SETTING_FIELDS_BY_KEY[key]
|
||||
value = str(submitted)
|
||||
if field.ceiling is not None and int(submitted) > field.ceiling:
|
||||
value = str(field.ceiling)
|
||||
clamped.append(key)
|
||||
before = setting_value(field)
|
||||
if before == value:
|
||||
continue
|
||||
set_setting(key, value)
|
||||
changes[key] = {"old": before, "new": value}
|
||||
audit.record(
|
||||
request,
|
||||
"node.admin.settings",
|
||||
user=admin,
|
||||
target_type="setting",
|
||||
target_uid="nodes",
|
||||
target_label="nodes",
|
||||
metadata={"changes": changes, "clamped": clamped},
|
||||
summary=f"admin {admin['username']} updated community node settings ({len(changes)} changed)",
|
||||
links=[audit.setting(key) for key in changes],
|
||||
)
|
||||
return action_result(request, LIST_URL, data={"changed": list(changes), "clamped": clamped})
|
||||
|
||||
|
||||
def require_device(device_uid: str) -> dict:
|
||||
device = get_device(device_uid)
|
||||
if device is None:
|
||||
raise not_found("Unknown device")
|
||||
return device
|
||||
|
||||
|
||||
def owner_of(device: dict) -> dict | None:
|
||||
return get_table("users").find_one(uid=device.get("user_uid"))
|
||||
|
||||
|
||||
@router.post("/nodes/devices/{device_uid}/suspend")
|
||||
async def admin_nodes_suspend(
|
||||
request: Request,
|
||||
device_uid: str,
|
||||
data: Annotated[AdminNodeSuspendForm, Depends(json_or_form(AdminNodeSuspendForm))],
|
||||
):
|
||||
admin = require_admin(request)
|
||||
device = require_device(device_uid)
|
||||
owner = owner_of(device)
|
||||
if is_senior_admin(admin, owner):
|
||||
return deny_senior(request, admin, owner["uid"], owner, "node.suspend", LIST_URL)
|
||||
if not suspend_device(device_uid, data.reason, data.minutes):
|
||||
return json_error(409, "This device can no longer be suspended")
|
||||
add_event("admin_suspend", device_uid=device_uid, user_uid=device.get("user_uid") or "", detail=data.reason)
|
||||
audit.record(
|
||||
request,
|
||||
"node.suspend",
|
||||
user=admin,
|
||||
target_type="node_device",
|
||||
target_uid=device_uid,
|
||||
target_label=device.get("label") or device_uid,
|
||||
metadata={"minutes": data.minutes, "reason": data.reason, "owner_uid": device.get("user_uid")},
|
||||
summary=f"admin {admin['username']} suspended node device {device.get('label') or device_uid} for {data.minutes} minutes",
|
||||
links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))],
|
||||
)
|
||||
return action_result(request, LIST_URL, data={"uid": device_uid, "minutes": data.minutes})
|
||||
|
||||
|
||||
@router.post("/nodes/devices/{device_uid}/unsuspend")
|
||||
async def admin_nodes_unsuspend(request: Request, device_uid: str):
|
||||
admin = require_admin(request)
|
||||
device = require_device(device_uid)
|
||||
owner = owner_of(device)
|
||||
if is_senior_admin(admin, owner):
|
||||
return deny_senior(request, admin, owner["uid"], owner, "node.unsuspend", LIST_URL)
|
||||
if not unsuspend_device(device_uid):
|
||||
return json_error(409, "This device is not suspended")
|
||||
add_event("admin_unsuspend", device_uid=device_uid, user_uid=device.get("user_uid") or "")
|
||||
audit.record(
|
||||
request,
|
||||
"node.unsuspend",
|
||||
user=admin,
|
||||
target_type="node_device",
|
||||
target_uid=device_uid,
|
||||
target_label=device.get("label") or device_uid,
|
||||
metadata={"owner_uid": device.get("user_uid")},
|
||||
summary=f"admin {admin['username']} lifted the suspension of node device {device.get('label') or device_uid}",
|
||||
links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))],
|
||||
)
|
||||
return action_result(request, LIST_URL, data={"uid": device_uid})
|
||||
|
||||
|
||||
@router.post("/nodes/flags/{flag_uid}/resolve")
|
||||
async def admin_nodes_flag_resolve(request: Request, flag_uid: str):
|
||||
admin = require_admin(request)
|
||||
flag = get_table("node_flags").find_one(uid=flag_uid, deleted_at=None)
|
||||
if flag is None:
|
||||
raise not_found("Unknown flag")
|
||||
owner = get_table("users").find_one(uid=flag.get("user_uid")) if flag.get("user_uid") else None
|
||||
if is_senior_admin(admin, owner):
|
||||
return deny_senior(request, admin, owner["uid"], owner, "node.flag.resolve", LIST_URL)
|
||||
if not resolve_flag(flag_uid, admin["uid"]):
|
||||
return json_error(409, "This flag is already resolved")
|
||||
audit.record(
|
||||
request,
|
||||
"node.flag.resolve",
|
||||
user=admin,
|
||||
target_type="node_flag",
|
||||
target_uid=flag_uid,
|
||||
target_label=flag.get("kind") or flag_uid,
|
||||
metadata={"device_uid": flag.get("device_uid"), "owner_uid": flag.get("user_uid")},
|
||||
summary=f"admin {admin['username']} resolved node flag {flag.get('kind') or flag_uid}",
|
||||
)
|
||||
return action_result(request, LIST_URL, data={"uid": flag_uid})
|
||||
@@ -16,7 +16,7 @@ from devplacepy.dependencies import json_or_form
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
CLEARABLE_SETTINGS = {"extra_head", "ios_app_url"}
|
||||
CLEARABLE_SETTINGS = {"extra_head", "ios_app_url", "sync_origins"}
|
||||
|
||||
@router.get("/settings", response_class=HTMLResponse)
|
||||
async def admin_settings(request: Request):
|
||||
|
||||
@@ -75,6 +75,12 @@ DOCS_PAGES = [
|
||||
"kind": "prose",
|
||||
"section": SECTION_GENERAL,
|
||||
},
|
||||
{
|
||||
"slug": "nodes",
|
||||
"title": "Donate your connection",
|
||||
"kind": "prose",
|
||||
"section": SECTION_GENERAL,
|
||||
},
|
||||
{
|
||||
"slug": "quizzes",
|
||||
"title": "Quizzes",
|
||||
|
||||
@@ -2,10 +2,13 @@
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from . import devices, spec, ticket, ws
|
||||
from . import clients, devices, me, social, spec, ticket, ws
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(spec.router)
|
||||
router.include_router(clients.router)
|
||||
router.include_router(me.router)
|
||||
router.include_router(devices.router)
|
||||
router.include_router(ticket.router)
|
||||
router.include_router(social.router)
|
||||
router.include_router(ws.router)
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
|
||||
from devplacepy import config
|
||||
from devplacepy.responses import json_error
|
||||
from devplacepy.schemas.nodes import NodeArtifactOut, NodeClientVersionOut
|
||||
from devplacepy.services.nodes.spec import SPEC
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
MANIFEST_NAME = "manifest.json"
|
||||
ARTIFACT_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$")
|
||||
MAX_ARTIFACTS = 64
|
||||
MAX_ARTIFACT_BYTES = 200 * 1024 * 1024
|
||||
_digests: dict[tuple[str, int, int], str] = {}
|
||||
|
||||
|
||||
def _digest(path) -> str:
|
||||
stat = path.stat()
|
||||
key = (path.name, int(stat.st_mtime_ns), stat.st_size)
|
||||
found = _digests.get(key)
|
||||
if found is None:
|
||||
sha = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
sha.update(block)
|
||||
found = sha.hexdigest()
|
||||
if len(_digests) > MAX_ARTIFACTS * 4:
|
||||
_digests.clear()
|
||||
_digests[key] = found
|
||||
return found
|
||||
|
||||
|
||||
def _manifest_version() -> str:
|
||||
path = config.NODE_CLIENTS_DIR / MANIFEST_NAME
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return ""
|
||||
if isinstance(data, dict):
|
||||
return str(data.get("version") or "")[:40]
|
||||
if isinstance(data, list) and data and isinstance(data[0], dict):
|
||||
return str(data[0].get("version") or "")[:40]
|
||||
return ""
|
||||
|
||||
|
||||
def _artifacts() -> list[NodeArtifactOut]:
|
||||
root = config.NODE_CLIENTS_DIR
|
||||
if not root.is_dir():
|
||||
return []
|
||||
found = []
|
||||
for path in sorted(root.iterdir())[:MAX_ARTIFACTS]:
|
||||
if (
|
||||
path.name == MANIFEST_NAME
|
||||
or not path.is_file()
|
||||
or not ARTIFACT_NAME.match(path.name)
|
||||
or path.stat().st_size > MAX_ARTIFACT_BYTES
|
||||
):
|
||||
continue
|
||||
found.append(
|
||||
NodeArtifactOut(name=path.name, size=path.stat().st_size, sha256=_digest(path))
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
@router.get("/client-version")
|
||||
async def client_version():
|
||||
payload = NodeClientVersionOut(
|
||||
version=_manifest_version(),
|
||||
protocol=SPEC.version,
|
||||
min_protocol=SPEC.min_supported,
|
||||
artifacts=_artifacts(),
|
||||
)
|
||||
return JSONResponse(payload.model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.get("/download/{artifact}")
|
||||
async def download_client(artifact: str):
|
||||
if not ARTIFACT_NAME.match(artifact) or artifact == MANIFEST_NAME:
|
||||
return json_error(404, "No such client")
|
||||
path = config.NODE_CLIENTS_DIR / artifact
|
||||
if not path.is_file() or path.is_symlink() or path.stat().st_size > MAX_ARTIFACT_BYTES:
|
||||
return json_error(404, "No such client")
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type="application/octet-stream",
|
||||
filename=artifact,
|
||||
headers={"X-Content-SHA256": _digest(path), "X-Content-Type-Options": "nosniff"},
|
||||
)
|
||||
@@ -0,0 +1,143 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from devplacepy.cache import TTLCache
|
||||
from devplacepy.database import get_users_by_uids
|
||||
from devplacepy.database.moderation import consent_granted
|
||||
from devplacepy.database.nodes import (
|
||||
LIMITS,
|
||||
PLATFORMS,
|
||||
get_node_settings,
|
||||
list_devices,
|
||||
public_stats,
|
||||
save_node_settings,
|
||||
top_donors,
|
||||
)
|
||||
from devplacepy.dependencies import json_or_form
|
||||
from devplacepy.models import NodeSettingsForm
|
||||
from devplacepy.responses import json_error
|
||||
from devplacepy.routers.nodes._shared import GATE, NO_STORE, device_view
|
||||
from devplacepy.schemas.nodes import (
|
||||
NodeDonorOut,
|
||||
NodeLeaderboardOut,
|
||||
NodeMeOut,
|
||||
NodeStatsOut,
|
||||
)
|
||||
from devplacepy.services.audit import record as audit
|
||||
from devplacepy.services.nodes.ledger import UsageLedger
|
||||
from devplacepy.utils import require_user_api
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
AGENT_HEADER = "x-devii-agent"
|
||||
LEADERBOARD_LIMIT = 10
|
||||
STATS_TTL_SECONDS = int(os.environ.get("DEVPLACE_NODES_STATS_TTL", "30"))
|
||||
_public_cache = TTLCache(STATS_TTL_SECONDS, max_size=4)
|
||||
|
||||
|
||||
def _me(user: dict) -> NodeMeOut:
|
||||
verdict = GATE.allows(user["uid"])
|
||||
return NodeMeOut(
|
||||
consent=consent_granted("user", user["uid"], "bandwidth_sharing"),
|
||||
can_donate=verdict.allowed,
|
||||
reason=verdict.reason,
|
||||
message=verdict.message,
|
||||
settings=get_node_settings(user["uid"]),
|
||||
platforms=list(PLATFORMS),
|
||||
devices=[device_view(row) for row in list_devices(user["uid"])],
|
||||
max_devices=LIMITS.max_devices_per_user,
|
||||
donated=UsageLedger().summary(user["uid"]).describe(),
|
||||
limits={
|
||||
"max_donor_mb_per_day": LIMITS.max_donor_mb_per_day,
|
||||
"max_concurrent_per_node": LIMITS.max_concurrent_per_node,
|
||||
"default_donor_mb_per_day": LIMITS.default_donor_mb_per_day,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
async def my_donation(request: Request):
|
||||
user = require_user_api(request)
|
||||
return JSONResponse(_me(user).model_dump(mode="json"), headers=NO_STORE)
|
||||
|
||||
|
||||
@router.post("/settings")
|
||||
async def save_my_settings(
|
||||
request: Request,
|
||||
data: Annotated[NodeSettingsForm, Depends(json_or_form(NodeSettingsForm))],
|
||||
):
|
||||
user = require_user_api(request)
|
||||
values = data.model_dump(exclude_none=True)
|
||||
if values.get("enabled"):
|
||||
if request.headers.get(AGENT_HEADER) == "1":
|
||||
audit.record(
|
||||
request,
|
||||
"security.authz.denied",
|
||||
user=user,
|
||||
result="denied",
|
||||
target_type="user",
|
||||
target_uid=user["uid"],
|
||||
metadata={"reason": "agent cannot enable donation"},
|
||||
summary="agent denied enabling bandwidth donation",
|
||||
)
|
||||
return json_error(
|
||||
403, "Donation can only be switched on by the member in their own profile"
|
||||
)
|
||||
if not consent_granted("user", user["uid"], "bandwidth_sharing"):
|
||||
return json_error(
|
||||
403,
|
||||
"Give the bandwidth sharing consent before switching donation on",
|
||||
reason="no_consent",
|
||||
)
|
||||
before = get_node_settings(user["uid"])
|
||||
saved = save_node_settings(user["uid"], values)
|
||||
audit.record(
|
||||
request,
|
||||
"node.settings.update",
|
||||
user=user,
|
||||
target_type="user",
|
||||
target_uid=user["uid"],
|
||||
target_label=user["username"],
|
||||
old_value=str(before.get("enabled")),
|
||||
new_value=str(saved.get("enabled")),
|
||||
metadata={key: saved[key] for key in values if key in saved},
|
||||
summary="updated bandwidth donation settings",
|
||||
)
|
||||
return JSONResponse(_me(user).model_dump(mode="json"), headers=NO_STORE)
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
async def donation_stats():
|
||||
cached = _public_cache.get("stats")
|
||||
if cached is None:
|
||||
cached = NodeStatsOut(**public_stats()).model_dump(mode="json")
|
||||
_public_cache.set("stats", cached)
|
||||
return JSONResponse(cached)
|
||||
|
||||
|
||||
@router.get("/leaderboard")
|
||||
async def donation_leaderboard():
|
||||
cached = _public_cache.get("leaderboard")
|
||||
if cached is None:
|
||||
rows = top_donors(LEADERBOARD_LIMIT)
|
||||
users = get_users_by_uids([row["user_uid"] for row in rows])
|
||||
cached = NodeLeaderboardOut(
|
||||
donors=[
|
||||
NodeDonorOut(
|
||||
username=users[row["user_uid"]]["username"],
|
||||
requests=int(row["requests_ok"] or 0),
|
||||
bytes=int(row["bytes_total"] or 0),
|
||||
)
|
||||
for row in rows
|
||||
if row["user_uid"] in users
|
||||
]
|
||||
).model_dump(mode="json")
|
||||
_public_cache.set("leaderboard", cached)
|
||||
return JSONResponse(cached)
|
||||
@@ -0,0 +1,98 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from devplacepy.database import get_int_setting
|
||||
from devplacepy.database.nodes import reserve_caller_request
|
||||
from devplacepy.dependencies import json_or_form
|
||||
from devplacepy.models import NodeFetchForm
|
||||
from devplacepy.responses import json_error
|
||||
from devplacepy.routers.nodes._shared import NO_STORE
|
||||
from devplacepy.schemas.nodes import NodeFetchOut, SocialSearchOut
|
||||
from devplacepy.services.nodes.fabric import (
|
||||
USER_KIND,
|
||||
FabricQuotaExceeded,
|
||||
FabricRefused,
|
||||
FabricUnavailable,
|
||||
shared_fabric,
|
||||
)
|
||||
from devplacepy.services.nodes.social import DEFAULT_LIMIT, SocialSearch
|
||||
from devplacepy.utils import require_user_api
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
SOCIAL_KIND = "social"
|
||||
SOCIAL_DAILY_DEFAULT = 300
|
||||
FETCH_TEXT_MAX = 1024 * 1024
|
||||
|
||||
|
||||
def social_quota_ok(user_uid: str) -> bool:
|
||||
cap = get_int_setting("nodes_daily_social_user", SOCIAL_DAILY_DEFAULT)
|
||||
return reserve_caller_request(SOCIAL_KIND, user_uid, cap)
|
||||
|
||||
|
||||
@router.get("/social/search")
|
||||
async def social_search(
|
||||
request: Request,
|
||||
q: str = "",
|
||||
platforms: str = "",
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
):
|
||||
user = require_user_api(request)
|
||||
query = " ".join(q.split())
|
||||
if not query:
|
||||
return json_error(400, "A search query is required")
|
||||
wanted = [item.strip().lower() for item in platforms.split(",") if item.strip()]
|
||||
if not social_quota_ok(user["uid"]):
|
||||
return json_error(429, "Daily social search quota reached")
|
||||
search = SocialSearch()
|
||||
outcome = await search.search(
|
||||
query,
|
||||
caller_kind=USER_KIND,
|
||||
caller_id=user["uid"],
|
||||
platforms=wanted,
|
||||
limit=limit,
|
||||
)
|
||||
payload = SocialSearchOut(
|
||||
query=query[:200],
|
||||
platforms=search.platforms_for(wanted),
|
||||
**outcome.as_dict(),
|
||||
)
|
||||
return JSONResponse(payload.model_dump(mode="json"), headers=NO_STORE)
|
||||
|
||||
|
||||
@router.post("/internal/fetch")
|
||||
async def internal_fetch(
|
||||
request: Request,
|
||||
data: Annotated[NodeFetchForm, Depends(json_or_form(NodeFetchForm))],
|
||||
):
|
||||
user = require_user_api(request)
|
||||
try:
|
||||
result = await shared_fabric().fetch(
|
||||
data.url,
|
||||
caller_kind=USER_KIND,
|
||||
caller_id=user["uid"],
|
||||
use_cache=data.cache,
|
||||
timeout=data.timeout,
|
||||
)
|
||||
except FabricRefused as exc:
|
||||
return json_error(403, exc.message, reason=exc.code)
|
||||
except FabricQuotaExceeded as exc:
|
||||
return json_error(429, exc.message, reason=exc.code)
|
||||
except FabricUnavailable as exc:
|
||||
return json_error(503, exc.message, reason=exc.code)
|
||||
payload = NodeFetchOut(
|
||||
url=result.url,
|
||||
status=result.status,
|
||||
via=result.via,
|
||||
truncated=result.truncated or len(result.body) > FETCH_TEXT_MAX,
|
||||
elapsed_ms=result.elapsed_ms,
|
||||
content_type=str(result.headers.get("content-type", ""))[:120],
|
||||
text=result.body[:FETCH_TEXT_MAX].decode("utf-8", errors="replace"),
|
||||
)
|
||||
return JSONResponse(payload.model_dump(mode="json"), headers=NO_STORE)
|
||||
@@ -67,6 +67,8 @@ async def node_ws(websocket: WebSocket, ticket: str = ""):
|
||||
await websocket.close(code=CLOSE_POLICY, reason=verdict.reason)
|
||||
return
|
||||
session = NodeSession(websocket, device)
|
||||
service.dispatcher.attach(session)
|
||||
service.ensure_dispatching()
|
||||
admitted, previous = registry.admit(session)
|
||||
if not admitted:
|
||||
await websocket.close(code=CLOSE_CAPACITY, reason="capacity")
|
||||
@@ -99,6 +101,7 @@ async def node_ws(websocket: WebSocket, ticket: str = ""):
|
||||
if ended == "heartbeat":
|
||||
counters.inc("evicted_heartbeat")
|
||||
registry.release(session)
|
||||
service.dispatcher.release_session(session)
|
||||
add_event(
|
||||
"disconnect",
|
||||
device_uid=device["uid"],
|
||||
|
||||
@@ -38,6 +38,7 @@ from devplacepy.database.awards import (
|
||||
get_user_awards,
|
||||
)
|
||||
from devplacepy.content import can_view_project, enrich_items
|
||||
from devplacepy.routers.nodes.me import _me as node_me_view
|
||||
from devplacepy.routers.profile.consent import consent_view
|
||||
from devplacepy.services.moderation.deletion import grace_hours
|
||||
from devplacepy.services.moderation.screening import (
|
||||
@@ -401,8 +402,18 @@ async def profile_page(
|
||||
}
|
||||
)
|
||||
|
||||
node_me = (
|
||||
node_me_view(current_user).model_dump(mode="json")
|
||||
if (tab == "donate" and is_owner)
|
||||
else {}
|
||||
)
|
||||
|
||||
base = site_url(request)
|
||||
robots = "noindex,follow" if posts_count < 2 else "index,follow"
|
||||
robots = (
|
||||
"noindex,follow"
|
||||
if posts_count < 2 or (tab == "donate" and is_owner)
|
||||
else "index,follow"
|
||||
)
|
||||
bio = profile_user.get("bio", "")
|
||||
desc = (
|
||||
bio
|
||||
@@ -473,6 +484,7 @@ async def profile_page(
|
||||
"cust_disable_pagetype": customization_prefs["disable_pagetype"],
|
||||
"notification_prefs": notification_prefs,
|
||||
"consents": consents,
|
||||
"node_me": node_me,
|
||||
**privacy_fields,
|
||||
"current_terms_version": get_setting("terms_version", "1") or "1",
|
||||
"deletion_grace_hours": grace_hours(),
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from . import comments, posts, spec, users
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(spec.router)
|
||||
router.include_router(users.router)
|
||||
router.include_router(posts.router)
|
||||
router.include_router(comments.router)
|
||||
@@ -0,0 +1,66 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from devplacepy.responses import json_error
|
||||
from devplacepy.services.audit import record as audit
|
||||
from devplacepy.services.sync.batch import BatchRunner
|
||||
from devplacepy.services.sync.gate import allowed_origins, sync_enabled
|
||||
from devplacepy.services.sync.records import SyncBatchForm
|
||||
from devplacepy.utils import is_admin, require_user_api
|
||||
|
||||
NO_STORE = {"Cache-Control": "no-store"}
|
||||
|
||||
|
||||
def require_sync_caller(request: Request) -> dict:
|
||||
user = require_user_api(request)
|
||||
if not is_admin(user):
|
||||
audit.record(
|
||||
request,
|
||||
"security.authz.denied",
|
||||
user=user,
|
||||
result="denied",
|
||||
summary=f"non-admin denied access to {request.url.path}",
|
||||
)
|
||||
raise HTTPException(status_code=403, detail="Administrator access required")
|
||||
return user
|
||||
|
||||
|
||||
def run_batch(
|
||||
request: Request, user: dict, kind: str, data: SyncBatchForm
|
||||
) -> JSONResponse:
|
||||
if not sync_enabled():
|
||||
return json_error(503, "Community sync is disabled")
|
||||
allowed = allowed_origins()
|
||||
if allowed and data.origin not in allowed:
|
||||
audit.record(
|
||||
request,
|
||||
"sync.batch",
|
||||
user=user,
|
||||
result="denied",
|
||||
target_type="sync",
|
||||
target_label=f"{data.origin}/{kind}",
|
||||
summary=f"sync from origin {data.origin} refused: not in sync_origins",
|
||||
)
|
||||
return json_error(403, f"origin {data.origin} is not allowed")
|
||||
outcome = BatchRunner(kind, request=request).run(data)
|
||||
audit.record(
|
||||
request,
|
||||
"sync.batch",
|
||||
user=user,
|
||||
target_type="sync",
|
||||
target_label=f"{data.origin}/{kind}",
|
||||
metadata={
|
||||
"origin": data.origin,
|
||||
"kind": kind,
|
||||
"dry_run": data.dry_run,
|
||||
"total": outcome.total,
|
||||
"counts": outcome.counts,
|
||||
},
|
||||
summary=(
|
||||
f"{user['username']} synced {outcome.total} {kind} from {data.origin}"
|
||||
f"{' (dry run)' if data.dry_run else ''}"
|
||||
),
|
||||
)
|
||||
return JSONResponse(outcome.model_dump(mode="json"), headers=NO_STORE)
|
||||
@@ -0,0 +1,21 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from devplacepy.dependencies import json_or_form
|
||||
from devplacepy.routers.sync._shared import require_sync_caller, run_batch
|
||||
from devplacepy.services.sync.records import SyncBatchForm
|
||||
from devplacepy.services.sync.spec import KIND_COMMENTS
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/comments")
|
||||
async def sync_comments(
|
||||
request: Request,
|
||||
user: Annotated[dict, Depends(require_sync_caller)],
|
||||
data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))],
|
||||
):
|
||||
return run_batch(request, user, KIND_COMMENTS, data)
|
||||
@@ -0,0 +1,21 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from devplacepy.dependencies import json_or_form
|
||||
from devplacepy.routers.sync._shared import require_sync_caller, run_batch
|
||||
from devplacepy.services.sync.records import SyncBatchForm
|
||||
from devplacepy.services.sync.spec import KIND_POSTS
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/posts")
|
||||
async def sync_posts(
|
||||
request: Request,
|
||||
user: Annotated[dict, Depends(require_sync_caller)],
|
||||
data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))],
|
||||
):
|
||||
return run_batch(request, user, KIND_POSTS, data)
|
||||
@@ -0,0 +1,18 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from fastapi import APIRouter
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from devplacepy.services.sync.records import RECORD_MODELS
|
||||
from devplacepy.services.sync.spec import SPEC
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/spec.json")
|
||||
async def sync_spec() -> JSONResponse:
|
||||
description = SPEC.describe()
|
||||
description["records"] = {
|
||||
kind: model.model_json_schema() for kind, model in RECORD_MODELS.items()
|
||||
}
|
||||
return JSONResponse(description)
|
||||
@@ -0,0 +1,21 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from devplacepy.dependencies import json_or_form
|
||||
from devplacepy.routers.sync._shared import require_sync_caller, run_batch
|
||||
from devplacepy.services.sync.records import SyncBatchForm
|
||||
from devplacepy.services.sync.spec import KIND_USERS
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/users")
|
||||
async def sync_users(
|
||||
request: Request,
|
||||
user: Annotated[dict, Depends(require_sync_caller)],
|
||||
data: Annotated[SyncBatchForm, Depends(json_or_form(SyncBatchForm))],
|
||||
):
|
||||
return run_batch(request, user, KIND_USERS, data)
|
||||
@@ -113,10 +113,19 @@ from devplacepy.schemas.backups import (
|
||||
BackupStoragePathOut,
|
||||
)
|
||||
from devplacepy.schemas.admin import (
|
||||
AdminCommunitiesOut,
|
||||
AdminCommunityItemOut,
|
||||
AdminCommunityOut,
|
||||
AdminDeviiTaskItemOut,
|
||||
AdminDeviiTasksOut,
|
||||
AdminGameOut,
|
||||
AdminMediaItemOut,
|
||||
AdminNodeDeviceOut,
|
||||
AdminNodeFlagOut,
|
||||
AdminNodePoolOut,
|
||||
AdminNodeQueueOut,
|
||||
AdminNodeSettingOut,
|
||||
AdminNodesOut,
|
||||
AdminMediaOut,
|
||||
AdminNewsItemOut,
|
||||
AdminNewsOut,
|
||||
|
||||
@@ -107,3 +107,128 @@ class AdminGameOut(_Out):
|
||||
era_started_at: str = ""
|
||||
era_ends_at: str = ""
|
||||
admin_section: Optional[str] = None
|
||||
|
||||
|
||||
class AdminCommunityItemOut(_Out):
|
||||
uid: str
|
||||
slug: str
|
||||
name: str = ""
|
||||
origin: str = ""
|
||||
url: str = ""
|
||||
query: str = ""
|
||||
status: str = ""
|
||||
enabled: bool = False
|
||||
description: str = ""
|
||||
language: str = ""
|
||||
user_count: int = 0
|
||||
interval_seconds: int = 0
|
||||
sync_options: dict = {}
|
||||
run_count: int = 0
|
||||
failure_count: int = 0
|
||||
last_run_at: Optional[str] = None
|
||||
next_run_at: Optional[str] = None
|
||||
last_status: Optional[str] = None
|
||||
last_error: Optional[str] = None
|
||||
learned_at: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
login_username: str = ""
|
||||
login_email: str = ""
|
||||
has_login: bool = False
|
||||
|
||||
|
||||
class AdminCommunitiesOut(_Out):
|
||||
items: list[AdminCommunityItemOut] = []
|
||||
sync_enabled: bool = False
|
||||
service_enabled: bool = False
|
||||
service_dry_run: bool = True
|
||||
admin_section: Optional[str] = None
|
||||
|
||||
|
||||
class AdminCommunityOut(_Out):
|
||||
community: AdminCommunityItemOut
|
||||
rules: str = ""
|
||||
favicon: str = ""
|
||||
totals: dict = {}
|
||||
feeds: list[dict] = []
|
||||
runs: list[dict] = []
|
||||
collectors_path: str = ""
|
||||
collectors_lines: int = 0
|
||||
busy: bool = False
|
||||
sync_enabled: bool = False
|
||||
admin_section: Optional[str] = None
|
||||
|
||||
|
||||
class AdminNodeDeviceOut(_Out):
|
||||
uid: str = ""
|
||||
label: str = ""
|
||||
kind: str = ""
|
||||
platform: str = ""
|
||||
client_version: str = ""
|
||||
status: str = "active"
|
||||
suspended_until: Optional[str] = None
|
||||
suspend_reason: str = ""
|
||||
country: str = ""
|
||||
asn_class: str = "unknown"
|
||||
last_seen_at: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
owner_uid: str = ""
|
||||
owner_username: str = ""
|
||||
|
||||
|
||||
class AdminNodeFlagOut(_Out):
|
||||
uid: str = ""
|
||||
kind: str = ""
|
||||
detail: str = ""
|
||||
status: str = "open"
|
||||
device_uid: str = ""
|
||||
device_label: str = ""
|
||||
user_uid: str = ""
|
||||
owner_username: str = ""
|
||||
created_at: Optional[str] = None
|
||||
|
||||
|
||||
class AdminNodeSettingOut(_Out):
|
||||
key: str
|
||||
label: str = ""
|
||||
kind: str = "int"
|
||||
value: str = ""
|
||||
default: str = ""
|
||||
ceiling: Optional[int] = None
|
||||
|
||||
|
||||
class AdminNodePoolOut(_Out):
|
||||
online: bool = False
|
||||
connected: int = 0
|
||||
capacity: int = 0
|
||||
accepted: int = 0
|
||||
kicked: int = 0
|
||||
jobs_done: int = 0
|
||||
jobs_failed: int = 0
|
||||
open_circuits: int = 0
|
||||
heartbeat: Optional[str] = None
|
||||
|
||||
|
||||
class AdminNodeQueueOut(_Out):
|
||||
depth: int = 0
|
||||
max_depth: int = 0
|
||||
queued: int = 0
|
||||
assigned: int = 0
|
||||
done: int = 0
|
||||
failed: int = 0
|
||||
expired: int = 0
|
||||
|
||||
|
||||
class AdminNodesOut(_Out):
|
||||
enabled: bool = False
|
||||
pool: AdminNodePoolOut = AdminNodePoolOut()
|
||||
queue: AdminNodeQueueOut = AdminNodeQueueOut()
|
||||
devices_by_status: dict[str, int] = {}
|
||||
devices_by_kind: dict[str, int] = {}
|
||||
devices_by_asn_class: dict[str, int] = {}
|
||||
devices_by_country: dict[str, int] = {}
|
||||
usage_today: dict[str, int] = {}
|
||||
flags: list[AdminNodeFlagOut] = []
|
||||
devices: list[AdminNodeDeviceOut] = []
|
||||
settings: list[AdminNodeSettingOut] = []
|
||||
limits: dict[str, int] = {}
|
||||
admin_section: Optional[str] = None
|
||||
|
||||
@@ -40,3 +40,74 @@ class NodeTicketOut(_Out):
|
||||
ticket: str
|
||||
expires_in: int
|
||||
ws_path: str
|
||||
|
||||
|
||||
class SocialHitOut(_Out):
|
||||
platform: str = ""
|
||||
title: str = ""
|
||||
url: str = ""
|
||||
snippet: str = ""
|
||||
author: str = ""
|
||||
score: int = 0
|
||||
comments: int = 0
|
||||
created_at: str = ""
|
||||
via: str = "direct"
|
||||
|
||||
|
||||
class SocialSearchOut(_Out):
|
||||
query: str = ""
|
||||
hits: list[SocialHitOut] = []
|
||||
unavailable: dict = {}
|
||||
platforms: list[str] = []
|
||||
|
||||
|
||||
class NodeFetchOut(_Out):
|
||||
url: str = ""
|
||||
status: int = 0
|
||||
via: str = ""
|
||||
truncated: bool = False
|
||||
elapsed_ms: int = 0
|
||||
content_type: str = ""
|
||||
text: str = ""
|
||||
|
||||
|
||||
class NodeMeOut(_Out):
|
||||
consent: bool = False
|
||||
can_donate: bool = False
|
||||
reason: str = ""
|
||||
message: str = ""
|
||||
settings: dict = {}
|
||||
platforms: list[str] = []
|
||||
devices: list[NodeDeviceOut] = []
|
||||
max_devices: int = 0
|
||||
donated: dict = {}
|
||||
limits: dict = {}
|
||||
|
||||
|
||||
class NodeStatsOut(_Out):
|
||||
requests: int = 0
|
||||
bytes: int = 0
|
||||
donors: int = 0
|
||||
|
||||
|
||||
class NodeDonorOut(_Out):
|
||||
username: str = ""
|
||||
requests: int = 0
|
||||
bytes: int = 0
|
||||
|
||||
|
||||
class NodeLeaderboardOut(_Out):
|
||||
donors: list[NodeDonorOut] = []
|
||||
|
||||
|
||||
class NodeArtifactOut(_Out):
|
||||
name: str = ""
|
||||
size: int = 0
|
||||
sha256: str = ""
|
||||
|
||||
|
||||
class NodeClientVersionOut(_Out):
|
||||
version: str = ""
|
||||
protocol: int = 0
|
||||
min_protocol: int = 0
|
||||
artifacts: list[NodeArtifactOut] = []
|
||||
|
||||
@@ -84,6 +84,7 @@ class ProfileOut(_Out):
|
||||
prominent_award: Optional[AwardOut] = None
|
||||
can_give_award: bool = False
|
||||
consents: list[Any] = []
|
||||
node_me: dict = {}
|
||||
mature_opt_in: bool = False
|
||||
age_band: Optional[str] = None
|
||||
terms_version: Optional[str] = None
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from devplacepy.schemas.base import _Out
|
||||
|
||||
|
||||
class SyncRecordResultOut(_Out):
|
||||
index: int
|
||||
origin_uid: str = ""
|
||||
status: str
|
||||
reason: str = ""
|
||||
local_uid: str = ""
|
||||
errors: list[str] = []
|
||||
ignored: list[str] = []
|
||||
|
||||
|
||||
class SyncBatchOut(_Out):
|
||||
origin: str
|
||||
kind: str
|
||||
dry_run: bool = False
|
||||
total: int = 0
|
||||
counts: dict[str, int] = {}
|
||||
results: list[SyncRecordResultOut] = []
|
||||
@@ -50,6 +50,7 @@ CATEGORY_BY_PREFIX: dict[str, str] = {
|
||||
"devii": "devii",
|
||||
"skill": "devii",
|
||||
"node": "nodes",
|
||||
"sync": "sync",
|
||||
"email": "email",
|
||||
"cli": "cli",
|
||||
"reward": "reward",
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
This file documents community discovery (`devplacepy/services/community/`, `routers/admin/communities.py`, `cli/community.py`): finding an external community, learning its structure once, generating collectors, and feeding what they collect through the sync API. Claude Code auto-loads it whenever a file under `devplacepy/services/community/` is read or edited. The design record is `add_community.md`.
|
||||
|
||||
## What exists today
|
||||
|
||||
| Piece | File | State |
|
||||
|-------|------|-------|
|
||||
| Constants, caps, statuses | `spec.py` (`CommunityLimits`) | built |
|
||||
| Path, feed and CSS extraction (stdlib only) | `extract.py` | built, tested |
|
||||
| Collector engine (retry, backoff, pagination, incremental, comments) | `runtime.py` | built, tested |
|
||||
| Search (rsearch), feed detection, structure learning, metadata | `search.py`, `feeds.py`, `learner.py`, `discoverer.py` | built, tested |
|
||||
| Code generation | `generator.py` | built, tested |
|
||||
| Persistence | `store.py` (`communities`, `community_runs`), `files.py` | built |
|
||||
| Push into sync, with dry run | `pusher.py` | built |
|
||||
| Run orchestration | `runner.py` (`CommunityRunner`) | built |
|
||||
| Scheduled service | `service.py` (`CommunityService`, opt-in, dry run by default) | built |
|
||||
| Admin pages | `/admin/communities`, `/admin/communities/{slug}` | built |
|
||||
| CLI | `devplace community discover\|run\|login\|list` | built |
|
||||
| Devii tools (`community_*`), docs group, privacy row | `services/devii/actions/catalog/community.py`, `docs_api/groups/community.py` | built, all administrator-only |
|
||||
| AI structure assistant | `ai.py` | built, optional, every proposal validated against the sample |
|
||||
| Login and credentials | `extract.find_login_form`, `runtime.Collector.login`, `discoverer.LoginLearner` | built |
|
||||
|
||||
## Flow
|
||||
|
||||
`discover` -> `SearchPhase` (a direct address skips rsearch; otherwise `rsearch.app.molodetz.nl/search`, ranked by host/title match, social sites dropped, at most 5 candidates) -> `FeedDetector` (the page itself, `<link rel=alternate>`, same-host "latest/posts/..." links one level deep, then a fixed list of probe paths; JS-only pages escalate to `Fetcher.render`) -> `StructureLearner` per feed -> `MetadataLearner` (name, description, language, favicon, member count, rules text) -> `Collector` verification run on a small sample. A community is only `ready` when the verification run collected at least one post. No feed plus a login wall is `login_required`; anything else is `failed` with the reason.
|
||||
|
||||
Feed types: `rss`, `atom`, `json_api`, `html_list`. Learned field maps use the **sync record field names** (`id`, `title`, `content`, `created_at`, `updated_at`, author `username`/`id`, comment `parent_id`), each an ordered list of alternates; the collector emits records ready for `UserRecord`/`PostRecord`/`CommentRecord`. JSON paths are `$.a.b[0]`, `[*]`. XML feeds are converted to dicts so the same paths work (`$.link.@href`). HTML fields are `selector::text` or `selector::attr(name)`.
|
||||
|
||||
## Browser, JavaScript, login, AI
|
||||
|
||||
- **stealthii is a project dependency** (`pyproject.toml`, git URL; the Dockerfile installs `git`). `fetcher.Fetcher` shares one stealth Chromium per event loop (`SharedBrowser`), applies the SSRF guard (`guard_public_url`) before every request and every `page.goto`, and waits at least `delay_seconds` between requests to one host. Without the browser it falls back to the guarded plain client and logs that JavaScript rendering is unavailable. `CommunityService.on_disable` closes the browser.
|
||||
- **JavaScript pages.** If a page yields nothing statically, detection renders it (`Fetcher.render_html`, `page.content()`); the feed is then marked `"render": true` and the collector renders it on every run. The same fallback applies to post pages when comments only exist after rendering.
|
||||
- **Comments and post bodies from HTML pages.** For any feed (RSS, Atom, JSON, HTML list) whose posts expose an address, `PageProbe` opens up to three post pages, finds the repeated comment element (class hint `comment|reply|response|answer`, nesting from DOM containment, id from `::attr(id)` else a stable hash), and learns a `mode: "page"` comments spec. It also learns a `post.detail.content` selector when the post page carries more text than the listing. JSON feeds still prefer `inline`/`url` comment modes.
|
||||
- **Login.** A community can store a username, an email and a password (columns `login_*`, never returned by any endpoint, admin form, Devii tool or CLI). Discovery finds the login form (page, login links, `/login`-style paths, rendered page), logs in with the session `community:<origin>`, then learns behind the wall. `schema.login` holds the form (action, field names, hidden defaults re-read at login for CSRF tokens, or selectors for a JavaScript page); `login_required` makes a missing account a clear `LoginRequired`. A rejected password raises `LoginFailed` and the community goes to `login_required`. Not supported: JSON-API logins, two-factor, captchas.
|
||||
- **AI.** When HTML heuristics are weak (no author, no title, content only raw text, no date) or find no item container, comment structure or post body, `AiAssistant` asks the internal gateway model (site setting `community_ai_model`, default `molodetz`; `community_ai=0` turns it off) for selectors. A proposal is accepted per field only if it matches at least 60% of the sampled elements. If the gateway is unreachable the learner continues with heuristics and logs it.
|
||||
- **Duplicate feeds.** Feeds whose sample posts overlap by half or more are learned once (the one with comments wins); comment feeds (`/comments.rss`) are never treated as post feeds; generic probe paths are only tried when no feed is advertised.
|
||||
|
||||
## Output: one engine, concrete file
|
||||
|
||||
`runtime.py` is the only collector implementation. `CodeGenerator` writes `data/communities/<slug>/collectors.py` by inlining `extract.py` and `runtime.py` (stdlib only), the learned `SCHEMA` as a literal, and one `collect_<feed>` wrapper per feed with the selectors in comments; plus `schema.json`. The service imports that file, so a human edit to `SCHEMA` takes effect. `python collectors.py` runs it standalone with `stealthii`; a login-protected community reads `COMMUNITY_USERNAME`/`COMMUNITY_EMAIL`/`COMMUNITY_PASSWORD` from the environment. The database row holds the schema; the files are regenerated when missing.
|
||||
|
||||
## Rules that are deliberate
|
||||
|
||||
- **Dry run is first class.** `devplace community run --dry-run`, the admin "Dry run" button and the service setting `community_dry_run` (default on) collect and validate through the real sync processors with `dry_run=True`, log every step, write nothing and keep the watermark. Dependent records (a post whose author this dry run would create) are projected as `created` instead of `unknown_author`. A live run needs `sync_enabled=1` and honors `sync_origins` (`services/sync/gate.py`, shared with the `/sync` routes).
|
||||
- **Nothing unbounded.** Pages, posts, comment fetches, request delay, retries, body size, run time, runs kept (30) and communities per tick are static constants in `CommunityLimits`; no autotuning.
|
||||
- **Floor.** After `max_failures` (5) consecutive failed runs a community is disabled with status `failing` and the log says to relearn. The machine never deletes; a human relearns or deletes (`delete` is soft, `communities` is in `SOFT_DELETE_TABLES` and `UNREPORTABLE_TABLES`).
|
||||
- **Incremental.** The watermark is the newest `updated_at`/`created_at` per feed; a page that is entirely older ends pagination. The newest 10 posts have their comments re-fetched every run; duplicates are absorbed by the sync fingerprint (`unchanged`).
|
||||
- **Outbound HTTP** goes through `Fetcher` (stealthii plus the SSRF guard), never a bare client.
|
||||
- **Order.** users, posts, comments, in batches of 50. A comment's post and parent are always earlier in the batch.
|
||||
- Admin add and relearn run as per-worker `asyncio` tasks; the page shows a `busy` banner and a run row while they work.
|
||||
|
||||
## Measured
|
||||
|
||||
Real run against `https://lobste.rs` (dry run, scratch database): RSS, JSON and HTML feeds detected, description/favicon/language/rules text learned, 25 posts and 24 authors collected from one page. Fixture tests cover RSS, Atom, JSON with inline comments and page pagination, and HTML lists. With stealthii and Chromium the same run learned lobste.rs from its `/rss` feed plus post-page comments (`li.comments_subtree`). Not measured: a real login-protected site, cursor pagination against a live site, the AI path against a live gateway (the local gateway rejected the scratch key, so heuristics ran). Fixtures cover JavaScript rendering, form login, a rejected password, RSS and HTML comments from post pages, feed deduplication and the AI validation.
|
||||
|
||||
## Gaps
|
||||
|
||||
- Comment pagination on HTML post pages is not followed (first page only); `max_comment_posts` bounds the pages opened per run.
|
||||
- No two-factor, captcha or JSON-API login.
|
||||
- HTML learning is heuristic plus AI; a site whose markup defeats both needs a hand-edited `SCHEMA`.
|
||||
- Votes, reactions and attachments are not collected (the sync API does not take them).
|
||||
- Cross-feed duplicate detection compares the sampled post addresses at discovery time only.
|
||||
- Passwords are stored in plain text in the database, as chosen for now; they are masked everywhere they could be displayed.
|
||||
|
||||
## Selection, aliasing, translation
|
||||
|
||||
Per-community options (`sync_options`) and the `community_alias_enabled` kill switch are documented in `devplacepy/services/sync/CLAUDE.md` ("Selection, identity aliasing and translation"). The runner calls `prepare` between collection and `SyncPusher`; skip counts appear in the run's `pushed["skipped"]` and in the totals as `skipped_<reason>`.
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,183 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from devplacepy.config import INTERNAL_MODEL
|
||||
from devplacepy.database import get_setting
|
||||
from devplacepy.database.settings import internal_gateway_key
|
||||
from devplacepy.services.community.extract import Node, field_text, select_nodes
|
||||
from devplacepy.services.community.spec import LIMITS
|
||||
|
||||
logger = logging.getLogger("community.ai")
|
||||
|
||||
SELECTOR_SYNTAX = (
|
||||
"Selector syntax is a small CSS subset: tag, .class, #id, [attr] or [attr=value], descendant (space) "
|
||||
"and child (>) combinators. A field is a selector plus an accessor: 'h2 a::text' for the text, "
|
||||
"'h2 a::attr(href)' for an attribute, and '::attr(id)' (empty selector) for the item element itself. "
|
||||
"No :nth-child, no pseudo-classes, no commas."
|
||||
)
|
||||
FENCE = re.compile(r"^```(?:json)?\s*|\s*```$", re.I)
|
||||
OUTLINE_LINES = 160
|
||||
SAMPLE_HTML_CHARS = 1800
|
||||
AI_TIMEOUT_SECONDS = 90
|
||||
|
||||
|
||||
def outline(root: Node, limit: int = OUTLINE_LINES) -> str:
|
||||
lines: list[str] = []
|
||||
|
||||
def walk(node: Node, depth: int) -> None:
|
||||
if len(lines) >= limit or depth > 9:
|
||||
return
|
||||
seen: dict[str, int] = {}
|
||||
for child in node.children:
|
||||
if len(lines) >= limit:
|
||||
return
|
||||
if child.tag in ("script", "style", "svg", "noscript", "link", "meta", "head"):
|
||||
continue
|
||||
label = child.tag + (f"#{child.attrs['id']}" if child.attrs.get("id") else "") + "".join(f".{name}" for name in sorted(child.classes()))
|
||||
seen[label] = seen.get(label, 0) + 1
|
||||
if seen[label] > 2:
|
||||
continue
|
||||
text = child.text()[:40]
|
||||
lines.append(f"{' ' * depth}{label}" + (f" \"{text}\"" if text and not child.children else ""))
|
||||
walk(child, depth + 1)
|
||||
|
||||
walk(root, 0)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_html(node: Node, limit: int = SAMPLE_HTML_CHARS) -> str:
|
||||
parts: list[str] = []
|
||||
|
||||
def walk(current: Node) -> None:
|
||||
attributes = "".join(f' {key}="{value[:80]}"' for key, value in current.attrs.items() if key in ("class", "id", "href", "datetime", "rel", "itemprop", "data-id", "name"))
|
||||
parts.append(f"<{current.tag}{attributes}>")
|
||||
for chunk in current.chunks:
|
||||
if isinstance(chunk, Node):
|
||||
if chunk.tag not in ("script", "style", "svg"):
|
||||
walk(chunk)
|
||||
elif chunk.strip():
|
||||
parts.append(chunk.strip()[:160])
|
||||
parts.append(f"</{current.tag}>")
|
||||
|
||||
walk(node)
|
||||
return "".join(parts)[:limit]
|
||||
|
||||
|
||||
def parse_json(text: str):
|
||||
cleaned = FENCE.sub("", text.strip())
|
||||
start, end = cleaned.find("{"), cleaned.rfind("}")
|
||||
if start < 0 or end <= start:
|
||||
return None
|
||||
try:
|
||||
return json.loads(cleaned[start : end + 1])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def coverage(nodes: list[Node], spec: str) -> float:
|
||||
if not nodes or not spec:
|
||||
return 0.0
|
||||
try:
|
||||
return sum(1 for node in nodes if field_text(node, spec)) / len(nodes)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
|
||||
|
||||
class AiAssistant:
|
||||
def enabled(self) -> bool:
|
||||
return (get_setting("community_ai", "1") or "1") == "1"
|
||||
|
||||
async def ask(self, system: str, user: str) -> dict | None:
|
||||
if not self.enabled():
|
||||
return None
|
||||
from devplacepy.services.correction import gateway_complete
|
||||
|
||||
model = get_setting("community_ai_model", "") or INTERNAL_MODEL
|
||||
reply, usage = await asyncio.to_thread(
|
||||
gateway_complete, internal_gateway_key(), system, user, AI_TIMEOUT_SECONDS, None, model, True, False
|
||||
)
|
||||
if usage is None:
|
||||
logger.warning("ai: the %s model is unreachable, continuing with heuristics only", model)
|
||||
return None
|
||||
answer = parse_json(reply)
|
||||
if answer is None:
|
||||
logger.warning("ai: the model did not return JSON")
|
||||
else:
|
||||
logger.info("ai: %s answered %s", model, json.dumps(answer)[:300])
|
||||
return answer
|
||||
|
||||
def validated(self, nodes: list[Node], proposal: dict | None, names: tuple[str, ...]) -> dict:
|
||||
accepted: dict[str, str] = {}
|
||||
for name in names:
|
||||
spec = (proposal or {}).get(name)
|
||||
if isinstance(spec, str) and coverage(nodes, spec) >= LIMITS.min_field_coverage:
|
||||
accepted[name] = spec
|
||||
elif spec:
|
||||
logger.info("ai: rejected %s=%r, it matches too few sampled items", name, spec)
|
||||
return accepted
|
||||
|
||||
async def item_selector(self, root: Node) -> str | None:
|
||||
answer = await self.ask(
|
||||
"You analyse the HTML outline of a community listing page. Reply with JSON only: "
|
||||
'{"item_selector": "<selector of the repeated element that wraps exactly ONE post or thread>"}. '
|
||||
+ SELECTOR_SYNTAX,
|
||||
outline(root),
|
||||
)
|
||||
selector = (answer or {}).get("item_selector")
|
||||
if isinstance(selector, str):
|
||||
try:
|
||||
if len(select_nodes(root, selector)) >= 3:
|
||||
return selector
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
async def post_fields(self, nodes: list[Node]) -> dict:
|
||||
samples = "\n\n".join(to_html(node) for node in nodes[:3])
|
||||
answer = await self.ask(
|
||||
"You analyse HTML of community listing items. Reply with JSON only, keys: title, link, content, "
|
||||
"author, created_at. Each value is a field selector relative to ONE item (omit a key when the item "
|
||||
"does not contain it). link should end in ::attr(href), created_at in ::attr(datetime) or ::text. "
|
||||
+ SELECTOR_SYNTAX,
|
||||
samples,
|
||||
)
|
||||
return self.validated(nodes, answer, ("title", "link", "content", "author", "created_at"))
|
||||
|
||||
async def comment_structure(self, root: Node) -> dict | None:
|
||||
answer = await self.ask(
|
||||
"You analyse the HTML outline of a community post page that has comments. Reply with JSON only: "
|
||||
'{"item_selector": "<selector of the element wrapping ONE comment, matching every comment including '
|
||||
'nested replies>", "id": "<field selector for a unique id, usually ::attr(id)>", "content": "...", '
|
||||
'"author": "...", "created_at": "..."}. Field selectors are relative to one comment. ' + SELECTOR_SYNTAX,
|
||||
outline(root, OUTLINE_LINES * 2),
|
||||
)
|
||||
selector = (answer or {}).get("item_selector")
|
||||
if not isinstance(selector, str):
|
||||
return None
|
||||
try:
|
||||
nodes = select_nodes(root, selector)
|
||||
except ValueError:
|
||||
return None
|
||||
if not nodes:
|
||||
return None
|
||||
fields = self.validated(nodes, answer, ("id", "content", "author", "created_at"))
|
||||
if "content" not in fields:
|
||||
return None
|
||||
return {"item_selector": selector, **fields}
|
||||
|
||||
async def detail_content(self, root: Node) -> str | None:
|
||||
answer = await self.ask(
|
||||
"You analyse the HTML outline of a single community post page. Reply with JSON only: "
|
||||
'{"content": "<field selector for the full text body of the post>"}. ' + SELECTOR_SYNTAX,
|
||||
outline(root),
|
||||
)
|
||||
spec = (answer or {}).get("content")
|
||||
if isinstance(spec, str) and len(field_text(root, spec)) > 40:
|
||||
return spec
|
||||
return None
|
||||
@@ -0,0 +1,261 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urljoin
|
||||
from dataclasses import dataclass, field
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from devplacepy.services.community.ai import AiAssistant
|
||||
from devplacepy.services.community.extract import field_text, find_login_form, parse_html, select_nodes
|
||||
from devplacepy.services.community.feeds import FeedDetector
|
||||
from devplacepy.services.community.learner import MetadataLearner, StructureLearner
|
||||
from devplacepy.services.community.runtime import Collector, LoginFailed, LoginRequired
|
||||
from devplacepy.services.community.search import SearchPhase
|
||||
from devplacepy.services.community.spec import (
|
||||
LIMITS,
|
||||
SCHEMA_VERSION,
|
||||
STATUS_FAILED,
|
||||
STATUS_LOGIN,
|
||||
STATUS_READY,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("community.discover")
|
||||
|
||||
MAX_FEEDS = 4
|
||||
|
||||
|
||||
@dataclass
|
||||
class Discovery:
|
||||
query: str
|
||||
status: str = STATUS_FAILED
|
||||
name: str = ""
|
||||
slug: str = ""
|
||||
origin: str = ""
|
||||
url: str = ""
|
||||
schema: dict = field(default_factory=dict)
|
||||
metadata: dict = field(default_factory=dict)
|
||||
sample: dict = field(default_factory=dict)
|
||||
error: str = ""
|
||||
|
||||
|
||||
def origin_of(url: str) -> str:
|
||||
host = (urlsplit(url).hostname or "").lower()
|
||||
return host[4:] if host.startswith("www.") else host
|
||||
|
||||
|
||||
def slug_of(origin: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", origin).strip("-")[:63]
|
||||
|
||||
|
||||
def slug_for_query(query: str) -> str:
|
||||
from devplacepy.services.community.search import direct_url
|
||||
|
||||
address = direct_url(query)
|
||||
return slug_of(origin_of(address)) if address else re.sub(r"[^a-z0-9]+", "-", query.lower()).strip("-")[:63]
|
||||
|
||||
|
||||
LOGIN_PATHS = ("/login", "/signin", "/sign-in", "/accounts/login", "/users/sign_in", "/auth/login", "/user/login")
|
||||
LOGIN_LABEL = re.compile(r"\b(log ?in|sign ?in)\b", re.I)
|
||||
MAX_LOGIN_PAGES = 6
|
||||
|
||||
|
||||
class LoginLearner:
|
||||
def __init__(self, fetcher):
|
||||
self.fetcher = fetcher
|
||||
|
||||
async def page(self, url: str, render: bool) -> str:
|
||||
try:
|
||||
if render:
|
||||
status, text = await self.fetcher.render_html(url)
|
||||
else:
|
||||
response = await self.fetcher.get(url)
|
||||
status, text = response.status, await response.text()
|
||||
except Exception as exc:
|
||||
logger.info("login: %s unreachable: %s: %s", url, type(exc).__name__, exc)
|
||||
return ""
|
||||
return text if status < 400 else ""
|
||||
|
||||
async def find(self, base_url: str, html: str) -> dict | None:
|
||||
spec = find_login_form(parse_html(html), base_url) if html else None
|
||||
if spec:
|
||||
return spec
|
||||
urls = []
|
||||
if html:
|
||||
for node in select_nodes(parse_html(html), "a[href]"):
|
||||
if LOGIN_LABEL.search(node.text()[:30]):
|
||||
urls.append(urljoin(base_url, node.attrs["href"]))
|
||||
parts = urlsplit(base_url)
|
||||
urls.extend(f"{parts.scheme}://{parts.netloc}{path}" for path in LOGIN_PATHS)
|
||||
urls = list(dict.fromkeys(urls))[:MAX_LOGIN_PAGES]
|
||||
for render in (False, True):
|
||||
for url in urls[: MAX_LOGIN_PAGES if not render else 2]:
|
||||
spec = find_login_form(parse_html(await self.page(url, render)), url)
|
||||
if spec:
|
||||
spec["render"] = render
|
||||
logger.info("login: found a %s login form at %s%s", spec["mode"], url, " (needs JavaScript)" if render else "")
|
||||
return spec
|
||||
return None
|
||||
|
||||
|
||||
class CommunityDiscoverer:
|
||||
def __init__(self, fetcher, ai=None, credentials: dict | None = None):
|
||||
self.fetcher = fetcher
|
||||
self.ai = ai or AiAssistant()
|
||||
self.credentials = credentials or {}
|
||||
self.search = SearchPhase(fetcher)
|
||||
self.detector = FeedDetector(fetcher, self.ai)
|
||||
self.learner = StructureLearner(fetcher, self.ai)
|
||||
self.metadata = MetadataLearner(fetcher)
|
||||
self.logins = LoginLearner(fetcher)
|
||||
|
||||
def has_credentials(self) -> bool:
|
||||
return bool(self.credentials.get("password") and (self.credentials.get("username") or self.credentials.get("email")))
|
||||
|
||||
async def discover(self, query: str) -> Discovery:
|
||||
result = Discovery(query=query)
|
||||
logger.info("discover: starting for %r", query)
|
||||
try:
|
||||
candidates = await self.search.run(query)
|
||||
except Exception as exc:
|
||||
result.error = f"search failed: {exc}"
|
||||
logger.error("discover: %s", result.error)
|
||||
return result
|
||||
if not candidates:
|
||||
result.error = "search returned no usable candidates"
|
||||
logger.warning("discover: %s", result.error)
|
||||
return result
|
||||
for candidate in candidates:
|
||||
logger.info("discover: trying %s", candidate.url)
|
||||
found = await self.try_candidate(query, candidate.url, result)
|
||||
if found:
|
||||
return result
|
||||
if result.status == STATUS_LOGIN:
|
||||
return result
|
||||
result.error = result.error or "no candidate exposed a learnable feed"
|
||||
logger.warning("discover: gave up: %s", result.error)
|
||||
return result
|
||||
|
||||
async def try_candidate(self, query: str, url: str, result: Discovery) -> bool:
|
||||
origin = origin_of(url)
|
||||
self.fetcher.session = f"community:{origin}"
|
||||
candidates, context = await self.detector.detect(url)
|
||||
walled = bool(context.get("login_required")) or not candidates
|
||||
login = None
|
||||
if walled or self.has_credentials():
|
||||
login = await self.logins.find(url, context.get("html", ""))
|
||||
if login and self.has_credentials():
|
||||
probe = {"community": origin, "origin": origin, "base_url": url, "login": login, "limits": LIMITS.runtime()}
|
||||
try:
|
||||
await Collector(self.fetcher, probe, credentials=self.credentials).login()
|
||||
logger.info("discover: logged in to %s", origin)
|
||||
except LoginFailed as exc:
|
||||
result.error, result.status = str(exc), STATUS_FAILED
|
||||
logger.error("discover: %s", exc)
|
||||
return False
|
||||
if walled:
|
||||
candidates, context = await self.detector.detect(url)
|
||||
logger.info("discover: %d feed candidate(s) at %s", len(candidates), url)
|
||||
learned = []
|
||||
for candidate in candidates:
|
||||
if len(learned) >= MAX_FEEDS * 2:
|
||||
break
|
||||
feed = await self.learner.learn(candidate, len(learned))
|
||||
if feed:
|
||||
learned.append((candidate, feed))
|
||||
feeds = self.distinct(learned)
|
||||
home = await self.home_page(url, context.get("html", ""))
|
||||
meta = await self.metadata.learn(home[0], home[1], {})
|
||||
result.url, result.origin, result.slug = url, origin, slug_of(origin)
|
||||
result.metadata = meta
|
||||
result.name = meta.get("name") or origin
|
||||
wall = walled and login is not None
|
||||
if not feeds:
|
||||
if walled and login:
|
||||
result.status = STATUS_LOGIN
|
||||
result.error = "the community requires a login; add the account's username or email and password and relearn" if not self.has_credentials() else "no feed could be learned even after logging in"
|
||||
result.schema = self.schema(result, [], login_required=True, login=login)
|
||||
logger.warning("discover: %s", result.error)
|
||||
else:
|
||||
result.error = f"no feed could be learned at {url}"
|
||||
return False
|
||||
schema = self.schema(result, feeds, login_required=wall, login=login)
|
||||
sample = await self.verify(schema)
|
||||
if not sample["records"]["posts"]:
|
||||
result.error = f"learned {len(feeds)} feed(s) but the verification run collected no posts"
|
||||
logger.warning("discover: %s", result.error)
|
||||
return False
|
||||
result.schema, result.sample = schema, sample
|
||||
result.status = STATUS_READY
|
||||
logger.info(
|
||||
"discover: ready %s: %d feed(s), sample %d users / %d posts / %d comments",
|
||||
origin, len(feeds), *(len(sample["records"][kind]) for kind in ("users", "posts", "comments")),
|
||||
)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def canonical(link: str) -> str:
|
||||
parts = urlsplit(link)
|
||||
return f"{(parts.hostname or '').removeprefix('www.')}{parts.path.rstrip('/')}".lower()
|
||||
|
||||
def links_of(self, candidate, feed: dict) -> set[str]:
|
||||
spec = feed["post"]["fields"].get("url") or feed["post"]["fields"].get("id")
|
||||
return {
|
||||
self.canonical(urljoin(candidate.url, field_text(item, spec)))
|
||||
for item in candidate.items[: LIMITS.sample_posts]
|
||||
if field_text(item, spec)
|
||||
}
|
||||
|
||||
def distinct(self, learned: list) -> list[dict]:
|
||||
ranked = sorted(enumerate(learned), key=lambda entry: (entry[1][1]["comments"]["mode"] == "none", entry[0]))
|
||||
kept: list[tuple[set, dict]] = []
|
||||
for _, (candidate, feed) in ranked:
|
||||
links = self.links_of(candidate, feed)
|
||||
twin = next((other for other, _ in kept if links and other and len(links & other) / min(len(links), len(other)) >= 0.5), None)
|
||||
if twin is not None:
|
||||
logger.info("discover: %s lists the same posts as a better feed, skipping it", feed["url"])
|
||||
continue
|
||||
kept.append((links, feed))
|
||||
if len(kept) >= MAX_FEEDS:
|
||||
break
|
||||
feeds = [feed for _, feed in kept]
|
||||
for index, feed in enumerate(feeds):
|
||||
feed["id"] = f"feed{index + 1}"
|
||||
return feeds
|
||||
|
||||
async def home_page(self, url: str, html: str) -> tuple[str, str]:
|
||||
if html.lstrip()[:1] == "<" and not html.lstrip().startswith("<?xml"):
|
||||
return url, html
|
||||
parts = urlsplit(url)
|
||||
home = f"{parts.scheme}://{parts.netloc}/"
|
||||
try:
|
||||
response = await self.fetcher.get(home)
|
||||
return home, await response.text() if response.status < 400 else ""
|
||||
except Exception as exc:
|
||||
logger.info("discover: home page %s unavailable: %s", home, exc)
|
||||
return home, ""
|
||||
|
||||
def schema(self, result: Discovery, feeds: list[dict], login_required: bool = False, login: dict | None = None) -> dict:
|
||||
schema = {
|
||||
"version": SCHEMA_VERSION,
|
||||
"community": result.name,
|
||||
"origin": result.origin,
|
||||
"base_url": result.url,
|
||||
"login_required": login_required,
|
||||
"limits": LIMITS.runtime(),
|
||||
"feeds": feeds,
|
||||
}
|
||||
if login:
|
||||
schema["login"] = login
|
||||
return schema
|
||||
|
||||
async def verify(self, schema: dict) -> dict:
|
||||
trial = {**schema, "limits": {**schema["limits"], "max_pages": 1, "max_posts": LIMITS.sample_posts, "max_comment_posts": LIMITS.sample_comment_posts, "delay_seconds": 0.2}}
|
||||
collector = Collector(self.fetcher, trial, credentials=self.credentials)
|
||||
try:
|
||||
return await collector.run()
|
||||
except LoginRequired as exc:
|
||||
logger.warning("discover: verification hit a login wall: %s", exc)
|
||||
return {"records": {"users": [], "posts": [], "comments": []}, "errors": [str(exc)]}
|
||||
@@ -0,0 +1,469 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import xml.etree.ElementTree as ElementTree
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urljoin
|
||||
|
||||
PATH_TOKEN = re.compile(r"""\.\s*([^.\[\]]+)|\[\s*(\*|-?\d+|"[^"]*"|'[^']*')\s*\]""")
|
||||
SELECTOR_PART = re.compile(r"^([a-zA-Z][a-zA-Z0-9-]*|\*)?((?:[#.][\w-]+|\[[^\]]+\])*)$")
|
||||
SELECTOR_ATOM = re.compile(r"#([\w-]+)|\.([\w-]+)|\[\s*([\w:-]+)\s*(?:([~^$*|]?=)\s*[\"']?([^\"'\]]*)[\"']?)?\s*\]")
|
||||
VOID_TAGS = frozenset(
|
||||
{"br", "hr", "img", "meta", "link", "input", "source", "area", "base", "col", "embed", "track", "wbr"}
|
||||
)
|
||||
SKIP_TEXT_TAGS = frozenset({"script", "style", "noscript", "template"})
|
||||
EPOCH_MS_THRESHOLD = 100_000_000_000
|
||||
MAX_PATH_DEPTH = 24
|
||||
NOW_TOKEN = "$now"
|
||||
|
||||
|
||||
def parse_path(path: str) -> list:
|
||||
text = path.strip()
|
||||
if not text.startswith("$"):
|
||||
raise ValueError(f"path must start with $: {path!r}")
|
||||
rest = text[1:]
|
||||
tokens: list = []
|
||||
position = 0
|
||||
while position < len(rest):
|
||||
match = PATH_TOKEN.match(rest, position)
|
||||
if not match:
|
||||
raise ValueError(f"cannot parse path {path!r} at {rest[position:]!r}")
|
||||
key, bracket = match.groups()
|
||||
if key is not None:
|
||||
tokens.append(key.strip())
|
||||
elif bracket == "*":
|
||||
tokens.append(None)
|
||||
elif bracket[0] in "\"'":
|
||||
tokens.append(bracket[1:-1])
|
||||
else:
|
||||
tokens.append(int(bracket))
|
||||
position = match.end()
|
||||
if len(tokens) > MAX_PATH_DEPTH:
|
||||
raise ValueError(f"path too deep: {path!r}")
|
||||
return tokens
|
||||
|
||||
|
||||
def select(data, path: str) -> list:
|
||||
current = [data]
|
||||
for token in parse_path(path):
|
||||
following = []
|
||||
for node in current:
|
||||
if token is None:
|
||||
if isinstance(node, list):
|
||||
following.extend(node)
|
||||
elif isinstance(node, dict):
|
||||
following.extend(node.values())
|
||||
elif isinstance(token, int):
|
||||
if isinstance(node, list) and -len(node) <= token < len(node):
|
||||
following.append(node[token])
|
||||
elif isinstance(node, dict) and token in node:
|
||||
following.append(node[token])
|
||||
elif isinstance(node, list):
|
||||
following.extend(
|
||||
item[token] for item in node if isinstance(item, dict) and token in item
|
||||
)
|
||||
current = following
|
||||
return current
|
||||
|
||||
|
||||
def scalar_text(value) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
if isinstance(value, dict):
|
||||
for key in ("#text", "text", "value", "@href", "href", "url", "name"):
|
||||
if key in value:
|
||||
return scalar_text(value[key])
|
||||
return ""
|
||||
if isinstance(value, list):
|
||||
for entry in value:
|
||||
text = scalar_text(entry)
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def text_at(data, path: str) -> str:
|
||||
for value in select(data, path):
|
||||
text = scalar_text(value)
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def strip_namespace(tag: str) -> str:
|
||||
if "}" in tag:
|
||||
tag = tag.split("}", 1)[1]
|
||||
return tag
|
||||
|
||||
|
||||
def element_value(element):
|
||||
children = list(element)
|
||||
attributes = {f"@{strip_namespace(key)}": value for key, value in element.attrib.items()}
|
||||
text = (element.text or "").strip()
|
||||
if not children and not attributes:
|
||||
return text
|
||||
entry = dict(attributes)
|
||||
if text:
|
||||
entry["#text"] = text
|
||||
for child in children:
|
||||
name = strip_namespace(child.tag)
|
||||
value = element_value(child)
|
||||
if name in entry:
|
||||
if not isinstance(entry[name], list):
|
||||
entry[name] = [entry[name]]
|
||||
entry[name].append(value)
|
||||
else:
|
||||
entry[name] = value
|
||||
return entry
|
||||
|
||||
|
||||
def feed_kind(text: str) -> str | None:
|
||||
head = text.lstrip()[:2048].lower()
|
||||
if "<rss" in head or "<rdf:rdf" in head:
|
||||
return "rss"
|
||||
if "<feed" in head:
|
||||
return "atom"
|
||||
return None
|
||||
|
||||
|
||||
def parse_feed(text: str) -> tuple[str, dict, list[dict]]:
|
||||
kind = feed_kind(text)
|
||||
if kind is None:
|
||||
raise ValueError("not an RSS or Atom document")
|
||||
root = ElementTree.fromstring(text.encode("utf-8") if isinstance(text, str) else text)
|
||||
channel = root.find("channel") if kind == "rss" else root
|
||||
if channel is None:
|
||||
channel = root
|
||||
item_tag = "item" if kind == "rss" else "entry"
|
||||
meta: dict = {}
|
||||
items: list[dict] = []
|
||||
for child in list(channel) + (list(root) if channel is not root else []):
|
||||
name = strip_namespace(child.tag)
|
||||
if name == item_tag:
|
||||
value = element_value(child)
|
||||
if isinstance(value, dict):
|
||||
items.append(value)
|
||||
elif child is not channel and name in ("title", "description", "subtitle", "language", "link"):
|
||||
meta.setdefault(name, element_value(child))
|
||||
return kind, meta, items
|
||||
|
||||
|
||||
class Node:
|
||||
__slots__ = ("tag", "attrs", "children", "parent", "chunks")
|
||||
|
||||
def __init__(self, tag: str, attrs: dict, parent: "Node | None" = None):
|
||||
self.tag = tag
|
||||
self.attrs = attrs
|
||||
self.children: list[Node] = []
|
||||
self.parent = parent
|
||||
self.chunks: list = []
|
||||
|
||||
def text(self) -> str:
|
||||
parts = []
|
||||
for chunk in self.chunks:
|
||||
if isinstance(chunk, Node):
|
||||
if chunk.tag not in SKIP_TEXT_TAGS:
|
||||
parts.append(chunk.text())
|
||||
else:
|
||||
parts.append(chunk)
|
||||
return re.sub(r"\s+", " ", " ".join(parts)).strip()
|
||||
|
||||
def classes(self) -> set[str]:
|
||||
return set(self.attrs.get("class", "").split())
|
||||
|
||||
def descendants(self):
|
||||
for child in self.children:
|
||||
yield child
|
||||
yield from child.descendants()
|
||||
|
||||
|
||||
class TreeBuilder(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.root = Node("#root", {})
|
||||
self.current = self.root
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
node = Node(tag, {key: (value or "") for key, value in attrs}, self.current)
|
||||
self.current.children.append(node)
|
||||
self.current.chunks.append(node)
|
||||
if tag not in VOID_TAGS:
|
||||
self.current = node
|
||||
|
||||
def handle_startendtag(self, tag, attrs):
|
||||
node = Node(tag, {key: (value or "") for key, value in attrs}, self.current)
|
||||
self.current.children.append(node)
|
||||
self.current.chunks.append(node)
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
cursor = self.current
|
||||
while cursor is not self.root and cursor.tag != tag:
|
||||
cursor = cursor.parent
|
||||
if cursor is not self.root:
|
||||
self.current = cursor.parent
|
||||
|
||||
def handle_data(self, data):
|
||||
self.current.chunks.append(data)
|
||||
|
||||
|
||||
def parse_html(text: str) -> Node:
|
||||
builder = TreeBuilder()
|
||||
builder.feed(text)
|
||||
builder.close()
|
||||
return builder.root
|
||||
|
||||
|
||||
def compile_compound(part: str):
|
||||
match = SELECTOR_PART.match(part)
|
||||
if not match:
|
||||
raise ValueError(f"unsupported selector: {part!r}")
|
||||
tag, rest = match.groups()
|
||||
ident = None
|
||||
classes: list[str] = []
|
||||
attributes: list[tuple] = []
|
||||
for atom in SELECTOR_ATOM.finditer(rest):
|
||||
atom_id, atom_class, name, operator, value = atom.groups()
|
||||
if atom_id:
|
||||
ident = atom_id
|
||||
elif atom_class:
|
||||
classes.append(atom_class)
|
||||
else:
|
||||
attributes.append((name, operator, value))
|
||||
return (None if tag in (None, "*") else tag.lower(), ident, classes, attributes)
|
||||
|
||||
|
||||
def compound_matches(node: Node, compound) -> bool:
|
||||
tag, ident, classes, attributes = compound
|
||||
if tag and node.tag != tag:
|
||||
return False
|
||||
if ident and node.attrs.get("id") != ident:
|
||||
return False
|
||||
if classes and not set(classes) <= node.classes():
|
||||
return False
|
||||
for name, operator, value in attributes:
|
||||
if name not in node.attrs:
|
||||
return False
|
||||
actual = node.attrs[name]
|
||||
if operator == "=" and actual != value:
|
||||
return False
|
||||
if operator == "^=" and not actual.startswith(value):
|
||||
return False
|
||||
if operator == "$=" and not actual.endswith(value):
|
||||
return False
|
||||
if operator == "*=" and value not in actual:
|
||||
return False
|
||||
if operator == "~=" and value not in actual.split():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def parse_selector(selector: str) -> list[tuple[str, tuple]]:
|
||||
spaced = re.sub(r"\s*>\s*", " > ", selector.strip())
|
||||
steps: list[tuple[str, tuple]] = []
|
||||
combinator = " "
|
||||
for part in spaced.split():
|
||||
if part == ">":
|
||||
combinator = ">"
|
||||
continue
|
||||
steps.append((combinator, compile_compound(part)))
|
||||
combinator = " "
|
||||
if not steps:
|
||||
raise ValueError("empty selector")
|
||||
return steps
|
||||
|
||||
|
||||
def path_matches(node: Node, steps: list[tuple[str, tuple]], index: int) -> bool:
|
||||
combinator, compound = steps[index]
|
||||
if not compound_matches(node, compound):
|
||||
return False
|
||||
if index == 0:
|
||||
return True
|
||||
parent = node.parent
|
||||
if combinator == ">":
|
||||
return parent is not None and path_matches(parent, steps, index - 1)
|
||||
while parent is not None and parent.tag != "#root":
|
||||
if path_matches(parent, steps, index - 1):
|
||||
return True
|
||||
parent = parent.parent
|
||||
return False
|
||||
|
||||
|
||||
def select_nodes(root: Node, selector: str) -> list[Node]:
|
||||
steps = parse_selector(selector)
|
||||
return [node for node in root.descendants() if path_matches(node, steps, len(steps) - 1)]
|
||||
|
||||
|
||||
def html_field(node: Node, spec: str) -> str:
|
||||
selector, _, accessor = spec.partition("::")
|
||||
target = node
|
||||
if selector.strip():
|
||||
matches = select_nodes(node, selector)
|
||||
if not matches:
|
||||
return ""
|
||||
target = matches[0]
|
||||
accessor = accessor or "text"
|
||||
if accessor == "text":
|
||||
return target.text()
|
||||
if accessor.startswith("attr(") and accessor.endswith(")"):
|
||||
return target.attrs.get(accessor[5:-1], "").strip()
|
||||
raise ValueError(f"unsupported accessor: {accessor!r}")
|
||||
|
||||
|
||||
def field_text(item, spec) -> str:
|
||||
specs = spec if isinstance(spec, (list, tuple)) else [spec]
|
||||
for entry in specs:
|
||||
if not entry:
|
||||
continue
|
||||
if entry == NOW_TOKEN:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
text = html_field(item, entry) if isinstance(item, Node) else text_at(item, entry)
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def normalize_instant(value) -> str | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
if isinstance(value, (int, float)) or (isinstance(value, str) and value.strip().isdigit()):
|
||||
number = float(value)
|
||||
if number >= EPOCH_MS_THRESHOLD:
|
||||
number /= 1000
|
||||
try:
|
||||
return datetime.fromtimestamp(number, timezone.utc).isoformat()
|
||||
except (OverflowError, OSError, ValueError):
|
||||
return None
|
||||
text = str(value).strip()
|
||||
try:
|
||||
moment = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
try:
|
||||
moment = parsedate_to_datetime(text)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if moment.tzinfo is None:
|
||||
moment = moment.replace(tzinfo=timezone.utc)
|
||||
return moment.astimezone(timezone.utc).isoformat()
|
||||
|
||||
|
||||
BLOCK_TAGS = frozenset(
|
||||
{"p", "div", "section", "article", "li", "ul", "ol", "tr", "blockquote", "pre", "br",
|
||||
"h1", "h2", "h3", "h4", "h5", "h6"}
|
||||
)
|
||||
|
||||
|
||||
def block_text(node: Node) -> str:
|
||||
parts = []
|
||||
for chunk in node.chunks:
|
||||
if isinstance(chunk, Node):
|
||||
if chunk.tag in SKIP_TEXT_TAGS:
|
||||
continue
|
||||
inner = block_text(chunk)
|
||||
parts.append(f"\n\n{inner}\n\n" if chunk.tag in BLOCK_TAGS else inner)
|
||||
else:
|
||||
parts.append(chunk)
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def strip_tags(text: str) -> str:
|
||||
if "<" not in text:
|
||||
return text.strip()
|
||||
raw = block_text(parse_html(text))
|
||||
lines = [re.sub(r"[ \t]+", " ", line).strip() for line in raw.split("\n")]
|
||||
return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip()
|
||||
|
||||
|
||||
def json_or_none(text: str):
|
||||
try:
|
||||
return json.loads(text)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def leaf_paths(data, prefix: str = "$", depth: int = 0) -> dict[str, object]:
|
||||
found: dict[str, object] = {}
|
||||
if depth > 4:
|
||||
return found
|
||||
if isinstance(data, dict):
|
||||
for key, value in data.items():
|
||||
if not re.fullmatch(r"[\w@#:-]+", str(key)):
|
||||
continue
|
||||
found.update(leaf_paths(value, f"{prefix}.{key}", depth + 1))
|
||||
elif isinstance(data, list):
|
||||
if data and all(not isinstance(entry, (dict, list)) for entry in data[:3]):
|
||||
found[prefix] = data
|
||||
elif data:
|
||||
found.update(leaf_paths(data[0], f"{prefix}[0]", depth + 1))
|
||||
else:
|
||||
found[prefix] = data
|
||||
return found
|
||||
|
||||
|
||||
IDENTITY_NAME = re.compile(r"(user|login|email|mail|account|identifier|handle|name)", re.I)
|
||||
SKIP_INPUT_TYPES = frozenset({"submit", "button", "image", "reset", "checkbox", "radio", "file"})
|
||||
|
||||
|
||||
def find_login_form(root: Node, base_url: str) -> dict | None:
|
||||
for form in select_nodes(root, "form"):
|
||||
inputs = [node for node in form.descendants() if node.tag == "input"]
|
||||
passwords = [node for node in inputs if node.attrs.get("type", "").lower() == "password"]
|
||||
if not passwords:
|
||||
continue
|
||||
password = passwords[0]
|
||||
identity = next(
|
||||
(
|
||||
node
|
||||
for node in inputs
|
||||
if node.attrs.get("type", "text").lower() in ("text", "email", "tel")
|
||||
and node.attrs.get("name")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if identity is None or not password.attrs.get("name"):
|
||||
continue
|
||||
defaults = {
|
||||
node.attrs["name"]: node.attrs.get("value", "")
|
||||
for node in inputs
|
||||
if node.attrs.get("name")
|
||||
and node.attrs.get("type", "text").lower() not in SKIP_INPUT_TYPES
|
||||
and node is not password
|
||||
and node is not identity
|
||||
}
|
||||
kind = identity.attrs.get("type", "text").lower()
|
||||
name = identity.attrs["name"]
|
||||
wants_email = kind == "email" or bool(re.search(r"mail", name, re.I))
|
||||
return {
|
||||
"mode": "form",
|
||||
"url": base_url,
|
||||
"action": urljoin(base_url, form.attrs.get("action") or base_url),
|
||||
"method": (form.attrs.get("method") or "post").upper(),
|
||||
"fields": defaults,
|
||||
"username_field": name,
|
||||
"password_field": password.attrs["name"],
|
||||
"identity": "email" if wants_email else "username",
|
||||
"username_selector": f'input[name="{name}"]',
|
||||
"password_selector": f'input[name="{password.attrs["name"]}"]',
|
||||
"submit_selector": "button[type=submit]",
|
||||
}
|
||||
passwords = select_nodes(root, "input[type=password]")
|
||||
if passwords:
|
||||
return {
|
||||
"mode": "page",
|
||||
"url": base_url,
|
||||
"identity": "username",
|
||||
"username_selector": "input[type=email], input[type=text]",
|
||||
"password_selector": "input[type=password]",
|
||||
"submit_selector": "button[type=submit], input[type=submit]",
|
||||
}
|
||||
return None
|
||||
@@ -0,0 +1,262 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import statistics
|
||||
from dataclasses import dataclass, field
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from devplacepy.services.community.extract import (
|
||||
Node,
|
||||
json_or_none,
|
||||
parse_feed,
|
||||
parse_html,
|
||||
select_nodes,
|
||||
)
|
||||
from devplacepy.services.community.spec import (
|
||||
FEED_ATOM,
|
||||
FEED_HTML,
|
||||
FEED_JSON,
|
||||
FEED_PROBE_PATHS,
|
||||
FEED_RSS,
|
||||
LIMITS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("community.feeds")
|
||||
|
||||
ITEM_KEYS = re.compile(
|
||||
r"^(title|name|subject|headline|content|body|text|description|summary|id|uid|guid|url|link|"
|
||||
r"created_at|created|published|date|author|user|username|creator|content_html|excerpt)$",
|
||||
re.I,
|
||||
)
|
||||
ITEM_HINT = re.compile(r"post|thread|topic|item|entry|article|story|rant|card|result|row", re.I)
|
||||
ACTIVITY_LINK = re.compile(r"\b(latest|recent|new|activity|posts|threads|topics|discussions|feed)\b", re.I)
|
||||
LOGIN_WORDS = re.compile(r"(sign in|log in|login|password)", re.I)
|
||||
MIN_REPEAT = 3
|
||||
COMMENT_FEED = re.compile(r"/(comments?|repl(?:y|ies)|responses?)(?:[./_-][\w.]*)?/?$", re.I)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FeedCandidate:
|
||||
url: str
|
||||
type: str
|
||||
items: list = field(default_factory=list)
|
||||
document: object = None
|
||||
list_path: str | None = None
|
||||
item_selector: str | None = None
|
||||
login_required: bool = False
|
||||
render: bool = False
|
||||
|
||||
|
||||
def item_score(item) -> int:
|
||||
if not isinstance(item, dict):
|
||||
return 0
|
||||
return sum(1 for key in item if ITEM_KEYS.match(str(key).lstrip("@")))
|
||||
|
||||
|
||||
def find_item_list(document, path: str = "$", depth: int = 0):
|
||||
best = (0, None, None)
|
||||
if depth > 3:
|
||||
return best
|
||||
if isinstance(document, list):
|
||||
dicts = [entry for entry in document if isinstance(entry, dict)]
|
||||
if len(dicts) >= 2:
|
||||
score = statistics.median(item_score(entry) for entry in dicts[:8]) * min(len(dicts), 20)
|
||||
best = (score, path, document)
|
||||
return best
|
||||
if isinstance(document, dict):
|
||||
for key, value in document.items():
|
||||
if isinstance(value, (list, dict)):
|
||||
found = find_item_list(value, f"{path}.{key}", depth + 1)
|
||||
if found[0] > best[0]:
|
||||
best = found
|
||||
return best
|
||||
|
||||
|
||||
def find_item_selector(root: Node) -> tuple[str, list[Node]] | None:
|
||||
best_score = 0.0
|
||||
best: tuple[str, list[Node]] | None = None
|
||||
groups: dict[tuple, list[Node]] = {}
|
||||
for node in root.descendants():
|
||||
if node.tag in ("script", "style", "nav", "header", "footer", "select", "option", "a", "span", "td", "th"):
|
||||
continue
|
||||
key = (node.tag, tuple(sorted(node.classes())))
|
||||
groups.setdefault(key, []).append(node)
|
||||
for (tag, classes), nodes in groups.items():
|
||||
if len(nodes) < MIN_REPEAT:
|
||||
continue
|
||||
linked = [node for node in nodes if any(child.tag == "a" and child.attrs.get("href") for child in node.descendants())]
|
||||
if len(linked) < MIN_REPEAT:
|
||||
continue
|
||||
lengths = [len(node.text()) for node in linked]
|
||||
median = statistics.median(lengths)
|
||||
if median < 25:
|
||||
continue
|
||||
selector = tag + "".join(f".{name}" for name in classes)
|
||||
hint = 2.0 if ITEM_HINT.search(selector) else 1.0
|
||||
score = min(len(linked), 30) * min(median, 400) * hint
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best = (selector, linked)
|
||||
return best
|
||||
|
||||
|
||||
def classify_text(url: str, text: str) -> FeedCandidate | None:
|
||||
stripped = text.lstrip()
|
||||
if stripped.startswith(("<?xml", "<rss", "<feed", "<rdf")):
|
||||
try:
|
||||
kind, _, items = parse_feed(text)
|
||||
except Exception as exc:
|
||||
logger.info("feed %s looked like XML but did not parse: %s", url, exc)
|
||||
return None
|
||||
if len(items) >= 1:
|
||||
return FeedCandidate(url, FEED_RSS if kind == "rss" else FEED_ATOM, items=items)
|
||||
return None
|
||||
if stripped[:1] in "[{":
|
||||
document = json_or_none(text)
|
||||
if document is None:
|
||||
return None
|
||||
score, path, items = find_item_list(document)
|
||||
if path and score >= 4:
|
||||
return FeedCandidate(url, FEED_JSON, items=items, document=document, list_path=path)
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
COMMENT_HINT = re.compile(r"comment|repl(?:y|ies)|response|answer", re.I)
|
||||
|
||||
|
||||
def find_comment_selector(roots: list[Node]) -> str | None:
|
||||
tallies: dict[str, int] = {}
|
||||
for root in roots:
|
||||
for node in root.descendants():
|
||||
if node.tag in ("script", "style", "a", "span", "form", "textarea", "input", "button", "ul", "ol", "section", "nav"):
|
||||
continue
|
||||
hinted = sorted(name for name in node.classes() if COMMENT_HINT.search(name) and not re.search(r"form|count|list|wrap|section|header|input|box|area|title|link|button", name, re.I))
|
||||
if not hinted:
|
||||
continue
|
||||
selector = f"{node.tag}.{hinted[0]}"
|
||||
if len(node.text()) < 4:
|
||||
continue
|
||||
tallies[selector] = tallies.get(selector, 0) + 1
|
||||
ranked = sorted(tallies.items(), key=lambda entry: entry[1], reverse=True)
|
||||
return ranked[0][0] if ranked and ranked[0][1] >= 2 else None
|
||||
|
||||
|
||||
class FeedDetector:
|
||||
def __init__(self, fetcher, ai=None):
|
||||
self.fetcher = fetcher
|
||||
self.ai = ai
|
||||
|
||||
async def fetch(self, url: str):
|
||||
try:
|
||||
response = await self.fetcher.get(url)
|
||||
return response.status, await response.text()
|
||||
except Exception as exc:
|
||||
logger.warning("detect: %s unreachable: %s: %s", url, type(exc).__name__, exc)
|
||||
return 0, ""
|
||||
|
||||
async def detect(self, base_url: str) -> tuple[list[FeedCandidate], dict]:
|
||||
parts = urlsplit(base_url)
|
||||
origin = f"{parts.scheme}://{parts.netloc}"
|
||||
status, text = await self.fetch(base_url)
|
||||
context = {"status": status, "html": text, "login_required": status in (401, 403)}
|
||||
found: dict[str, FeedCandidate] = {}
|
||||
direct = classify_text(base_url, text) if text else None
|
||||
if direct:
|
||||
found[base_url] = direct
|
||||
root = parse_html(text) if text and not direct else None
|
||||
queue: list[str] = []
|
||||
if root is not None:
|
||||
queue.extend(self.advertised(root, base_url))
|
||||
queue.extend(self.activity_links(root, base_url))
|
||||
context["login_required"] = context["login_required"] or self.looks_like_login(root)
|
||||
for url in dict.fromkeys(queue):
|
||||
if len(found) < LIMITS.max_candidates and url not in found and not COMMENT_FEED.search(urlsplit(url).path):
|
||||
await self.consider(url, found, context)
|
||||
if not found:
|
||||
for path in FEED_PROBE_PATHS[: LIMITS.max_probe_paths]:
|
||||
await self.consider(urljoin(origin, path), found, context)
|
||||
if not found and root is not None:
|
||||
candidate = await self.html_with_ai(base_url, text, root)
|
||||
if candidate:
|
||||
found[base_url] = candidate
|
||||
if not found and not direct:
|
||||
rendered = await self.rendered(base_url)
|
||||
if rendered:
|
||||
context["html"] = rendered
|
||||
candidate = await self.html_with_ai(base_url, rendered, parse_html(rendered))
|
||||
if candidate:
|
||||
candidate.render = True
|
||||
logger.info("detect: %s only renders with JavaScript, using the browser", base_url)
|
||||
found[base_url] = candidate
|
||||
return sorted(found.values(), key=self.preference), context
|
||||
|
||||
async def consider(self, url: str, found: dict, context: dict) -> None:
|
||||
status, body = await self.fetch(url)
|
||||
if status == 401:
|
||||
context["login_required"] = True
|
||||
if status >= 400 or not body:
|
||||
return
|
||||
candidate = classify_text(url, body)
|
||||
if candidate is None and status < 300:
|
||||
candidate = self.html_candidate(url, body)
|
||||
if candidate:
|
||||
logger.info("detect: %s feed at %s (%d sample items)", candidate.type, url, len(candidate.items))
|
||||
found[url] = candidate
|
||||
|
||||
async def rendered(self, url: str) -> str:
|
||||
try:
|
||||
status, html = await self.fetcher.render_html(url)
|
||||
except Exception as exc:
|
||||
logger.info("detect: rendering %s failed: %s: %s", url, type(exc).__name__, exc)
|
||||
return ""
|
||||
return html if status < 400 else ""
|
||||
|
||||
async def html_with_ai(self, url: str, text: str, root: Node) -> FeedCandidate | None:
|
||||
candidate = self.html_candidate(url, text)
|
||||
if candidate or self.ai is None:
|
||||
return candidate
|
||||
selector = await self.ai.item_selector(root)
|
||||
if not selector:
|
||||
return None
|
||||
return FeedCandidate(url, FEED_HTML, items=select_nodes(root, selector), document=root, item_selector=selector)
|
||||
|
||||
def preference(self, candidate: FeedCandidate) -> int:
|
||||
order = {FEED_JSON: 0, FEED_ATOM: 1, FEED_RSS: 2, FEED_HTML: 3}
|
||||
return order[candidate.type]
|
||||
|
||||
def advertised(self, root: Node, base_url: str) -> list[str]:
|
||||
urls = []
|
||||
for node in select_nodes(root, "link[rel=alternate]"):
|
||||
kind = node.attrs.get("type", "").lower()
|
||||
if any(marker in kind for marker in ("rss", "atom", "json")) and node.attrs.get("href"):
|
||||
urls.append(urljoin(base_url, node.attrs["href"]))
|
||||
return urls
|
||||
|
||||
def activity_links(self, root: Node, base_url: str) -> list[str]:
|
||||
urls = []
|
||||
host = urlsplit(base_url).hostname
|
||||
for node in select_nodes(root, "a[href]"):
|
||||
label = node.text()[:40]
|
||||
target = urljoin(base_url, node.attrs["href"])
|
||||
if urlsplit(target).hostname == host and ACTIVITY_LINK.search(label) and target != base_url:
|
||||
urls.append(target)
|
||||
return urls[: LIMITS.max_candidates]
|
||||
|
||||
def html_candidate(self, url: str, text: str) -> FeedCandidate | None:
|
||||
if "<" not in text[:2000]:
|
||||
return None
|
||||
root = parse_html(text)
|
||||
found = find_item_selector(root)
|
||||
if found is None:
|
||||
return None
|
||||
selector, nodes = found
|
||||
return FeedCandidate(url, FEED_HTML, items=nodes, document=root, item_selector=selector)
|
||||
|
||||
def looks_like_login(self, root: Node) -> bool:
|
||||
if select_nodes(root, "input[type=password]") and not find_item_selector(root):
|
||||
return True
|
||||
return bool(LOGIN_WORDS.search(root.text()[:400])) and not find_item_selector(root)
|
||||
@@ -0,0 +1,153 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy.net_guard import guard_public_url, guarded_async_client
|
||||
from devplacepy.services.community.spec import LIMITS
|
||||
|
||||
logger = logging.getLogger("community.fetch")
|
||||
|
||||
RENDER_SETTLE_MS = 800
|
||||
|
||||
|
||||
class SharedBrowser:
|
||||
instance = None
|
||||
loop = None
|
||||
lock: asyncio.Lock | None = None
|
||||
|
||||
@classmethod
|
||||
async def acquire(cls):
|
||||
running = asyncio.get_running_loop()
|
||||
if cls.instance is not None and cls.loop is running:
|
||||
return cls.instance
|
||||
if cls.instance is not None:
|
||||
logger.info("browser: event loop changed, starting a new Chromium")
|
||||
cls.instance = None
|
||||
from stealthii import Stealthii
|
||||
|
||||
browser = Stealthii(request_timeout=LIMITS.timeout_seconds)
|
||||
await browser.start()
|
||||
cls.instance, cls.loop = browser, running
|
||||
logger.info("browser: stealth Chromium started")
|
||||
return browser
|
||||
|
||||
@classmethod
|
||||
async def close(cls) -> None:
|
||||
if cls.instance is not None:
|
||||
try:
|
||||
await cls.instance.shutdown()
|
||||
finally:
|
||||
cls.instance = None
|
||||
cls.loop = None
|
||||
|
||||
|
||||
class GuardedPage:
|
||||
def __init__(self, page):
|
||||
self.page = page
|
||||
|
||||
async def goto(self, url, **kwargs):
|
||||
await guard_public_url(url)
|
||||
return await self.page.goto(url, **kwargs)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.page, name)
|
||||
|
||||
|
||||
class FetchResponse:
|
||||
def __init__(self, response: httpx.Response):
|
||||
self.status = response.status_code
|
||||
self.url = str(response.url)
|
||||
self.headers = response.headers
|
||||
self._response = response
|
||||
|
||||
async def text(self) -> str:
|
||||
return self._response.text
|
||||
|
||||
async def read(self) -> bytes:
|
||||
return self._response.content
|
||||
|
||||
async def json(self):
|
||||
return self._response.json()
|
||||
|
||||
|
||||
class Fetcher:
|
||||
def __init__(self):
|
||||
self.browser = None
|
||||
self.client: httpx.AsyncClient | None = None
|
||||
self.session: str | None = None
|
||||
self.last_request: dict[str, float] = {}
|
||||
|
||||
async def polite(self, url: str) -> None:
|
||||
host = urlsplit(url).hostname or ""
|
||||
wait = LIMITS.delay_seconds - (time.monotonic() - self.last_request.get(host, 0.0))
|
||||
if wait > 0:
|
||||
await asyncio.sleep(wait)
|
||||
self.last_request[host] = time.monotonic()
|
||||
|
||||
async def start(self) -> "Fetcher":
|
||||
try:
|
||||
self.browser = await SharedBrowser.acquire()
|
||||
except Exception as exc:
|
||||
logger.warning("browser: stealthii unavailable (%s: %s); falling back to plain HTTP, no JavaScript rendering", type(exc).__name__, exc)
|
||||
self.client = guarded_async_client(follow_redirects=True, max_redirects=5, timeout=LIMITS.timeout_seconds)
|
||||
return self
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
if self.client is not None:
|
||||
await self.client.aclose()
|
||||
self.client = None
|
||||
|
||||
async def __aenter__(self) -> "Fetcher":
|
||||
return await self.start()
|
||||
|
||||
async def __aexit__(self, *exc) -> None:
|
||||
await self.shutdown()
|
||||
|
||||
async def ready(self) -> None:
|
||||
if self.browser is None and self.client is None:
|
||||
await self.start()
|
||||
|
||||
async def get(self, url: str, params: dict | None = None, timeout: float | None = None, session: str | None = None):
|
||||
await self.ready()
|
||||
await guard_public_url(url)
|
||||
await self.polite(url)
|
||||
logger.info("GET %s params=%s", url, params or {})
|
||||
if self.browser is not None:
|
||||
response = await self.browser.get(url, params=params, timeout=timeout or LIMITS.timeout_seconds, session=session or self.session)
|
||||
logger.info("GET %s -> %s", url, response.status)
|
||||
return response
|
||||
raw = await self.client.get(url, params=params, timeout=timeout or LIMITS.timeout_seconds)
|
||||
logger.info("GET %s -> %s", url, raw.status_code)
|
||||
return FetchResponse(raw)
|
||||
|
||||
async def post(self, url: str, data: dict | None = None, timeout: float | None = None, session: str | None = None):
|
||||
await self.ready()
|
||||
await guard_public_url(url)
|
||||
await self.polite(url)
|
||||
logger.info("POST %s fields=%s", url, sorted((data or {}).keys()))
|
||||
if self.browser is not None:
|
||||
return await self.browser.post(url, data=data, timeout=timeout or LIMITS.timeout_seconds, session=session or self.session)
|
||||
return FetchResponse(await self.client.post(url, data=data, timeout=timeout or LIMITS.timeout_seconds))
|
||||
|
||||
@asynccontextmanager
|
||||
async def page(self, session: str | None = None):
|
||||
await self.ready()
|
||||
if self.browser is None:
|
||||
raise RuntimeError("JavaScript rendering needs stealthii and its Chromium")
|
||||
async with self.browser.page(session=session or self.session) as page:
|
||||
yield GuardedPage(page)
|
||||
|
||||
async def render_html(self, url: str, session: str | None = None) -> tuple[int, str]:
|
||||
async with self.page(session=session) as page:
|
||||
response = await page.goto(url, wait_until="domcontentloaded")
|
||||
await page.wait_for_timeout(RENDER_SETTLE_MS)
|
||||
logger.info("RENDER %s -> %s", url, response.status if response else "?")
|
||||
return (response.status if response else 200), await page.content()
|
||||
@@ -0,0 +1,40 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
from devplacepy import config
|
||||
from devplacepy.services.community.generator import CodeGenerator
|
||||
|
||||
|
||||
class CommunityFiles:
|
||||
def __init__(self, root: Path | None = None):
|
||||
self.root = root
|
||||
|
||||
def directory(self, slug: str) -> Path:
|
||||
return (self.root or config.COMMUNITIES_DIR) / slug
|
||||
|
||||
def collectors_path(self, slug: str) -> Path:
|
||||
return self.directory(slug) / "collectors.py"
|
||||
|
||||
def write(self, slug: str, schema: dict) -> Path:
|
||||
return CodeGenerator(schema).write(self.directory(slug))
|
||||
|
||||
def load(self, slug: str, schema: dict) -> ModuleType:
|
||||
path = self.collectors_path(slug)
|
||||
if not path.exists():
|
||||
self.write(slug, schema)
|
||||
spec = importlib.util.spec_from_file_location(f"community_collectors_{slug.replace('-', '_')}", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
def read(self, slug: str) -> str:
|
||||
path = self.collectors_path(slug)
|
||||
return path.read_text() if path.exists() else ""
|
||||
|
||||
|
||||
files = CommunityFiles()
|
||||
@@ -0,0 +1,117 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.services.community import extract, runtime
|
||||
|
||||
PACKAGE_IMPORT = re.compile(r"^from devplacepy\.services\.community\.extract import \([^)]*\)\n", re.M)
|
||||
FUTURE_IMPORT = re.compile(r"^from __future__ import annotations\n", re.M)
|
||||
HEADER_LINE = re.compile(r"^# retoor <[^>]*>\n", re.M)
|
||||
|
||||
ENTRYPOINT = '''
|
||||
|
||||
async def collect_all(stealth, state=None, only=None, sleep=None, credentials=None):
|
||||
collector = Collector(stealth, SCHEMA, state, sleep, credentials)
|
||||
return await collector.run(only)
|
||||
{wrappers}
|
||||
|
||||
async def main():
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
try:
|
||||
from stealthii import Stealthii
|
||||
except ImportError:
|
||||
sys.exit("stealthii is required to run this file directly: pip install stealthii")
|
||||
here = Path(__file__).resolve().parent
|
||||
state_file = here / "last_run.json"
|
||||
state = json.loads(state_file.read_text()) if state_file.exists() else None
|
||||
stealth = Stealthii()
|
||||
await stealth.start()
|
||||
try:
|
||||
credentials = {{
|
||||
"username": os.environ.get("COMMUNITY_USERNAME", ""),
|
||||
"email": os.environ.get("COMMUNITY_EMAIL", ""),
|
||||
"password": os.environ.get("COMMUNITY_PASSWORD", ""),
|
||||
}}
|
||||
result = await collect_all(stealth, state, credentials=credentials)
|
||||
finally:
|
||||
await stealth.shutdown()
|
||||
state_file.write_text(json.dumps(result["state"], indent=2))
|
||||
(here / "collected.json").write_text(json.dumps(result["records"], indent=2))
|
||||
counts = {{kind: len(rows) for kind, rows in result["records"].items()}}
|
||||
print(json.dumps({{"ok": result["ok"], "counts": counts, "errors": result["errors"]}}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
'''
|
||||
|
||||
WRAPPER = '''
|
||||
|
||||
async def collect_{name}(stealth, state=None, sleep=None, credentials=None):
|
||||
# {kind} feed {url}
|
||||
# post fields: {fields}
|
||||
# comments: {comments}
|
||||
# pagination: {pagination}
|
||||
return await collect_all(stealth, state, only="{feed_id}", sleep=sleep, credentials=credentials)
|
||||
'''
|
||||
|
||||
|
||||
class CodeGenerator:
|
||||
def __init__(self, schema: dict):
|
||||
self.schema = schema
|
||||
|
||||
@staticmethod
|
||||
def module_source(module) -> str:
|
||||
text = Path(module.__file__).read_text()
|
||||
text = FUTURE_IMPORT.sub("", text)
|
||||
text = HEADER_LINE.sub("", text)
|
||||
return PACKAGE_IMPORT.sub("", text).strip("\n")
|
||||
|
||||
def wrappers(self) -> str:
|
||||
parts = []
|
||||
for feed in self.schema["feeds"]:
|
||||
parts.append(
|
||||
WRAPPER.format(
|
||||
name=re.sub(r"\W", "_", feed["id"]),
|
||||
kind=feed["type"],
|
||||
url=feed["url"],
|
||||
fields=json.dumps(feed["post"]["fields"]),
|
||||
comments=(feed.get("comments") or {}).get("mode", "none"),
|
||||
pagination=(feed.get("pagination") or {}).get("style", "none"),
|
||||
feed_id=feed["id"],
|
||||
)
|
||||
)
|
||||
return "".join(parts)
|
||||
|
||||
def render(self) -> str:
|
||||
schema_json = json.dumps(self.schema, indent=2, ensure_ascii=False).replace("'", "\\u0027")
|
||||
header = (
|
||||
"# Generated by DevPlace community discovery. Community: "
|
||||
f"{self.schema.get('community')} (origin {self.schema.get('origin')}).\n"
|
||||
"# Self-contained: standard library plus stealthii when run directly.\n"
|
||||
"# A login-protected community reads COMMUNITY_USERNAME / COMMUNITY_EMAIL / COMMUNITY_PASSWORD from the environment.\n"
|
||||
"# Adjust SCHEMA below to change a selector or path; collect_all() reads it at runtime.\n"
|
||||
"from __future__ import annotations\n\n"
|
||||
)
|
||||
schema_block = f"\n\nSCHEMA = json.loads(r'''{schema_json}''')\n"
|
||||
return (
|
||||
header
|
||||
+ self.module_source(extract)
|
||||
+ "\n\n"
|
||||
+ self.module_source(runtime)
|
||||
+ schema_block
|
||||
+ ENTRYPOINT.format(wrappers=self.wrappers())
|
||||
)
|
||||
|
||||
def write(self, directory: Path) -> Path:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
target = directory / "collectors.py"
|
||||
target.write_text(self.render())
|
||||
(directory / "schema.json").write_text(json.dumps(self.schema, indent=2, ensure_ascii=False))
|
||||
return target
|
||||
@@ -0,0 +1,581 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import statistics
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from devplacepy.services.community.extract import (
|
||||
NOW_TOKEN,
|
||||
Node,
|
||||
field_text,
|
||||
html_field,
|
||||
json_or_none,
|
||||
leaf_paths,
|
||||
normalize_instant,
|
||||
parse_html,
|
||||
scalar_text,
|
||||
select,
|
||||
select_nodes,
|
||||
strip_tags,
|
||||
text_at,
|
||||
)
|
||||
from devplacepy.services.community.feeds import FeedCandidate, find_comment_selector, find_item_list
|
||||
from devplacepy.services.community.spec import (
|
||||
FEED_HTML,
|
||||
FEED_JSON,
|
||||
LIMITS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("community.learn")
|
||||
|
||||
PATTERNS = {
|
||||
"id": r"^(id|uid|guid|post_id|postid|uuid|slug|key|@id|@rdf:about)$",
|
||||
"url": r"^(url|link|permalink|href|@href|html_url|canonical_url|web_url|share_url)$",
|
||||
"title": r"^(title|headline|subject|topic_title|name)$",
|
||||
"content": r"^(content_html|content|body_html|body|text|selftext|message|encoded|raw|cooked|description|summary|excerpt)$",
|
||||
"created_at": r"^(created_at|created|createdat|published|published_at|pubdate|date|timestamp|created_utc|posted_at|time|dateadded)$",
|
||||
"updated_at": r"^(updated_at|updated|modified|edited_at|last_edited|bumped_at|last_activity)$",
|
||||
"parent_id": r"^(parent_id|parent|in_reply_to|reply_to|parent_comment_id|parentid)$",
|
||||
}
|
||||
AUTHOR_KEYS = re.compile(r"^(author|user|creator|poster|by|owner|dc:creator|created_by|member|profile)$", re.I)
|
||||
AUTHOR_NAME = re.compile(r"^(username|user_name|login|handle|name|display_name|screen_name|nickname|nick)$", re.I)
|
||||
AUTHOR_ID = re.compile(r"^(id|uid|user_id|userid|uuid)$", re.I)
|
||||
COMMENT_KEYS = re.compile(r"^(comments|replies|children|answers|responses|posts)$", re.I)
|
||||
COMMENT_URL_KEYS = re.compile(r"^(comments_url|comment_url|comments_link|replies_url|comments_href)$", re.I)
|
||||
REPLY_KEYS = re.compile(r"^(replies|children|answers|responses)$", re.I)
|
||||
NEXT_KEYS = re.compile(r"^(next|next_page|next_url|next_page_url|nextpage)$", re.I)
|
||||
CURSOR_KEYS = re.compile(r"^(next_cursor|cursor|after|next_token|nextcursor|continuation)$", re.I)
|
||||
PAGE_PARAMS = ("page", "p", "paged")
|
||||
USER_COUNT = re.compile(r"([\d][\d,.\s]{0,12}\d|\d)\s*(?:\+\s*)?(members|users|developers|registered|people)\b", re.I)
|
||||
RULE_PATHS = ("/rules", "/about", "/faq", "/guidelines", "/terms")
|
||||
COMMENT_PROBES = ("{feed}/{id}/comments", "{feed}/{id}", "/api/posts/{id}/comments", "/api/comments?post_id={id}")
|
||||
|
||||
|
||||
def last_segment(path: str) -> str:
|
||||
return re.split(r"[.\[]", path.rstrip("]"))[-1] if path else ""
|
||||
|
||||
|
||||
class FieldInferrer:
|
||||
def __init__(self, samples: list, coverage: float | None = None):
|
||||
self.samples = samples
|
||||
self.coverage = LIMITS.min_field_coverage if coverage is None else coverage
|
||||
self.leaves = [leaf_paths(sample) for sample in samples]
|
||||
self.paths = list(dict.fromkeys(path for leaf in self.leaves for path in leaf))
|
||||
|
||||
def values(self, path: str) -> list[str]:
|
||||
return [text_at(sample, path) for sample in self.samples]
|
||||
|
||||
def coverage_of(self, path: str, validator=None) -> float:
|
||||
values = self.values(path)
|
||||
good = [value for value in values if value and (validator is None or validator(value))]
|
||||
return len(good) / len(self.samples) if self.samples else 0.0
|
||||
|
||||
def candidates(self, pattern: str) -> list[str]:
|
||||
regex = re.compile(pattern, re.I)
|
||||
return [path for path in self.paths if regex.match(last_segment(path).lstrip("@")) or regex.match(last_segment(path))]
|
||||
|
||||
def pick_all(self, name: str, validator=None, longest: bool = False) -> list[str]:
|
||||
scored = []
|
||||
for index, path in enumerate(self.candidates(PATTERNS[name])):
|
||||
cover = self.coverage_of(path, validator)
|
||||
if cover < self.coverage:
|
||||
continue
|
||||
median = statistics.median(len(value) for value in self.values(path)) if longest else 0
|
||||
scored.append((-median if longest else 0, index, path))
|
||||
scored.sort()
|
||||
return [path for _, _, path in scored]
|
||||
|
||||
def post_fields(self) -> dict:
|
||||
valid_date = lambda value: normalize_instant(value) is not None
|
||||
fields = {
|
||||
"id": self.pick_all("id") or self.pick_all("url"),
|
||||
"url": self.pick_all("url"),
|
||||
"title": self.pick_all("title")[:1],
|
||||
"content": self.pick_all("content", longest=True)[:3],
|
||||
"created_at": self.pick_all("created_at", valid_date)[:1],
|
||||
"updated_at": self.pick_all("updated_at", valid_date)[:1],
|
||||
}
|
||||
if not fields["created_at"]:
|
||||
fields["created_at"] = [NOW_TOKEN]
|
||||
return {key: value[:1] if key in ("id", "url") else value for key, value in fields.items() if value}
|
||||
|
||||
def author_spec(self) -> dict:
|
||||
regex_keys = [path for path in self.paths if AUTHOR_KEYS.match(last_segment(path))]
|
||||
found: dict = {}
|
||||
objects = [path for path in self.paths if AUTHOR_NAME.match(last_segment(path)) and any(AUTHOR_KEYS.match(part) for part in re.split(r"[.\[\]]", path)[:-1])]
|
||||
objects.sort(key=lambda path: (len(path), path))
|
||||
for path in objects:
|
||||
if self.coverage_of(path) >= self.coverage:
|
||||
found["username"] = [path]
|
||||
base = path.rsplit(".", 1)[0]
|
||||
for other in self.paths:
|
||||
if other.startswith(base + ".") and AUTHOR_ID.match(last_segment(other)) and self.coverage_of(other) >= self.coverage:
|
||||
found["id"] = [other]
|
||||
break
|
||||
return {"fields": found}
|
||||
for path in regex_keys:
|
||||
if self.coverage_of(path) >= self.coverage:
|
||||
return {"fields": {"username": [path]}}
|
||||
flat = [path for path in self.paths if re.match(r"^\$\.(username|author_name|user|author_username)$", path)]
|
||||
for path in flat:
|
||||
if self.coverage_of(path) >= self.coverage:
|
||||
return {"fields": {"username": [path]}}
|
||||
return {"fields": {}}
|
||||
|
||||
def list_paths(self, pattern) -> list[str]:
|
||||
found = []
|
||||
for path in self.paths:
|
||||
if "[0]" not in path:
|
||||
continue
|
||||
candidate = path.split("[0]")[0]
|
||||
if pattern.match(last_segment(candidate)) and candidate not in found:
|
||||
found.append(candidate)
|
||||
return found
|
||||
|
||||
def comment_lists(self) -> list[str]:
|
||||
return [
|
||||
path
|
||||
for path in self.list_paths(COMMENT_KEYS)
|
||||
if any(isinstance(match, list) and match and isinstance(match[0], dict) for sample in self.samples for match in select(sample, path))
|
||||
]
|
||||
|
||||
|
||||
class HtmlInferrer:
|
||||
def __init__(self, nodes: list[Node], base_url: str):
|
||||
self.nodes = nodes
|
||||
self.base_url = base_url
|
||||
|
||||
def share(self, spec: str) -> float:
|
||||
hits = sum(1 for node in self.nodes if field_text(node, spec))
|
||||
return hits / len(self.nodes) if self.nodes else 0.0
|
||||
|
||||
def first(self, specs: list[str]) -> str | None:
|
||||
for spec in specs:
|
||||
if self.share(spec) >= LIMITS.min_field_coverage:
|
||||
return spec
|
||||
return None
|
||||
|
||||
def post_fields(self) -> dict:
|
||||
link = self.first(["h1 a::attr(href)", "h2 a::attr(href)", "h3 a::attr(href)", "a.title::attr(href)", "a[href]::attr(href)"])
|
||||
title = self.first(["h1", "h2", "h3", "a.title", "a[href]"])
|
||||
content = self.first(["p", ".content", ".body", ".excerpt", ".summary", "::text"])
|
||||
created = self.first(["time::attr(datetime)", "time", "[datetime]::attr(datetime)"])
|
||||
fields: dict = {}
|
||||
if link:
|
||||
fields["id"] = [link]
|
||||
fields["url"] = [link]
|
||||
if title:
|
||||
fields["title"] = [title]
|
||||
if content:
|
||||
fields["content"] = [content, "::text"] if content != "::text" else [content]
|
||||
fields["created_at"] = [created] if created else [NOW_TOKEN]
|
||||
return fields
|
||||
|
||||
@staticmethod
|
||||
def weak(post: dict, author: dict) -> bool:
|
||||
return (
|
||||
not author.get("fields")
|
||||
or "title" not in post
|
||||
or post.get("content") in (None, ["::text"])
|
||||
or post.get("created_at") == [NOW_TOKEN]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def merge(post: dict, author: dict, proposal: dict) -> tuple[dict, dict]:
|
||||
post, author = dict(post), dict(author)
|
||||
if proposal.get("title"):
|
||||
post["title"] = [proposal["title"]]
|
||||
if proposal.get("link"):
|
||||
post["id"] = post["url"] = [proposal["link"]]
|
||||
if proposal.get("content"):
|
||||
post["content"] = [proposal["content"], "::text"]
|
||||
if proposal.get("created_at"):
|
||||
post["created_at"] = [proposal["created_at"]]
|
||||
if proposal.get("author") and not author.get("fields"):
|
||||
author = {"fields": {"username": [proposal["author"]]}}
|
||||
return post, author
|
||||
|
||||
def author_spec(self) -> dict:
|
||||
name = self.first([".author", ".username", ".user", "a[rel=author]", "[itemprop=author]", ".byline"])
|
||||
return {"fields": {"username": [name]} if name else {}}
|
||||
|
||||
|
||||
class PaginationProbe:
|
||||
def __init__(self, fetcher):
|
||||
self.fetcher = fetcher
|
||||
|
||||
def from_document(self, candidate: FeedCandidate) -> dict | None:
|
||||
document = candidate.document
|
||||
if isinstance(candidate.document, Node):
|
||||
for spec in ("a[rel=next]::attr(href)", "link[rel=next]::attr(href)", "a.next::attr(href)", ".pagination .next a::attr(href)"):
|
||||
if html_field(candidate.document, spec):
|
||||
return {"style": "next_link", "next": spec}
|
||||
return None
|
||||
if isinstance(document, dict):
|
||||
leaves = leaf_paths(document)
|
||||
for path, value in leaves.items():
|
||||
if NEXT_KEYS.match(last_segment(path)) and isinstance(value, str) and value.startswith(("http", "/")):
|
||||
return {"style": "next_link", "next": path}
|
||||
for path, value in leaves.items():
|
||||
if CURSOR_KEYS.match(last_segment(path)) and value:
|
||||
return {"style": "cursor", "cursor_path": path, "param": last_segment(path).replace("next_", "")}
|
||||
return None
|
||||
|
||||
def signature(self, items: list) -> str:
|
||||
return "|".join(
|
||||
(item.text()[:60] if isinstance(item, Node) else scalar_text(select(item, "$.id")) or str(item)[:60])
|
||||
for item in items[:3]
|
||||
)
|
||||
|
||||
async def probe(self, candidate: FeedCandidate, load_items) -> dict:
|
||||
declared = self.from_document(candidate)
|
||||
if declared:
|
||||
return declared
|
||||
baseline = self.signature(candidate.items)
|
||||
feed_like = candidate.type in ("rss", "atom")
|
||||
for param in PAGE_PARAMS[:1] if feed_like else PAGE_PARAMS:
|
||||
try:
|
||||
other = await load_items(candidate, {param: 2})
|
||||
except Exception as exc:
|
||||
logger.info("pagination probe %s=2 failed: %s", param, exc)
|
||||
continue
|
||||
if other and self.signature(other) != baseline:
|
||||
start = 1
|
||||
return {"style": "page", "param": param, "start": start}
|
||||
per_page = len(candidate.items)
|
||||
if per_page and not feed_like:
|
||||
for param in ("offset", "skip"):
|
||||
try:
|
||||
other = await load_items(candidate, {param: per_page})
|
||||
except Exception:
|
||||
continue
|
||||
if other and self.signature(other) != baseline:
|
||||
return {"style": "offset", "param": param, "per_page": per_page}
|
||||
return {"style": "none"}
|
||||
|
||||
|
||||
class CommentProbe:
|
||||
def __init__(self, fetcher):
|
||||
self.fetcher = fetcher
|
||||
|
||||
def comment_fields(self, entries: list) -> dict:
|
||||
inferrer = FieldInferrer(entries)
|
||||
fields = {
|
||||
"id": inferrer.pick_all("id")[:1],
|
||||
"content": inferrer.pick_all("content", longest=True)[:2],
|
||||
"created_at": (inferrer.pick_all("created_at", lambda v: normalize_instant(v) is not None)[:1] or [NOW_TOKEN]),
|
||||
"parent_id": inferrer.pick_all("parent_id")[:1],
|
||||
"updated_at": inferrer.pick_all("updated_at", lambda v: normalize_instant(v) is not None)[:1],
|
||||
}
|
||||
spec = {"fields": {key: value for key, value in fields.items() if value}, "author": inferrer.author_spec()}
|
||||
for path in inferrer.list_paths(REPLY_KEYS):
|
||||
if any(isinstance(match, list) and match for sample in entries for match in select(sample, path)):
|
||||
spec["replies_path"] = path
|
||||
break
|
||||
return spec
|
||||
|
||||
def usable(self, spec: dict) -> bool:
|
||||
return "id" in spec["fields"] and "content" in spec["fields"]
|
||||
|
||||
async def learn(self, candidate: FeedCandidate, post_fields: dict, inferrer: FieldInferrer | None) -> dict:
|
||||
if candidate.type != FEED_JSON or inferrer is None or not candidate.items:
|
||||
return {"mode": "none", "reason": f"comments are not learned for {candidate.type} feeds"}
|
||||
for path in inferrer.comment_lists():
|
||||
entries = [entry for sample in candidate.items for match in select(sample, path) if isinstance(match, list) for entry in match if isinstance(entry, dict)]
|
||||
if len(entries) >= 1:
|
||||
spec = self.comment_fields(entries)
|
||||
if self.usable(spec):
|
||||
return {"mode": "inline", "list_path": path, **spec}
|
||||
id_path = (post_fields.get("id") or [None])[0]
|
||||
sample = candidate.items[0]
|
||||
identifier = text_at(sample, id_path) if id_path else ""
|
||||
for path in inferrer.paths:
|
||||
if COMMENT_URL_KEYS.match(last_segment(path)):
|
||||
target = text_at(sample, path)
|
||||
if target and identifier and identifier in target:
|
||||
template = urljoin(candidate.url, target.replace(identifier, "{id}"))
|
||||
result = await self.fetch_comments(template, identifier, candidate)
|
||||
if result:
|
||||
return result
|
||||
if not identifier:
|
||||
return {"mode": "none", "reason": "posts expose no id to build a comments address"}
|
||||
feed = candidate.url.split("?")[0].rstrip("/")
|
||||
for probe in COMMENT_PROBES[: LIMITS.max_probe_paths]:
|
||||
template = probe.replace("{feed}", feed)
|
||||
template = template if template.startswith("http") else urljoin(candidate.url, template)
|
||||
result = await self.fetch_comments(template, identifier, candidate)
|
||||
if result:
|
||||
return result
|
||||
return {"mode": "none", "reason": "no comments source found"}
|
||||
|
||||
async def fetch_comments(self, template: str, identifier: str, candidate: FeedCandidate) -> dict | None:
|
||||
try:
|
||||
response = await self.fetcher.get(template.replace("{id}", identifier))
|
||||
if response.status >= 400:
|
||||
return None
|
||||
document = json_or_none(await response.text())
|
||||
except Exception as exc:
|
||||
logger.info("comments probe %s failed: %s", template, exc)
|
||||
return None
|
||||
if document is None:
|
||||
return None
|
||||
score, path, entries = find_item_list(document)
|
||||
if not path or not entries:
|
||||
return None
|
||||
spec = self.comment_fields([entry for entry in entries if isinstance(entry, dict)])
|
||||
if not self.usable(spec):
|
||||
return None
|
||||
return {"mode": "url", "url_template": template, "list_path": path, **spec}
|
||||
|
||||
|
||||
class PageProbe:
|
||||
CONTENT_SELECTORS = ("article", "[itemprop=articleBody]", ".post-content", ".entry-content", ".post-body", ".markdown-body", ".content", "main")
|
||||
BODY_SELECTORS = (".comment-body", ".comment-content", ".comment-text", ".comment__body", ".body", ".content", ".text", "p")
|
||||
TIME_SELECTORS = ("time::attr(datetime)", "[datetime]::attr(datetime)", "time", ".date", ".timestamp")
|
||||
AUTHOR_SELECTORS = (".comment-author", ".author", ".username", ".user", "a[rel=author]", "[itemprop=author]", ".fn", ".byline")
|
||||
|
||||
def __init__(self, fetcher, ai=None):
|
||||
self.fetcher = fetcher
|
||||
self.ai = ai
|
||||
|
||||
async def load(self, link: str, render: bool):
|
||||
try:
|
||||
if render:
|
||||
status, text = await self.fetcher.render_html(link)
|
||||
else:
|
||||
response = await self.fetcher.get(link)
|
||||
status, text = response.status, await response.text()
|
||||
except Exception as exc:
|
||||
logger.info("pages: %s unreachable: %s: %s", link, type(exc).__name__, exc)
|
||||
return None
|
||||
return parse_html(text) if status < 400 and text else None
|
||||
|
||||
def links(self, candidate: FeedCandidate, post: dict) -> list[str]:
|
||||
found = []
|
||||
for item in candidate.items:
|
||||
target = field_text(item, post.get("url"))
|
||||
if target:
|
||||
absolute = urljoin(candidate.url, target)
|
||||
if absolute not in found:
|
||||
found.append(absolute)
|
||||
if len(found) >= LIMITS.sample_comment_posts + 1:
|
||||
break
|
||||
return found
|
||||
|
||||
@staticmethod
|
||||
def first_spec(nodes: list[Node], specs: tuple[str, ...], minimum: int = 0) -> str | None:
|
||||
for spec in specs:
|
||||
hits = [field_text(node, spec) for node in nodes]
|
||||
if sum(1 for hit in hits if len(hit) > minimum) / max(len(nodes), 1) >= LIMITS.min_field_coverage:
|
||||
return spec
|
||||
return None
|
||||
|
||||
def comment_spec(self, roots: list[Node], selector: str, render: bool) -> dict | None:
|
||||
nodes = [node for root in roots for node in select_nodes(root, selector)]
|
||||
if not nodes:
|
||||
return None
|
||||
fields: dict = {}
|
||||
content = self.first_spec(nodes, self.BODY_SELECTORS, 3)
|
||||
if not content:
|
||||
return None
|
||||
fields["content"] = [content]
|
||||
for name, specs in (("id", ("::attr(id)", "::attr(data-id)")), ("created_at", self.TIME_SELECTORS)):
|
||||
spec = self.first_spec(nodes, specs)
|
||||
if spec:
|
||||
fields[name] = [spec]
|
||||
author = self.first_spec(nodes, self.AUTHOR_SELECTORS)
|
||||
return {
|
||||
"mode": "page",
|
||||
"item_selector": selector,
|
||||
"fields": fields,
|
||||
"author": {"fields": {"username": [author]} if author else {}},
|
||||
"render": render,
|
||||
}
|
||||
|
||||
def ai_comment_spec(self, proposal: dict, render: bool) -> dict:
|
||||
fields = {"content": [proposal["content"]]}
|
||||
for name in ("id", "created_at"):
|
||||
if proposal.get(name):
|
||||
fields[name] = [proposal[name]]
|
||||
return {
|
||||
"mode": "page",
|
||||
"item_selector": proposal["item_selector"],
|
||||
"fields": fields,
|
||||
"author": {"fields": {"username": [proposal["author"]]} if proposal.get("author") else {}},
|
||||
"render": render,
|
||||
}
|
||||
|
||||
async def learn(self, candidate: FeedCandidate, post: dict) -> tuple[dict, dict | None]:
|
||||
none = {"mode": "none", "reason": "no comments found on the post pages"}
|
||||
if not post.get("url"):
|
||||
return {"mode": "none", "reason": "posts expose no address to open"}, None
|
||||
links = self.links(candidate, post)
|
||||
render = candidate.render
|
||||
roots = [root for root in [await self.load(link, render) for link in links[: LIMITS.sample_comment_posts]] if root is not None]
|
||||
if not roots:
|
||||
return {"mode": "none", "reason": "post pages could not be opened"}, None
|
||||
selector = find_comment_selector(roots)
|
||||
if not selector and not render:
|
||||
rendered = [root for root in [await self.load(link, True) for link in links[: LIMITS.sample_comment_posts]] if root is not None]
|
||||
selector = find_comment_selector(rendered)
|
||||
if selector:
|
||||
roots, render = rendered, True
|
||||
logger.info("pages: comments only appear after JavaScript rendering")
|
||||
comments = none
|
||||
if selector:
|
||||
comments = self.comment_spec(roots, selector, render) or none
|
||||
elif self.ai is not None:
|
||||
proposal = await self.ai.comment_structure(roots[0])
|
||||
if proposal:
|
||||
comments = self.ai_comment_spec(proposal, render)
|
||||
detail = await self.detail(candidate, post, roots, render)
|
||||
if comments["mode"] == "page":
|
||||
logger.info("pages: comments via %s (%s)", comments["item_selector"], sorted(comments["fields"]))
|
||||
return comments, detail
|
||||
|
||||
async def detail(self, candidate: FeedCandidate, post: dict, roots: list[Node], render: bool) -> dict | None:
|
||||
listed = statistics.median(len(field_text(item, post.get("content"))) for item in candidate.items[: LIMITS.sample_posts]) if post.get("content") else 0
|
||||
chosen = None
|
||||
for spec in self.CONTENT_SELECTORS:
|
||||
lengths = [len(field_text(root, spec)) for root in roots]
|
||||
if lengths and statistics.median(lengths) >= 80:
|
||||
chosen = spec
|
||||
break
|
||||
if chosen is None and self.ai is not None:
|
||||
chosen = await self.ai.detail_content(roots[0])
|
||||
if chosen is None:
|
||||
return None
|
||||
gain = statistics.median(len(field_text(root, chosen)) for root in roots)
|
||||
if gain <= listed * 1.2:
|
||||
return None
|
||||
logger.info("pages: full text from %s on the post page (listing shows %d chars, page %d)", chosen, listed, gain)
|
||||
return {"content": [chosen], "render": render}
|
||||
|
||||
|
||||
class MetadataLearner:
|
||||
def __init__(self, fetcher):
|
||||
self.fetcher = fetcher
|
||||
|
||||
def from_html(self, html: str, base_url: str) -> dict:
|
||||
if not html:
|
||||
return {}
|
||||
root = parse_html(html)
|
||||
meta: dict = {}
|
||||
|
||||
def pick(selector: str, accessor: str = "::attr(content)") -> str:
|
||||
nodes = select_nodes(root, selector)
|
||||
return html_field(nodes[0], accessor) if nodes else ""
|
||||
|
||||
meta["name"] = pick("meta[property=og:site_name]") or (select_nodes(root, "title")[0].text() if select_nodes(root, "title") else "")
|
||||
meta["description"] = pick("meta[property=og:description]") or pick("meta[name=description]")
|
||||
icon = pick("link[rel=icon]", "::attr(href)") or pick("link[rel=shortcut]", "::attr(href)")
|
||||
meta["favicon"] = urljoin(base_url, icon) if icon else ""
|
||||
html_nodes = select_nodes(root, "html")
|
||||
meta["language"] = html_nodes[0].attrs.get("lang", "") if html_nodes else ""
|
||||
counted = USER_COUNT.search(root.text())
|
||||
if counted:
|
||||
digits = re.sub(r"[^\d]", "", counted.group(1))
|
||||
meta["user_count"] = int(digits) if digits else None
|
||||
return {key: value for key, value in meta.items() if value}
|
||||
|
||||
async def rules(self, base_url: str) -> str:
|
||||
parts = urlsplit(base_url)
|
||||
origin = f"{parts.scheme}://{parts.netloc}"
|
||||
for path in RULE_PATHS[:3]:
|
||||
try:
|
||||
response = await self.fetcher.get(origin + path)
|
||||
except Exception:
|
||||
continue
|
||||
if response.status >= 400:
|
||||
continue
|
||||
text = strip_tags(await response.text())
|
||||
if len(text) > 120:
|
||||
logger.info("metadata: rules text from %s (%d chars)", path, len(text))
|
||||
return text[:2000]
|
||||
return ""
|
||||
|
||||
async def learn(self, base_url: str, html: str, feed_meta: dict) -> dict:
|
||||
meta = self.from_html(html, base_url)
|
||||
for key in ("name", "description", "language"):
|
||||
if not meta.get(key) and feed_meta.get(key if key != "name" else "title"):
|
||||
meta[key] = scalar_text(feed_meta[key if key != "name" else "title"])
|
||||
if not meta.get("user_count"):
|
||||
try:
|
||||
response = await self.fetcher.get(urljoin(base_url, "/about"))
|
||||
if response.status < 400:
|
||||
counted = USER_COUNT.search(strip_tags(await response.text()))
|
||||
if counted:
|
||||
meta["user_count"] = int(re.sub(r"[^\d]", "", counted.group(1)) or 0) or None
|
||||
except Exception as exc:
|
||||
logger.info("metadata: about page unavailable: %s", exc)
|
||||
meta["rules"] = await self.rules(base_url)
|
||||
return {key: value for key, value in meta.items() if value}
|
||||
|
||||
|
||||
class StructureLearner:
|
||||
def __init__(self, fetcher, ai=None):
|
||||
self.fetcher = fetcher
|
||||
self.ai = ai
|
||||
self.pagination = PaginationProbe(fetcher)
|
||||
self.comments = CommentProbe(fetcher)
|
||||
self.pages = PageProbe(fetcher, ai)
|
||||
|
||||
async def load_items(self, candidate: FeedCandidate, params: dict) -> list:
|
||||
from devplacepy.services.community.runtime import Collector
|
||||
|
||||
collector = Collector(self.fetcher, {"feeds": [], "limits": {**LIMITS.runtime(), "delay_seconds": 0, "retries": 1}})
|
||||
text = await collector.fetch(candidate.url, params)
|
||||
_, items = collector.parse(self.as_feed(candidate, {}, {}, {}), text)
|
||||
return items
|
||||
|
||||
def as_feed(self, candidate: FeedCandidate, post: dict, author: dict, pagination: dict) -> dict:
|
||||
feed = {
|
||||
"id": "probe",
|
||||
"url": candidate.url,
|
||||
"type": candidate.type,
|
||||
"post": {"fields": post},
|
||||
"author": author,
|
||||
"pagination": pagination,
|
||||
}
|
||||
if candidate.render:
|
||||
feed["render"] = True
|
||||
if candidate.list_path:
|
||||
feed["list_path"] = candidate.list_path
|
||||
if candidate.item_selector:
|
||||
feed["item_selector"] = candidate.item_selector
|
||||
return feed
|
||||
|
||||
async def learn(self, candidate: FeedCandidate, index: int) -> dict | None:
|
||||
samples = candidate.items[: LIMITS.sample_posts]
|
||||
inferrer = None
|
||||
if candidate.type == FEED_HTML:
|
||||
html = HtmlInferrer(samples, candidate.url)
|
||||
post, author = html.post_fields(), html.author_spec()
|
||||
if self.ai is not None and html.weak(post, author):
|
||||
logger.info("learn: heuristics are weak for %s, asking the AI model", candidate.url)
|
||||
post, author = html.merge(post, author, await self.ai.post_fields(samples))
|
||||
else:
|
||||
inferrer = FieldInferrer(samples)
|
||||
post, author = inferrer.post_fields(), inferrer.author_spec()
|
||||
if "id" not in post or not ({"title", "content"} & set(post)):
|
||||
logger.info("learn: %s rejected, no stable id or no title/content (%s)", candidate.url, sorted(post))
|
||||
return None
|
||||
pagination = await self.pagination.probe(candidate, self.load_items)
|
||||
comments = await self.comments.learn(candidate, post, inferrer)
|
||||
detail = None
|
||||
if comments["mode"] == "none":
|
||||
comments, detail = await self.pages.learn(candidate, post)
|
||||
feed = self.as_feed(candidate, post, author, pagination)
|
||||
if detail:
|
||||
feed["post"]["detail"] = detail
|
||||
feed["id"] = f"feed{index + 1}"
|
||||
feed["comments"] = comments
|
||||
feed["post"]["source_footer"] = True
|
||||
logger.info(
|
||||
"learn: %s as %s: fields=%s author=%s pagination=%s comments=%s",
|
||||
candidate.url, candidate.type, sorted(post), sorted(author.get("fields", {})), pagination["style"], comments["mode"],
|
||||
)
|
||||
return feed
|
||||
@@ -0,0 +1,39 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from collections import deque
|
||||
from datetime import datetime, timezone
|
||||
|
||||
ROOT = "community"
|
||||
MAX_LINES = 400
|
||||
|
||||
|
||||
class LogBook(logging.Handler):
|
||||
def __init__(self, echo: bool = False, level: int = logging.INFO):
|
||||
super().__init__(level)
|
||||
self.lines: deque[str] = deque(maxlen=MAX_LINES)
|
||||
self.echo = echo
|
||||
self.setFormatter(logging.Formatter("%(message)s"))
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
stamp = datetime.now(timezone.utc).strftime("%H:%M:%S")
|
||||
name = record.name.split(".", 1)[-1] if "." in record.name else record.name
|
||||
line = f"{stamp} {record.levelname:<7} [{name}] {self.format(record)}"
|
||||
self.lines.append(line)
|
||||
if self.echo:
|
||||
print(line, file=sys.stderr, flush=True)
|
||||
|
||||
def __enter__(self) -> "LogBook":
|
||||
logger = logging.getLogger(ROOT)
|
||||
logger.addHandler(self)
|
||||
logger.setLevel(self.level)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> None:
|
||||
logging.getLogger(ROOT).removeHandler(self)
|
||||
|
||||
def text(self) -> str:
|
||||
return "\n".join(self.lines)
|
||||
@@ -0,0 +1,73 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from devplacepy.services.community.spec import LIMITS
|
||||
from devplacepy.services.sync.batch import BatchRunner
|
||||
from devplacepy.services.sync.records import SyncBatchForm
|
||||
from devplacepy.services.sync.spec import KINDS, STATUSES
|
||||
|
||||
logger = logging.getLogger("community.push")
|
||||
|
||||
|
||||
@dataclass
|
||||
class PushReport:
|
||||
dry_run: bool
|
||||
counts: dict = field(default_factory=dict)
|
||||
problems: list = field(default_factory=list)
|
||||
|
||||
def total(self, status: str) -> int:
|
||||
return sum(per.get(status, 0) for per in self.counts.values())
|
||||
|
||||
|
||||
DEPENDENCIES = {
|
||||
"unknown_author": ("users", "author_uid"),
|
||||
"unknown_post": ("posts", "post_uid"),
|
||||
"unknown_parent": ("comments", "parent_uid"),
|
||||
}
|
||||
|
||||
|
||||
class SyncPusher:
|
||||
def __init__(self, origin: str, dry_run: bool = False):
|
||||
self.origin = origin
|
||||
self.dry_run = dry_run
|
||||
|
||||
@staticmethod
|
||||
def satisfied_by_projection(result, raw: dict, projected: dict) -> bool:
|
||||
if result.status != "rejected" or result.reason not in DEPENDENCIES:
|
||||
return False
|
||||
kind, field_name = DEPENDENCIES[result.reason]
|
||||
return str(raw.get(field_name)) in projected[kind]
|
||||
|
||||
def push(self, records: dict[str, list[dict]]) -> PushReport:
|
||||
report = PushReport(dry_run=self.dry_run)
|
||||
mode = "DRY RUN" if self.dry_run else "LIVE"
|
||||
projected: dict[str, set[str]] = {kind: set() for kind in KINDS}
|
||||
for kind in KINDS:
|
||||
rows = records.get(kind, [])
|
||||
tally: Counter = Counter()
|
||||
logger.info("push[%s]: %s %d %s record(s) in batches of %d", mode, self.origin, len(rows), kind, LIMITS.batch_records)
|
||||
for start in range(0, len(rows), LIMITS.batch_records):
|
||||
chunk = rows[start : start + LIMITS.batch_records]
|
||||
outcome = BatchRunner(kind).run(SyncBatchForm(origin=self.origin, records=chunk, dry_run=self.dry_run))
|
||||
tally.update(outcome.counts)
|
||||
for result, raw in zip(outcome.results, chunk):
|
||||
if result.status == "created":
|
||||
projected[kind].add(result.origin_uid)
|
||||
if self.dry_run and self.satisfied_by_projection(result, raw, projected):
|
||||
tally["rejected"] -= 1
|
||||
tally["created"] += 1
|
||||
projected[kind].add(result.origin_uid)
|
||||
logger.info("push[%s]: %s %s depends on a record this dry run would create; counted as created", mode, kind, result.origin_uid)
|
||||
continue
|
||||
if result.status in ("rejected", "failed"):
|
||||
line = f"{kind} {result.origin_uid}: {result.status} {result.reason} {'; '.join(result.errors[:2])}".strip()
|
||||
report.problems.append(line)
|
||||
logger.warning("push[%s]: %s", mode, line)
|
||||
report.counts[kind] = {status: tally.get(status, 0) for status in STATUSES if tally.get(status, 0)}
|
||||
logger.info("push[%s]: %s result %s", mode, kind, report.counts[kind] or "nothing to send")
|
||||
return report
|
||||
@@ -0,0 +1,232 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from devplacepy.services.sync.aliaser import AliasRefused, Aliaser
|
||||
from devplacepy.services.sync.options import SyncOptions, alias_allowed
|
||||
from devplacepy.services.sync.selection import Selector
|
||||
from devplacepy.services.community.discoverer import CommunityDiscoverer, Discovery
|
||||
from devplacepy.services.community.fetcher import Fetcher
|
||||
from devplacepy.services.community.files import CommunityFiles, files as default_files
|
||||
from devplacepy.services.community.logbook import LogBook
|
||||
from devplacepy.services.community.pusher import SyncPusher
|
||||
from devplacepy.services.community.runtime import LoginRequired
|
||||
from devplacepy.services.community.spec import (
|
||||
LIMITS,
|
||||
RUN_DRY,
|
||||
RUN_FAILED,
|
||||
RUN_OK,
|
||||
RUN_PARTIAL,
|
||||
STATUS_LOGIN,
|
||||
STATUS_READY,
|
||||
)
|
||||
from devplacepy.services.community.store import CommunityStore, iso, now, store as default_store
|
||||
from devplacepy.services.sync.gate import refusal
|
||||
|
||||
logger = logging.getLogger("community.run")
|
||||
|
||||
TRIGGER_CLI = "cli"
|
||||
TRIGGER_ADMIN = "admin"
|
||||
TRIGGER_SERVICE = "service"
|
||||
KINDS = ("users", "posts", "comments")
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunReport:
|
||||
status: str
|
||||
dry_run: bool
|
||||
collected: dict = field(default_factory=dict)
|
||||
pushed: dict = field(default_factory=dict)
|
||||
errors: list = field(default_factory=list)
|
||||
log: str = ""
|
||||
run_uid: str = ""
|
||||
watermark: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class CommunityRunner:
|
||||
def __init__(self, fetcher_factory=Fetcher, store: CommunityStore | None = None, files: CommunityFiles | None = None, sleep=None):
|
||||
self.fetcher_factory = fetcher_factory
|
||||
self.sleep = sleep
|
||||
self.store = store or default_store
|
||||
self.files = files or default_files
|
||||
|
||||
async def discover(
|
||||
self,
|
||||
query: str,
|
||||
created_by: str,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
relearn_slug: str | None = None,
|
||||
credentials: dict | None = None,
|
||||
) -> tuple[Discovery, dict | None, str]:
|
||||
with LogBook() as book:
|
||||
logger.info("discover%s: %r", " (dry run, nothing is saved)" if dry_run else "", query)
|
||||
existing = self.store.get(relearn_slug) if relearn_slug else None
|
||||
if existing and not dry_run:
|
||||
self.store.claim(existing["uid"])
|
||||
secrets = credentials or (self.store.credentials(existing) if existing else {})
|
||||
try:
|
||||
async with self.fetcher_factory() as fetcher:
|
||||
result = await asyncio.wait_for(
|
||||
CommunityDiscoverer(fetcher, credentials=secrets).discover(query), timeout=LIMITS.run_timeout_seconds
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
result = Discovery(query=query, error=f"discovery exceeded {LIMITS.run_timeout_seconds}s")
|
||||
logger.error("discover: %s", result.error)
|
||||
except Exception as exc:
|
||||
logger.exception("discover: unexpected failure")
|
||||
result = Discovery(query=query, error=f"{type(exc).__name__}: {exc}")
|
||||
row = None
|
||||
if not dry_run and (result.slug or relearn_slug):
|
||||
row = self.persist(result, query, created_by, relearn_slug, credentials)
|
||||
logger.info("discover: saved community %s with status %s", result.slug, result.status)
|
||||
elif dry_run:
|
||||
logger.info("discover: dry run, would save %s with status %s (%s feed(s))", result.slug, result.status, len(result.schema.get("feeds", [])))
|
||||
return result, row, book.text()
|
||||
|
||||
def persist(self, result: Discovery, query: str, created_by: str, relearn_slug: str | None, credentials: dict | None = None) -> dict:
|
||||
slug = relearn_slug or result.slug
|
||||
existing = self.store.get(slug)
|
||||
learned = result.status in (STATUS_READY, STATUS_LOGIN)
|
||||
if existing and existing["status"] == STATUS_READY and not learned:
|
||||
logger.warning("discover: relearn failed, keeping the working schema of %s: %s", slug, result.error)
|
||||
return self.store.update(existing["uid"], last_error=result.error[:1000])
|
||||
fields = {
|
||||
"name": result.name or existing and existing.get("name") or slug,
|
||||
"origin": result.origin,
|
||||
"url": result.url,
|
||||
"status": result.status,
|
||||
"description": result.metadata.get("description", ""),
|
||||
"rules": result.metadata.get("rules", ""),
|
||||
"language": result.metadata.get("language", ""),
|
||||
"favicon": result.metadata.get("favicon", ""),
|
||||
"user_count": int(result.metadata.get("user_count") or 0),
|
||||
"schema": result.schema,
|
||||
"learned_at": iso(now()) if learned else "",
|
||||
"last_error": result.error,
|
||||
"failure_count": 0,
|
||||
}
|
||||
if credentials and any(credentials.values()):
|
||||
fields.update(
|
||||
login_username=credentials.get("username", ""),
|
||||
login_email=credentials.get("email", ""),
|
||||
login_password=credentials.get("password", ""),
|
||||
)
|
||||
if existing:
|
||||
row = self.store.update(existing["uid"], **fields, enabled=1 if result.status == STATUS_READY else 0)
|
||||
else:
|
||||
row = self.store.create(slug, query, created_by, **fields)
|
||||
if result.schema.get("feeds"):
|
||||
self.files.write(slug, result.schema)
|
||||
return row
|
||||
|
||||
async def run(self, row: dict, *, dry_run: bool, trigger: str) -> RunReport:
|
||||
run_uid = self.store.start_run(row["uid"], trigger, dry_run)
|
||||
report = RunReport(status=RUN_FAILED, dry_run=dry_run, run_uid=run_uid)
|
||||
with LogBook() as book:
|
||||
mode = "DRY RUN" if dry_run else "LIVE"
|
||||
logger.info("run[%s]: %s (%s) trigger=%s", mode, row["name"], row["origin"], trigger)
|
||||
try:
|
||||
await asyncio.wait_for(self.execute(row, report), timeout=LIMITS.run_timeout_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
report.errors.append(f"run exceeded {LIMITS.run_timeout_seconds}s")
|
||||
except LoginRequired as exc:
|
||||
report.errors.append(str(exc))
|
||||
if not dry_run:
|
||||
self.store.update(row["uid"], status=STATUS_LOGIN, enabled=0, last_error=str(exc))
|
||||
except Exception as exc:
|
||||
logger.exception("run[%s]: unexpected failure", mode)
|
||||
report.errors.append(f"{type(exc).__name__}: {exc}")
|
||||
for error in report.errors:
|
||||
logger.error("run[%s]: %s", mode, error)
|
||||
logger.info("run[%s]: finished with status %s", mode, report.status)
|
||||
report.log = book.text()
|
||||
if not dry_run:
|
||||
self.settle(row, report)
|
||||
self.store.finish_run(run_uid, report.status, collected=report.collected, pushed=report.pushed, errors=report.errors, log=report.log)
|
||||
self.store.prune_runs(row["uid"])
|
||||
return report
|
||||
|
||||
async def execute(self, row: dict, report: RunReport) -> None:
|
||||
refused = refusal(row["origin"])
|
||||
if refused and not report.dry_run:
|
||||
report.errors.append(refused)
|
||||
return
|
||||
if refused:
|
||||
logger.warning("run[DRY RUN]: %s; continuing because nothing is written", refused)
|
||||
module = self.files.load(row["slug"], row["schema"])
|
||||
async with self.fetcher_factory() as fetcher:
|
||||
result = await module.collect_all(fetcher, row.get("watermark") or None, sleep=self.sleep, credentials=self.store.credentials(row))
|
||||
records = result["records"]
|
||||
report.collected = {kind: len(records[kind]) for kind in KINDS}
|
||||
logger.info("run: collected %s", report.collected)
|
||||
report.errors.extend(result.get("errors", []))
|
||||
if not result["ok"]:
|
||||
report.errors.append("no feed could be collected")
|
||||
return
|
||||
options = SyncOptions.from_row(row)
|
||||
prepared = await self.prepare(row, options, records, report)
|
||||
if prepared is None:
|
||||
return
|
||||
records, skipped, held = prepared
|
||||
push = await asyncio.to_thread(SyncPusher(row["origin"], dry_run=report.dry_run).push, records)
|
||||
report.pushed = push.counts | ({"skipped": dict(skipped)} if skipped else {})
|
||||
report.errors.extend(push.problems[:20])
|
||||
report.watermark = {} if held else result["state"]
|
||||
if held:
|
||||
report.errors.append(f"{held} item(s) held back for a transient reason and will be retried")
|
||||
logger.warning("run: %d item(s) were skipped for a transient reason, the watermark is held so the next run retries them", held)
|
||||
failed = push.total("failed")
|
||||
report.status = RUN_DRY if report.dry_run else (RUN_PARTIAL if report.errors or failed else RUN_OK)
|
||||
|
||||
async def prepare(self, row: dict, options: SyncOptions, records: dict, report: RunReport):
|
||||
records, skipped = Selector(row["origin"], options).select(records)
|
||||
held = 0
|
||||
if options.alias or options.translate_to:
|
||||
if options.alias and not alias_allowed():
|
||||
report.errors.append("identity aliasing is switched off: enable community_alias_enabled on the Community sync service")
|
||||
return None
|
||||
try:
|
||||
outcome = await Aliaser(row["origin"], options, language=row.get("language") or "", dry_run=report.dry_run).apply(records)
|
||||
except AliasRefused as exc:
|
||||
report.errors.append(str(exc))
|
||||
return None
|
||||
records, held = outcome.records, outcome.transient()
|
||||
skipped.update(outcome.skipped)
|
||||
logger.info("alias: %d substitution(s), %d translated, %d new identities", outcome.substitutions, outcome.translated, outcome.identities_created)
|
||||
if skipped:
|
||||
logger.info("select: skipped %s", dict(skipped))
|
||||
return records, skipped, held
|
||||
|
||||
def settle(self, row: dict, report: RunReport) -> None:
|
||||
current = self.store.get(row["uid"])
|
||||
if current is None:
|
||||
return
|
||||
if report.status in (RUN_OK, RUN_PARTIAL):
|
||||
totals = dict(current.get("totals") or {})
|
||||
for kind, counts in report.pushed.items():
|
||||
for status, number in counts.items():
|
||||
key = f"{kind}_{status}"
|
||||
totals[key] = int(totals.get(key, 0)) + number
|
||||
self.store.update(
|
||||
row["uid"],
|
||||
watermark=report.watermark or current.get("watermark") or {},
|
||||
totals=totals,
|
||||
run_count=int(current.get("run_count") or 0) + 1,
|
||||
failure_count=0,
|
||||
last_status=report.status,
|
||||
last_error="; ".join(report.errors)[:1000],
|
||||
last_run_at=iso(now()),
|
||||
)
|
||||
self.store.schedule_next(row["uid"], int(current.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(current).jitter_percent)
|
||||
elif current["status"] != STATUS_LOGIN:
|
||||
failed = self.store.fail(current, "; ".join(report.errors) or "run failed")
|
||||
self.store.schedule_next(row["uid"], int(failed.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(failed).jitter_percent)
|
||||
logger.error(
|
||||
"run: failure %d of %d for %s. Structure drift is likely: re-run discovery with relearn.",
|
||||
failed["failure_count"], LIMITS.max_failures, row["slug"],
|
||||
)
|
||||
@@ -0,0 +1,487 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from devplacepy.services.community.extract import (
|
||||
Node,
|
||||
field_text,
|
||||
find_login_form,
|
||||
html_field,
|
||||
json_or_none,
|
||||
normalize_instant,
|
||||
parse_feed,
|
||||
parse_html,
|
||||
select,
|
||||
select_nodes,
|
||||
strip_tags,
|
||||
text_at,
|
||||
)
|
||||
|
||||
UID_PATTERN = re.compile(r"^[A-Za-z0-9._:@-]+$")
|
||||
UID_MAX_LENGTH = 64
|
||||
MIN_POST_LENGTH = 10
|
||||
MIN_COMMENT_LENGTH = 3
|
||||
MAX_REPLY_DEPTH = 5
|
||||
RETRY_STATUSES = frozenset({408, 425, 429, 500, 502, 503, 504})
|
||||
RENDER_SETTLE_MS = 800
|
||||
DEFAULT_LIMITS = {
|
||||
"max_pages": 3,
|
||||
"max_posts": 150,
|
||||
"max_comment_posts": 25,
|
||||
"comment_recheck": 10,
|
||||
"delay_seconds": 1.0,
|
||||
"retries": 3,
|
||||
"backoff_seconds": 2.0,
|
||||
"timeout_seconds": 30,
|
||||
"max_bytes": 5_000_000,
|
||||
}
|
||||
|
||||
|
||||
class CollectError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class LoginRequired(CollectError):
|
||||
pass
|
||||
|
||||
|
||||
class LoginFailed(LoginRequired):
|
||||
pass
|
||||
|
||||
|
||||
class PageMissing(CollectError):
|
||||
pass
|
||||
|
||||
|
||||
def uid_for(value) -> str:
|
||||
text = str(value if value is not None else "").strip()
|
||||
if text and len(text) <= UID_MAX_LENGTH and UID_PATTERN.match(text):
|
||||
return text
|
||||
return "h:" + hashlib.sha256(text.encode()).hexdigest()[:32]
|
||||
|
||||
|
||||
def iso_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def absolute_id(value: str, base: str) -> str:
|
||||
return urljoin(base, value) if value.startswith("/") else value
|
||||
|
||||
|
||||
def flatten_matches(matches: list) -> list:
|
||||
if len(matches) == 1 and isinstance(matches[0], list):
|
||||
return matches[0]
|
||||
return matches
|
||||
|
||||
|
||||
class Collector:
|
||||
def __init__(self, stealth, schema: dict, state: dict | None = None, sleep=None, credentials: dict | None = None):
|
||||
self.stealth = stealth
|
||||
self.schema = schema
|
||||
self.state = {"feeds": dict((state or {}).get("feeds", {}))}
|
||||
self.sleep = sleep or asyncio.sleep
|
||||
self.credentials = credentials or {}
|
||||
self.limits = {**DEFAULT_LIMITS, **schema.get("limits", {})}
|
||||
self.session = f"community:{schema.get('origin') or 'default'}"
|
||||
self.requests = 0
|
||||
self.users: dict[str, dict] = {}
|
||||
self.posts: dict[str, dict] = {}
|
||||
self.comments: dict[str, dict] = {}
|
||||
self.pages: dict[str, Node] = {}
|
||||
self.errors: list[str] = []
|
||||
|
||||
async def attempt(self, label: str, action) -> str:
|
||||
last = "no attempt"
|
||||
for attempt in range(int(self.limits["retries"])):
|
||||
if self.requests:
|
||||
await self.sleep(float(self.limits["delay_seconds"]))
|
||||
self.requests += 1
|
||||
try:
|
||||
status, text = await action()
|
||||
if status in (401, 407):
|
||||
raise LoginRequired(f"{label} answered {status}: login required")
|
||||
if status in RETRY_STATUSES:
|
||||
last = f"status {status}"
|
||||
elif status in (404, 410):
|
||||
raise PageMissing(f"{label} answered {status}")
|
||||
elif status >= 400:
|
||||
raise CollectError(f"{label} answered {status}")
|
||||
else:
|
||||
return text[: int(self.limits["max_bytes"])]
|
||||
except CollectError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
last = f"{type(exc).__name__}: {exc}"
|
||||
await self.sleep(float(self.limits["backoff_seconds"]) * (2**attempt))
|
||||
raise CollectError(f"{label} failed after {self.limits['retries']} attempts ({last})")
|
||||
|
||||
async def fetch(self, url: str, params: dict | None = None, render: bool = False) -> str:
|
||||
if render:
|
||||
return await self.attempt(url, lambda: self.render_html(url))
|
||||
|
||||
async def action():
|
||||
response = await self.stealth.get(url, params=params, timeout=self.limits["timeout_seconds"], session=self.session)
|
||||
return int(response.status), await response.text()
|
||||
|
||||
return await self.attempt(url, action)
|
||||
|
||||
async def post(self, url: str, data: dict) -> str:
|
||||
async def action():
|
||||
response = await self.stealth.post(url, data=data, timeout=self.limits["timeout_seconds"], session=self.session)
|
||||
return int(response.status), await response.text()
|
||||
|
||||
return await self.attempt(url, action)
|
||||
|
||||
async def render_html(self, url: str):
|
||||
async with self.stealth.page(session=self.session) as page:
|
||||
response = await page.goto(url, wait_until="domcontentloaded")
|
||||
await page.wait_for_timeout(RENDER_SETTLE_MS)
|
||||
return (response.status if response else 200), await page.content()
|
||||
|
||||
async def page_root(self, url: str, render: bool) -> Node:
|
||||
key = f"{int(render)}|{url}"
|
||||
if key not in self.pages:
|
||||
self.pages[key] = parse_html(await self.fetch(url, render=render))
|
||||
return self.pages[key]
|
||||
|
||||
def has_credentials(self) -> bool:
|
||||
return bool(self.credentials.get("password") and (self.credentials.get("username") or self.credentials.get("email")))
|
||||
|
||||
async def login(self) -> bool:
|
||||
spec = self.schema.get("login")
|
||||
if not spec or not self.has_credentials():
|
||||
return False
|
||||
identity = self.credentials.get("email") if spec.get("identity") == "email" else self.credentials.get("username")
|
||||
identity = identity or self.credentials.get("username") or self.credentials.get("email")
|
||||
render = bool(spec.get("render"))
|
||||
if spec.get("mode") == "form":
|
||||
root = parse_html(await self.fetch(spec["url"], render=render))
|
||||
form = find_login_form(root, spec["url"]) or spec
|
||||
data = {**form.get("fields", {}), form["username_field"]: identity, form["password_field"]: self.credentials["password"]}
|
||||
if str(form.get("method", "POST")).upper() == "GET":
|
||||
await self.fetch(form["action"], params=data)
|
||||
else:
|
||||
await self.post(form["action"], data)
|
||||
else:
|
||||
async with self.stealth.page(session=self.session) as page:
|
||||
await page.goto(spec["url"], wait_until="domcontentloaded")
|
||||
await page.fill(spec["username_selector"], identity)
|
||||
await page.fill(spec["password_selector"], self.credentials["password"])
|
||||
await page.click(spec["submit_selector"])
|
||||
await page.wait_for_timeout(RENDER_SETTLE_MS * 2)
|
||||
after = parse_html(await self.fetch(spec.get("check_url") or self.schema.get("base_url") or spec["url"], render=render))
|
||||
if select_nodes(after, "input[type=password]"):
|
||||
raise LoginFailed(f"{self.schema.get('community', 'community')} rejected the stored credentials")
|
||||
return True
|
||||
|
||||
def parse(self, feed: dict, text: str):
|
||||
kind = feed["type"]
|
||||
if kind == "json_api":
|
||||
document = json_or_none(text)
|
||||
if document is None:
|
||||
raise CollectError(f"{feed['url']} did not return JSON")
|
||||
return document, flatten_matches(select(document, feed.get("list_path") or "$"))
|
||||
if kind in ("rss", "atom"):
|
||||
_, meta, items = parse_feed(text)
|
||||
return {"meta": meta}, items
|
||||
root = parse_html(text)
|
||||
return root, select_nodes(root, feed["item_selector"])
|
||||
|
||||
def next_request(self, feed: dict, index: int, document, current: str):
|
||||
pagination = feed.get("pagination") or {"style": "none"}
|
||||
style = pagination.get("style", "none")
|
||||
if style == "page":
|
||||
return feed["url"], {pagination["param"]: int(pagination.get("start", 1)) + index}
|
||||
if style == "offset":
|
||||
return feed["url"], {pagination["param"]: index * int(pagination["per_page"])}
|
||||
if style == "cursor":
|
||||
cursor = text_at(document, pagination["cursor_path"]) if isinstance(document, (dict, list)) else ""
|
||||
return (feed["url"], {pagination["param"]: cursor}) if cursor else None
|
||||
if style == "next_link":
|
||||
target = html_field(document, pagination["next"]) if isinstance(document, Node) else text_at(document, pagination["next"])
|
||||
return (urljoin(current, target), None) if target else None
|
||||
return None
|
||||
|
||||
def stamp_of(self, feed: dict, item) -> str | None:
|
||||
fields = feed["post"]["fields"]
|
||||
return normalize_instant(field_text(item, fields.get("updated_at")) or field_text(item, fields.get("created_at")))
|
||||
|
||||
def post_link(self, feed: dict, item, base: str) -> str:
|
||||
link = field_text(item, feed["post"]["fields"].get("url"))
|
||||
return urljoin(base, link) if link else ""
|
||||
|
||||
def post_identifier(self, feed: dict, item, base: str) -> str:
|
||||
raw = field_text(item, feed["post"]["fields"].get("id"))
|
||||
return absolute_id(raw, base) if raw else self.post_link(feed, item, base)
|
||||
|
||||
async def collect_feed(self, feed: dict) -> dict:
|
||||
feed_state = self.state["feeds"].get(feed["id"], {})
|
||||
since = feed_state.get("since")
|
||||
newest = since
|
||||
limit = int(self.limits["max_posts"])
|
||||
render = bool(feed.get("render"))
|
||||
fresh: list[tuple] = []
|
||||
recent: list[tuple] = []
|
||||
url, params = feed["url"], None
|
||||
seen: set[str] = set()
|
||||
stats = {"pages": 0, "items": 0, "fresh": 0}
|
||||
for index in range(int(self.limits["max_pages"])):
|
||||
try:
|
||||
text = await self.fetch(url, params, render=render)
|
||||
except PageMissing:
|
||||
if index == 0:
|
||||
raise
|
||||
stats["ended"] = "no further pages"
|
||||
break
|
||||
document, items = self.parse(feed, text)
|
||||
stats["pages"] += 1
|
||||
if not items:
|
||||
break
|
||||
page_old = bool(since)
|
||||
added = 0
|
||||
for item in items:
|
||||
stats["items"] += 1
|
||||
stamp = self.stamp_of(feed, item)
|
||||
key = self.post_identifier(feed, item, url) or json.dumps(item, sort_keys=True, default=str)[:200]
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
added += 1
|
||||
if len(recent) < int(self.limits["comment_recheck"]):
|
||||
recent.append((item, url))
|
||||
if stamp is None or since is None or stamp > since:
|
||||
page_old = False
|
||||
fresh.append((item, url))
|
||||
if stamp and (newest is None or stamp > newest):
|
||||
newest = stamp
|
||||
if len(fresh) >= limit:
|
||||
break
|
||||
if len(fresh) >= limit or page_old or not added:
|
||||
break
|
||||
following = self.next_request(feed, index + 1, document, url)
|
||||
if following is None:
|
||||
break
|
||||
url, params = following
|
||||
stats["fresh"] = len(fresh)
|
||||
details = await self.fetch_details(feed, fresh)
|
||||
for item, base in fresh:
|
||||
self.add_post(feed, item, base, details.get(self.post_link(feed, item, base)))
|
||||
await self.collect_comments(feed, fresh, recent)
|
||||
self.state["feeds"][feed["id"]] = {"since": newest or since or iso_now()}
|
||||
return stats
|
||||
|
||||
async def fetch_details(self, feed: dict, fresh: list) -> dict[str, Node]:
|
||||
detail = feed["post"].get("detail")
|
||||
found: dict[str, Node] = {}
|
||||
if not detail:
|
||||
return found
|
||||
for item, base in fresh[: int(self.limits["max_comment_posts"])]:
|
||||
link = self.post_link(feed, item, base)
|
||||
if not link:
|
||||
continue
|
||||
try:
|
||||
found[link] = await self.page_root(link, bool(feed.get("render") or detail.get("render")))
|
||||
except PageMissing:
|
||||
continue
|
||||
except LoginRequired:
|
||||
raise
|
||||
except CollectError as exc:
|
||||
self.errors.append(f"detail {link}: {exc}")
|
||||
return found
|
||||
|
||||
def add_post(self, feed: dict, item, base: str, detail: Node | None = None) -> str | None:
|
||||
fields = feed["post"]["fields"]
|
||||
link = self.post_link(feed, item, base)
|
||||
identifier = self.post_identifier(feed, item, base)
|
||||
if not identifier:
|
||||
return None
|
||||
title = strip_tags(field_text(item, fields.get("title")))
|
||||
content = strip_tags(field_text(item, fields.get("content")))
|
||||
if detail is not None:
|
||||
full = strip_tags(field_text(detail, feed["post"]["detail"]["content"]))
|
||||
if len(full) > len(content):
|
||||
content = full
|
||||
if feed["post"].get("source_footer", True) and link:
|
||||
content = f"{content}\n\nSource: {link}".strip()
|
||||
if len(content) < MIN_POST_LENGTH:
|
||||
content = f"{title}\n\n{content}".strip()
|
||||
if len(content) < MIN_POST_LENGTH:
|
||||
return None
|
||||
created = normalize_instant(field_text(item, fields.get("created_at"))) or iso_now()
|
||||
updated = normalize_instant(field_text(item, fields.get("updated_at")))
|
||||
post_uid = uid_for(identifier)
|
||||
record = {
|
||||
"origin_uid": post_uid,
|
||||
"author_uid": self.add_author(feed.get("author") or {}, item, created),
|
||||
"title": title,
|
||||
"content": content,
|
||||
"created_at": created,
|
||||
}
|
||||
if updated:
|
||||
record["updated_at"] = updated
|
||||
self.posts[post_uid] = record
|
||||
return post_uid
|
||||
|
||||
def add_author(self, spec: dict, item, moment: str) -> str:
|
||||
fields = spec.get("fields", {})
|
||||
username = strip_tags(field_text(item, fields.get("username")))
|
||||
identifier = field_text(item, fields.get("id"))
|
||||
if not username and not identifier:
|
||||
username, identifier = "anonymous", "anonymous"
|
||||
user_uid = uid_for(identifier or f"name:{username}")
|
||||
joined = normalize_instant(field_text(item, fields.get("created_at"))) or moment
|
||||
existing = self.users.get(user_uid)
|
||||
if existing:
|
||||
existing["created_at"] = min(existing["created_at"], joined)
|
||||
return user_uid
|
||||
self.users[user_uid] = {"origin_uid": user_uid, "username": username or identifier, "created_at": joined}
|
||||
return user_uid
|
||||
|
||||
async def collect_comments(self, feed: dict, fresh: list, recent: list) -> None:
|
||||
spec = feed.get("comments") or {"mode": "none"}
|
||||
mode = spec.get("mode", "none")
|
||||
if mode == "none":
|
||||
return
|
||||
targets: dict[str, tuple] = {}
|
||||
for item, base in [*fresh, *recent]:
|
||||
identifier = self.post_identifier(feed, item, base)
|
||||
if identifier:
|
||||
targets.setdefault(uid_for(identifier), (item, base, identifier))
|
||||
for post_uid, (item, base, identifier) in list(targets.items())[: int(self.limits["max_comment_posts"])]:
|
||||
try:
|
||||
if mode == "inline":
|
||||
self.add_comments(spec, flatten_matches(select(item, spec["list_path"])), post_uid, None, 0)
|
||||
elif mode == "url":
|
||||
target = spec["url_template"].replace("{id}", identifier).replace("{uid}", post_uid)
|
||||
document = json_or_none(await self.fetch(urljoin(base, target), render=bool(spec.get("render"))))
|
||||
entries = flatten_matches(select(document, spec.get("list_path") or "$")) if document is not None else []
|
||||
self.add_comments(spec, entries, post_uid, None, 0)
|
||||
elif mode == "page":
|
||||
link = self.post_link(feed, item, base)
|
||||
if link:
|
||||
root = await self.page_root(link, bool(feed.get("render") or spec.get("render")))
|
||||
self.add_page_comments(spec, root, post_uid)
|
||||
except PageMissing:
|
||||
continue
|
||||
except LoginRequired:
|
||||
raise
|
||||
except CollectError as exc:
|
||||
self.errors.append(f"comments {identifier}: {exc}")
|
||||
|
||||
def add_comments(self, spec: dict, entries: list, post_uid: str, parent_uid: str | None, depth: int) -> None:
|
||||
fields = spec["fields"]
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
content = strip_tags(field_text(entry, fields.get("content")))
|
||||
identifier = field_text(entry, fields.get("id"))
|
||||
if not identifier or len(content) < MIN_COMMENT_LENGTH:
|
||||
continue
|
||||
created = normalize_instant(field_text(entry, fields.get("created_at"))) or iso_now()
|
||||
declared = field_text(entry, fields.get("parent_id"))
|
||||
parent = parent_uid or (uid_for(declared) if declared else None)
|
||||
self.store_comment(spec, entry, uid_for(identifier), post_uid, parent, content, created)
|
||||
replies = spec.get("replies_path")
|
||||
if replies and depth < MAX_REPLY_DEPTH:
|
||||
self.add_comments(spec, flatten_matches(select(entry, replies)), post_uid, uid_for(identifier), depth + 1)
|
||||
|
||||
def comment_identifier(self, spec: dict, node: Node, post_uid: str, content: str) -> str:
|
||||
declared = field_text(node, spec["fields"].get("id"))
|
||||
if declared:
|
||||
return declared
|
||||
author = strip_tags(field_text(node, (spec.get("author") or {}).get("fields", {}).get("username")))
|
||||
return hashlib.sha256(f"{post_uid}|{author}|{content[:120]}".encode()).hexdigest()[:24]
|
||||
|
||||
def add_page_comments(self, spec: dict, root: Node, post_uid: str) -> None:
|
||||
fields = spec["fields"]
|
||||
nodes = select_nodes(root, spec["item_selector"])
|
||||
known = {id(node) for node in nodes}
|
||||
uids: dict[int, str] = {}
|
||||
for node in nodes:
|
||||
content = strip_tags(field_text(node, fields.get("content")))
|
||||
if len(content) < MIN_COMMENT_LENGTH:
|
||||
continue
|
||||
uids[id(node)] = uid_for(self.comment_identifier(spec, node, post_uid, content))
|
||||
for node in nodes:
|
||||
if id(node) not in uids:
|
||||
continue
|
||||
ancestor = node.parent
|
||||
while ancestor is not None and id(ancestor) not in known:
|
||||
ancestor = ancestor.parent
|
||||
parent = uids.get(id(ancestor)) if ancestor is not None else None
|
||||
content = strip_tags(field_text(node, fields.get("content")))
|
||||
created = normalize_instant(field_text(node, fields.get("created_at"))) or iso_now()
|
||||
self.store_comment(spec, node, uids[id(node)], post_uid, parent, content, created)
|
||||
|
||||
def store_comment(self, spec: dict, entry, comment_uid: str, post_uid: str, parent: str | None, content: str, created: str) -> None:
|
||||
record = {
|
||||
"origin_uid": comment_uid,
|
||||
"author_uid": self.add_author(spec.get("author", {}), entry, created),
|
||||
"post_uid": post_uid,
|
||||
"content": content,
|
||||
"created_at": created,
|
||||
}
|
||||
if parent and parent != comment_uid:
|
||||
record["parent_uid"] = parent
|
||||
updated = normalize_instant(field_text(entry, spec["fields"].get("updated_at")))
|
||||
if updated:
|
||||
record["updated_at"] = updated
|
||||
self.comments[comment_uid] = record
|
||||
|
||||
def ordered_comments(self) -> list[dict]:
|
||||
ordered: list[dict] = []
|
||||
placed: set[str] = set()
|
||||
pending = list(self.comments.values())
|
||||
while pending:
|
||||
progress = [record for record in pending if record.get("parent_uid") not in self.comments or record.get("parent_uid") in placed]
|
||||
if not progress:
|
||||
progress = pending
|
||||
for record in progress:
|
||||
if record.get("parent_uid") not in self.comments:
|
||||
record = {key: value for key, value in record.items() if key != "parent_uid"}
|
||||
ordered.append(record)
|
||||
placed.add(record["origin_uid"])
|
||||
done = {record["origin_uid"] for record in progress}
|
||||
pending = [record for record in pending if record["origin_uid"] not in done]
|
||||
return ordered
|
||||
|
||||
async def run(self, only: str | None = None) -> dict:
|
||||
community = self.schema.get("community", "community")
|
||||
spec = self.schema.get("login")
|
||||
if self.schema.get("login_required") and not (spec and self.has_credentials()):
|
||||
raise LoginRequired(f"{community} requires a login; add the account's username or email and password to the community")
|
||||
if spec and self.has_credentials():
|
||||
await self.login()
|
||||
reports = {}
|
||||
for feed in self.schema["feeds"]:
|
||||
if only and feed["id"] != only:
|
||||
continue
|
||||
try:
|
||||
reports[feed["id"]] = await self.collect_feed(feed)
|
||||
except LoginRequired:
|
||||
raise
|
||||
except CollectError as exc:
|
||||
self.errors.append(f"{feed['id']}: {exc}")
|
||||
reports[feed["id"]] = {"error": str(exc)}
|
||||
failed = [name for name, report in reports.items() if "error" in report]
|
||||
return {
|
||||
"community": community,
|
||||
"origin": self.schema.get("origin"),
|
||||
"collected_at": iso_now(),
|
||||
"records": {
|
||||
"users": list(self.users.values()),
|
||||
"posts": list(self.posts.values()),
|
||||
"comments": self.ordered_comments(),
|
||||
},
|
||||
"state": self.state,
|
||||
"feeds": reports,
|
||||
"errors": self.errors,
|
||||
"ok": bool(reports) and len(failed) < len(reports),
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from devplacepy.services.community.extract import json_or_none
|
||||
from devplacepy.services.community.spec import LIMITS, RSEARCH_URL
|
||||
|
||||
logger = logging.getLogger("community.search")
|
||||
|
||||
NOISE_HOSTS = (
|
||||
"linkedin.com",
|
||||
"facebook.com",
|
||||
"twitter.com",
|
||||
"x.com",
|
||||
"apps.apple.com",
|
||||
"play.google.com",
|
||||
"wikipedia.org",
|
||||
"youtube.com",
|
||||
"instagram.com",
|
||||
"tiktok.com",
|
||||
"pinterest.com",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Candidate:
|
||||
url: str
|
||||
title: str
|
||||
description: str
|
||||
score: float = 0.0
|
||||
|
||||
@property
|
||||
def host(self) -> str:
|
||||
return urlsplit(self.url).hostname or ""
|
||||
|
||||
|
||||
def words(text: str) -> list[str]:
|
||||
return [word for word in re.split(r"[^a-z0-9]+", text.lower()) if len(word) > 1]
|
||||
|
||||
|
||||
def direct_url(query: str) -> str | None:
|
||||
text = query.strip()
|
||||
if re.match(r"^https?://\S+$", text):
|
||||
return text
|
||||
if re.match(r"^[a-z0-9-]+(\.[a-z0-9-]+)+(/\S*)?$", text.lower()):
|
||||
return f"https://{text}"
|
||||
return None
|
||||
|
||||
|
||||
def rank(query: str, results: list[dict]) -> list[Candidate]:
|
||||
terms = words(query)
|
||||
ranked = []
|
||||
for entry in results:
|
||||
url = str(entry.get("url") or "")
|
||||
host = (urlsplit(url).hostname or "").lower()
|
||||
if not host or any(host == noise or host.endswith("." + noise) for noise in NOISE_HOSTS):
|
||||
continue
|
||||
title = str(entry.get("title") or "")
|
||||
description = str(entry.get("description") or "")
|
||||
score = 0.0
|
||||
for term in terms:
|
||||
if term in host:
|
||||
score += 3.0
|
||||
if term in title.lower():
|
||||
score += 1.5
|
||||
if term in description.lower():
|
||||
score += 0.5
|
||||
if urlsplit(url).path in ("", "/"):
|
||||
score += 1.0
|
||||
ranked.append(Candidate(url, title, description, score))
|
||||
ranked.sort(key=lambda candidate: candidate.score, reverse=True)
|
||||
return ranked
|
||||
|
||||
|
||||
class SearchPhase:
|
||||
def __init__(self, fetcher):
|
||||
self.fetcher = fetcher
|
||||
|
||||
async def run(self, query: str) -> list[Candidate]:
|
||||
direct = direct_url(query)
|
||||
if direct:
|
||||
logger.info("search: %r is a direct address", query)
|
||||
return [Candidate(direct, direct, "", 100.0)]
|
||||
response = await self.fetcher.get(
|
||||
RSEARCH_URL,
|
||||
params={"query": query, "count": LIMITS.search_results, "content": "false"},
|
||||
)
|
||||
body = json_or_none(await response.text())
|
||||
if response.status >= 400 or not isinstance(body, dict) or not body.get("success", True):
|
||||
raise RuntimeError(f"rsearch failed for {query!r}: status {response.status}")
|
||||
candidates = rank(query, body.get("results") or [])[: LIMITS.max_candidates]
|
||||
for candidate in candidates:
|
||||
logger.info("search: candidate %.1f %s (%s)", candidate.score, candidate.url, candidate.title)
|
||||
return candidates
|
||||
@@ -0,0 +1,113 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from devplacepy.services.base import BaseService, ConfigField
|
||||
from devplacepy.services.community.runner import TRIGGER_SERVICE, CommunityRunner
|
||||
from devplacepy.services.community.spec import LIMITS, STATUS_FAILING, STATUS_LOGIN, STATUS_READY
|
||||
from devplacepy.services.community.store import store
|
||||
from devplacepy.services.sync.options import ALIAS_KEY
|
||||
|
||||
DRY_RUN_KEY = "community_dry_run"
|
||||
MAX_PER_RUN_KEY = "community_max_per_run"
|
||||
DISCOVERY_ACTOR = "system"
|
||||
|
||||
|
||||
class CommunityService(BaseService):
|
||||
title = "Community sync"
|
||||
description = (
|
||||
"Discovers and learns external communities queued by an administrator, then runs their "
|
||||
"generated collectors on a schedule and feeds users, posts and comments through the sync API."
|
||||
)
|
||||
details = (
|
||||
"Opt-in and dry run by default: a dry run fetches, validates and logs exactly what would "
|
||||
"be created, updated or rejected, and writes nothing. Switch dry run off only after the "
|
||||
"log lists what you expect. Live runs also require the sync_enabled site setting and honor "
|
||||
"sync_origins. A community that fails repeatedly is disabled and flagged for a human."
|
||||
)
|
||||
default_enabled = False
|
||||
min_interval = 30
|
||||
METRICS_SECONDS = 60
|
||||
config_fields = [
|
||||
ConfigField(
|
||||
DRY_RUN_KEY,
|
||||
"Dry run",
|
||||
type="bool",
|
||||
default=True,
|
||||
help="Collect and validate everything, log the outcome, write nothing and advance no watermark.",
|
||||
group="Safety",
|
||||
),
|
||||
ConfigField(
|
||||
MAX_PER_RUN_KEY,
|
||||
"Communities per tick",
|
||||
type="int",
|
||||
default=LIMITS.max_communities_per_run,
|
||||
minimum=1,
|
||||
maximum=20,
|
||||
help="Upper bound on communities processed in one tick. The remainder waits for the next tick.",
|
||||
group="Safety",
|
||||
),
|
||||
ConfigField(
|
||||
ALIAS_KEY,
|
||||
"Identity aliasing",
|
||||
type="bool",
|
||||
default=False,
|
||||
help="Master kill switch. While off, no community with aliasing enabled is run and no new identity is created.",
|
||||
group="Safety",
|
||||
),
|
||||
]
|
||||
|
||||
def __init__(self, runner: CommunityRunner | None = None) -> None:
|
||||
super().__init__("community", interval_seconds=60)
|
||||
self.runner = runner or CommunityRunner()
|
||||
|
||||
async def on_disable(self) -> None:
|
||||
from devplacepy.services.community.fetcher import SharedBrowser
|
||||
|
||||
await SharedBrowser.close()
|
||||
|
||||
async def run_once(self) -> None:
|
||||
config = self.get_config()
|
||||
dry = config[DRY_RUN_KEY]
|
||||
budget = config[MAX_PER_RUN_KEY]
|
||||
mode = "DRY RUN" if dry else "LIVE"
|
||||
for row in store.pending(budget):
|
||||
budget -= 1
|
||||
self.log(f"[{mode}] discovering pending community {row['query']!r}")
|
||||
result, saved, text = await self.runner.discover(row["query"], DISCOVERY_ACTOR, dry_run=dry, relearn_slug=row["slug"])
|
||||
if dry:
|
||||
self.log(f"[DRY RUN] would learn {result.slug}: {result.status}{' - ' + result.error if result.error else ''}")
|
||||
continue
|
||||
self.log(f"discovery of {row['slug']}: {result.status}{' - ' + result.error if result.error else ''}")
|
||||
if budget <= 0:
|
||||
return
|
||||
for row in store.due(budget):
|
||||
self.log(f"[{mode}] running {row['slug']}")
|
||||
report = await self.runner.run(row, dry_run=dry, trigger=TRIGGER_SERVICE)
|
||||
counts = {kind: sum(per.values()) for kind, per in report.pushed.items()}
|
||||
self.log(f"[{mode}] {row['slug']}: {report.status}, collected {report.collected}, pushed {counts}")
|
||||
for line in report.log.splitlines()[-12:]:
|
||||
self.log(f" {line}", level=logging.DEBUG)
|
||||
|
||||
def collect_metrics(self) -> dict:
|
||||
rows = store.list()
|
||||
tally = {status: 0 for status in (STATUS_READY, STATUS_LOGIN, STATUS_FAILING)}
|
||||
for row in rows:
|
||||
tally[row["status"]] = tally.get(row["status"], 0) + 1
|
||||
return {
|
||||
"stats": [
|
||||
{"label": "Communities", "value": len(rows)},
|
||||
{"label": "Ready", "value": tally.get(STATUS_READY, 0)},
|
||||
{"label": "Need a human", "value": tally.get(STATUS_LOGIN, 0) + tally.get(STATUS_FAILING, 0)},
|
||||
{"label": "Dry run", "value": 1 if self.get_config()[DRY_RUN_KEY] else 0},
|
||||
],
|
||||
"table": {
|
||||
"columns": ["Community", "Status", "Runs", "Last run", "Next run"],
|
||||
"rows": [
|
||||
[row["name"], row["status"], row.get("run_count") or 0, row.get("last_run_at") or "-", row.get("next_run_at") or "-"]
|
||||
for row in rows
|
||||
],
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
FEED_RSS = "rss"
|
||||
FEED_ATOM = "atom"
|
||||
FEED_JSON = "json_api"
|
||||
FEED_HTML = "html_list"
|
||||
FEED_TYPES = (FEED_RSS, FEED_ATOM, FEED_JSON, FEED_HTML)
|
||||
|
||||
STATUS_PENDING = "pending"
|
||||
STATUS_LEARNING = "learning"
|
||||
STATUS_READY = "ready"
|
||||
STATUS_LOGIN = "login_required"
|
||||
STATUS_FAILED = "failed"
|
||||
STATUS_FAILING = "failing"
|
||||
STATUSES = (
|
||||
STATUS_PENDING,
|
||||
STATUS_LEARNING,
|
||||
STATUS_READY,
|
||||
STATUS_LOGIN,
|
||||
STATUS_FAILED,
|
||||
STATUS_FAILING,
|
||||
)
|
||||
|
||||
RUN_OK = "ok"
|
||||
RUN_PARTIAL = "partial"
|
||||
RUN_FAILED = "failed"
|
||||
RUN_DRY = "dry_run"
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
RSEARCH_URL = "https://rsearch.app.molodetz.nl/search"
|
||||
USER_AGENT_NOTE = "devplace-community"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CommunityLimits:
|
||||
search_results: int = 15
|
||||
max_candidates: int = 5
|
||||
max_probe_paths: int = 12
|
||||
sample_posts: int = 12
|
||||
sample_comment_posts: int = 3
|
||||
max_pages: int = 3
|
||||
max_posts: int = 150
|
||||
max_comment_posts: int = 25
|
||||
comment_recheck: int = 10
|
||||
delay_seconds: float = 1.0
|
||||
retries: int = 3
|
||||
backoff_seconds: float = 2.0
|
||||
timeout_seconds: int = 30
|
||||
max_bytes: int = 5_000_000
|
||||
min_interval_seconds: int = 900
|
||||
default_interval_seconds: int = 86_400
|
||||
run_timeout_seconds: int = 900
|
||||
max_failures: int = 5
|
||||
max_communities_per_run: int = 3
|
||||
runs_kept: int = 30
|
||||
batch_records: int = 50
|
||||
min_field_coverage: float = 0.6
|
||||
|
||||
def runtime(self) -> dict:
|
||||
return {
|
||||
"max_pages": self.max_pages,
|
||||
"max_posts": self.max_posts,
|
||||
"max_comment_posts": self.max_comment_posts,
|
||||
"comment_recheck": self.comment_recheck,
|
||||
"delay_seconds": self.delay_seconds,
|
||||
"retries": self.retries,
|
||||
"backoff_seconds": self.backoff_seconds,
|
||||
"timeout_seconds": self.timeout_seconds,
|
||||
"max_bytes": self.max_bytes,
|
||||
}
|
||||
|
||||
|
||||
LIMITS = CommunityLimits()
|
||||
FEED_PROBE_PATHS = (
|
||||
"/feed",
|
||||
"/feed.xml",
|
||||
"/rss",
|
||||
"/rss.xml",
|
||||
"/atom.xml",
|
||||
"/index.xml",
|
||||
"/feeds/posts",
|
||||
"/posts.json",
|
||||
"/latest.json",
|
||||
"/api/posts",
|
||||
"/api/v1/posts",
|
||||
"/json",
|
||||
)
|
||||
@@ -0,0 +1,200 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from devplacepy.database import get_table
|
||||
from devplacepy.database.soft_delete import soft_delete
|
||||
from devplacepy.services.community.spec import (
|
||||
LIMITS,
|
||||
STATUS_FAILING,
|
||||
STATUS_LEARNING,
|
||||
STATUS_PENDING,
|
||||
STATUS_READY,
|
||||
)
|
||||
from devplacepy.services.sync.selection import jittered
|
||||
from devplacepy.utils.text import generate_uid
|
||||
|
||||
JSON_COLUMNS = ("schema", "watermark", "totals", "sync_options")
|
||||
RUN_JSON_COLUMNS = ("collected", "pushed", "errors")
|
||||
EDITABLE = (
|
||||
"name",
|
||||
"description",
|
||||
"rules",
|
||||
"language",
|
||||
"favicon",
|
||||
"user_count",
|
||||
"status",
|
||||
"schema",
|
||||
"watermark",
|
||||
"enabled",
|
||||
"interval_seconds",
|
||||
"next_run_at",
|
||||
"last_run_at",
|
||||
"last_status",
|
||||
"last_error",
|
||||
"run_count",
|
||||
"failure_count",
|
||||
"totals",
|
||||
"learned_at",
|
||||
"url",
|
||||
"origin",
|
||||
"login_username",
|
||||
"login_email",
|
||||
"login_password",
|
||||
"sync_options",
|
||||
"claimed_at",
|
||||
)
|
||||
|
||||
|
||||
def now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def iso(moment: datetime) -> str:
|
||||
return moment.isoformat()
|
||||
|
||||
|
||||
def decode(row: dict | None, columns: tuple[str, ...]) -> dict | None:
|
||||
if row is None:
|
||||
return None
|
||||
view = dict(row)
|
||||
for column in columns:
|
||||
raw = view.get(column)
|
||||
try:
|
||||
view[column] = json.loads(raw) if raw else ({} if column != "errors" else [])
|
||||
except ValueError:
|
||||
view[column] = {} if column != "errors" else []
|
||||
return view
|
||||
|
||||
|
||||
def encode(fields: dict, columns: tuple[str, ...]) -> dict:
|
||||
return {
|
||||
key: json.dumps(value, ensure_ascii=False) if key in columns and not isinstance(value, str) else value
|
||||
for key, value in fields.items()
|
||||
}
|
||||
|
||||
|
||||
class CommunityStore:
|
||||
def get(self, ref: str) -> dict | None:
|
||||
table = get_table("communities")
|
||||
row = table.find_one(slug=ref, deleted_at=None) or table.find_one(uid=ref, deleted_at=None)
|
||||
return decode(row, JSON_COLUMNS)
|
||||
|
||||
def list(self) -> list[dict]:
|
||||
rows = get_table("communities").find(deleted_at=None, order_by=["name"])
|
||||
return [decode(row, JSON_COLUMNS) for row in rows]
|
||||
|
||||
def pending(self, limit: int) -> list[dict]:
|
||||
horizon = iso(now() - timedelta(seconds=LIMITS.run_timeout_seconds))
|
||||
table = get_table("communities")
|
||||
rows = list(table.find(status=STATUS_PENDING, deleted_at=None, order_by=["created_at"]))
|
||||
rows += [row for row in table.find(status=STATUS_LEARNING, deleted_at=None) if (row.get("claimed_at") or "") < horizon]
|
||||
return [decode(row, JSON_COLUMNS) for row in rows[:limit]]
|
||||
|
||||
def claim(self, uid: str) -> dict:
|
||||
return self.update(uid, status=STATUS_LEARNING, claimed_at=iso(now()))
|
||||
|
||||
@staticmethod
|
||||
def credentials(row: dict) -> dict:
|
||||
return {
|
||||
"username": row.get("login_username") or "",
|
||||
"email": row.get("login_email") or "",
|
||||
"password": row.get("login_password") or "",
|
||||
}
|
||||
|
||||
def due(self, limit: int) -> list[dict]:
|
||||
moment = iso(now())
|
||||
rows = get_table("communities").find(status=STATUS_READY, enabled=1, deleted_at=None, order_by=["next_run_at"])
|
||||
due = [row for row in rows if not row.get("next_run_at") or row["next_run_at"] <= moment]
|
||||
return [decode(row, JSON_COLUMNS) for row in due[:limit]]
|
||||
|
||||
def create(self, slug: str, query: str, created_by: str, **fields) -> dict:
|
||||
row = {
|
||||
"uid": generate_uid(),
|
||||
"slug": slug,
|
||||
"query": query,
|
||||
"status": STATUS_PENDING,
|
||||
"enabled": 1,
|
||||
"interval_seconds": LIMITS.default_interval_seconds,
|
||||
"run_count": 0,
|
||||
"failure_count": 0,
|
||||
"created_at": iso(now()),
|
||||
"created_by": created_by,
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
**fields,
|
||||
}
|
||||
get_table("communities").insert(encode(row, JSON_COLUMNS))
|
||||
return self.get(slug)
|
||||
|
||||
def update(self, uid: str, **fields) -> dict:
|
||||
unknown = set(fields) - set(EDITABLE)
|
||||
if unknown:
|
||||
raise KeyError(f"not editable: {sorted(unknown)}")
|
||||
get_table("communities").update({"uid": uid, **encode(fields, JSON_COLUMNS)}, ["uid"])
|
||||
return self.get(uid)
|
||||
|
||||
def delete(self, uid: str, actor: str) -> int:
|
||||
return soft_delete("communities", actor, uid=uid)
|
||||
|
||||
def schedule_next(self, uid: str, interval_seconds: int, jitter_percent: int = 0) -> str:
|
||||
seconds = max(jittered(interval_seconds, jitter_percent), LIMITS.min_interval_seconds)
|
||||
moment = iso(now() + timedelta(seconds=seconds))
|
||||
get_table("communities").update({"uid": uid, "next_run_at": moment}, ["uid"])
|
||||
return moment
|
||||
|
||||
def start_run(self, community_uid: str, trigger: str, dry_run: bool) -> str:
|
||||
uid = generate_uid()
|
||||
get_table("community_runs").insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"community_uid": community_uid,
|
||||
"started_at": iso(now()),
|
||||
"finished_at": "",
|
||||
"status": "running",
|
||||
"dry_run": int(dry_run),
|
||||
"trigger": trigger,
|
||||
"collected": "",
|
||||
"pushed": "",
|
||||
"errors": "",
|
||||
"log": "",
|
||||
}
|
||||
)
|
||||
return uid
|
||||
|
||||
def finish_run(self, uid: str, status: str, **fields) -> None:
|
||||
get_table("community_runs").update(
|
||||
{"uid": uid, "status": status, "finished_at": iso(now()), **encode(fields, RUN_JSON_COLUMNS)}, ["uid"]
|
||||
)
|
||||
|
||||
def runs(self, community_uid: str, limit: int = 20) -> list[dict]:
|
||||
rows = get_table("community_runs").find(community_uid=community_uid, order_by=["-started_at"], _limit=limit)
|
||||
return [decode(row, RUN_JSON_COLUMNS) for row in rows]
|
||||
|
||||
def busy(self, community_uid: str) -> bool:
|
||||
horizon = iso(now() - timedelta(seconds=LIMITS.run_timeout_seconds))
|
||||
rows = get_table("community_runs").find(community_uid=community_uid, status="running")
|
||||
if any(row["started_at"] >= horizon for row in rows):
|
||||
return True
|
||||
row = get_table("communities").find_one(uid=community_uid)
|
||||
return bool(row and row.get("status") == STATUS_LEARNING and (row.get("claimed_at") or "") >= horizon)
|
||||
|
||||
def prune_runs(self, community_uid: str) -> int:
|
||||
table = get_table("community_runs")
|
||||
stale = list(table.find(community_uid=community_uid, order_by=["-started_at"]))[LIMITS.runs_kept :]
|
||||
for row in stale:
|
||||
table.delete(uid=row["uid"])
|
||||
return len(stale)
|
||||
|
||||
def fail(self, row: dict, error: str) -> dict:
|
||||
failures = int(row.get("failure_count") or 0) + 1
|
||||
fields = {"failure_count": failures, "last_error": error[:1000], "last_status": "failed", "last_run_at": iso(now())}
|
||||
if failures >= LIMITS.max_failures:
|
||||
fields.update(status=STATUS_FAILING, enabled=0)
|
||||
return self.update(row["uid"], **fields)
|
||||
|
||||
|
||||
store = CommunityStore()
|
||||
@@ -17,6 +17,7 @@ from .jobs import JOB_ACTIONS
|
||||
from .messages import MESSAGE_ACTIONS
|
||||
from .moderation import MODERATION_ACTIONS
|
||||
from .news import NEWS_ACTIONS
|
||||
from .nodes import NODE_ACTIONS
|
||||
from .notifications import NOTIFICATION_ACTIONS
|
||||
from .posts import POSTS_ACTIONS
|
||||
from .profile import PROFILE_ACTIONS
|
||||
@@ -24,6 +25,8 @@ from .project_files import PROJECT_FILE_ACTIONS
|
||||
from .projects import PROJECTS_ACTIONS
|
||||
from .quizzes import QUIZ_ACTIONS
|
||||
from .skills import SKILL_ACTIONS
|
||||
from .community import COMMUNITY_ACTIONS
|
||||
from .sync import SYNC_ACTIONS
|
||||
from .social import SOCIAL_ACTIONS
|
||||
from .tools import TOOLS_ACTIONS
|
||||
from .uploads import UPLOAD_ACTIONS
|
||||
@@ -53,6 +56,9 @@ CATALOG_GROUPS: dict[str, tuple[Action, ...]] = {
|
||||
"skills": SKILL_ACTIONS,
|
||||
"battles": BATTLE_ACTIONS,
|
||||
"moderation": MODERATION_ACTIONS,
|
||||
"sync": SYNC_ACTIONS,
|
||||
"community": COMMUNITY_ACTIONS,
|
||||
"nodes": NODE_ACTIONS,
|
||||
}
|
||||
|
||||
ACTIONS: tuple[Action, ...] = tuple(
|
||||
|
||||
@@ -189,6 +189,8 @@ ADMIN_ACTIONS: tuple[Action, ...] = (
|
||||
body("maintenance_mode", "Whether maintenance mode is on."),
|
||||
body("maintenance_message", "Maintenance message."),
|
||||
body("happy_404_enabled", "Whether Happy 404 (rendering a random post instead of a 404 page) is on."),
|
||||
body("sync_enabled", "Whether community sync is on (1) or off (0)."),
|
||||
body("sync_origins", "Comma separated origins allowed to sync; empty allows any."),
|
||||
),
|
||||
requires_admin=True,
|
||||
),
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..spec import Action
|
||||
from ._shared import body, confirm, path
|
||||
|
||||
SLUG = path("slug", "Community slug, as returned by community_list.")
|
||||
ACCOUNT = (
|
||||
body("login_username", "Username of an account on that community, for a login-protected community."),
|
||||
body("login_email", "Email of that account, when the community logs in by email."),
|
||||
body("login_password", "Password of that account. Stored for the scheduled collectors, never shown again."),
|
||||
)
|
||||
|
||||
COMMUNITY_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="community_list",
|
||||
method="GET",
|
||||
path="/admin/communities",
|
||||
summary="List the external communities that are synchronised (admin only)",
|
||||
description=(
|
||||
"Every community with its status (pending, learning, ready, login_required, failed, failing), "
|
||||
"schedule, last run and failure count, plus whether community sync and the Community sync "
|
||||
"service are switched on and whether the service is in dry run."
|
||||
),
|
||||
requires_admin=True,
|
||||
),
|
||||
Action(
|
||||
name="community_get",
|
||||
method="GET",
|
||||
path="/admin/communities/{slug}",
|
||||
summary="Show one community: metadata, learned feeds, totals and recent runs (admin only)",
|
||||
description=(
|
||||
"Returns the community's description, rules, language and member count, the feeds the system "
|
||||
"learned (type, address, field mapping, comments mode, pagination), the totals synced so far "
|
||||
"and the log of recent runs including dry runs. Use it to explain why a run failed."
|
||||
),
|
||||
params=(SLUG,),
|
||||
requires_admin=True,
|
||||
),
|
||||
Action(
|
||||
name="community_add",
|
||||
method="POST",
|
||||
path="/admin/communities",
|
||||
summary="Add a community to synchronise by name or address (admin only)",
|
||||
description=(
|
||||
"Starts discovery: the system searches for the community, detects its feeds (RSS, Atom, JSON or "
|
||||
"HTML, with JavaScript rendering when needed), learns the structure once, generates collectors "
|
||||
"and records the community's metadata. Discovery runs in the background; poll community_get until "
|
||||
"the status is ready. Pass an account for a community that requires a login. Members never see "
|
||||
"these communities."
|
||||
),
|
||||
params=(body("query", "Community name or address, for example https://forum.example.com", required=True), *ACCOUNT),
|
||||
requires_admin=True,
|
||||
ajax=True,
|
||||
),
|
||||
Action(
|
||||
name="community_run",
|
||||
method="POST",
|
||||
path="/admin/communities/{slug}/run",
|
||||
summary="Run a community's collectors now, optionally as a dry run (admin only)",
|
||||
description=(
|
||||
"Collects and pushes users, posts and comments through the sync API in the background. With "
|
||||
"dry_run=true everything is validated and logged but nothing is written and the watermark does "
|
||||
"not move. A live run needs the sync_enabled setting. Read the result with community_get."
|
||||
),
|
||||
params=(SLUG, body("dry_run", "Validate and report without writing.", type="boolean")),
|
||||
requires_admin=True,
|
||||
ajax=True,
|
||||
),
|
||||
Action(
|
||||
name="community_relearn",
|
||||
method="POST",
|
||||
path="/admin/communities/{slug}/relearn",
|
||||
summary="Learn a community's structure again, for example after its site changed (admin only)",
|
||||
description="Runs discovery again with the stored account. A failed relearn keeps the working schema.",
|
||||
params=(SLUG,),
|
||||
requires_admin=True,
|
||||
ajax=True,
|
||||
),
|
||||
Action(
|
||||
name="community_update",
|
||||
method="POST",
|
||||
path="/admin/communities/{slug}/update",
|
||||
summary="Change a community's interval, enabled flag or login account (admin only)",
|
||||
description=(
|
||||
"interval_seconds is between 900 and 2592000. An empty password keeps the stored one; "
|
||||
"clear_login=true removes the stored account."
|
||||
),
|
||||
params=(
|
||||
SLUG,
|
||||
body("interval_seconds", "Seconds between runs, 900 to 2592000.", required=True, type="integer"),
|
||||
body("enabled", "Whether the scheduled service runs this community.", type="boolean"),
|
||||
*ACCOUNT,
|
||||
body("clear_login", "Remove the stored account.", type="boolean"),
|
||||
),
|
||||
requires_admin=True,
|
||||
ajax=True,
|
||||
),
|
||||
Action(
|
||||
name="community_delete",
|
||||
method="POST",
|
||||
path="/admin/communities/{slug}/delete",
|
||||
summary="Delete a community from the synchronisation list (admin only)",
|
||||
description=(
|
||||
"Moves the community to the trash and stops its collectors. Users, posts and comments that were "
|
||||
"already synchronised stay."
|
||||
),
|
||||
params=(SLUG, confirm()),
|
||||
requires_admin=True,
|
||||
ajax=True,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,149 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..spec import Action
|
||||
from ._shared import body, confirm, path, query
|
||||
|
||||
DEVICE = path("device_uid", "Device uid, as returned by node_device_list.")
|
||||
ADMIN_DEVICE = path("device_uid", "Device uid, as listed by node_admin_status.")
|
||||
|
||||
NODE_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="social_search",
|
||||
method="GET",
|
||||
path="/nodes/social/search",
|
||||
summary="Search public developer discussion outside this site (Hacker News, Stack Overflow, GitHub, dev.to, Mastodon, Reddit)",
|
||||
description=(
|
||||
"Returns merged hits with platform, title, url, snippet, author, score and comment count. "
|
||||
"Platforms that could not be reached are listed under 'unavailable' with a reason code; report "
|
||||
"that honestly and never invent results for them. Reddit is only reachable when community "
|
||||
"donors are online. Use it only when the user asks what people elsewhere say about something; "
|
||||
"platform tools come first."
|
||||
),
|
||||
params=(
|
||||
query("q", "What to search for.", required=True),
|
||||
query("platforms", "Comma separated subset of hackernews, stackexchange, github, devto, mastodon, reddit. Empty means all."),
|
||||
query("limit", "Hits per platform, 1 to 20."),
|
||||
),
|
||||
read_only=True,
|
||||
),
|
||||
Action(
|
||||
name="node_fetch",
|
||||
method="POST",
|
||||
path="/nodes/internal/fetch",
|
||||
summary="Fetch one public page from an allowlisted site through the community donor network",
|
||||
description=(
|
||||
"Only hosts on the published allowlist are accepted (today reddit.com and redd.it); anything else is "
|
||||
"refused with a reason. Returns status, text and whether the body was truncated. It fails with "
|
||||
"no_nodes when no donor is online; say so instead of guessing the page content."
|
||||
),
|
||||
params=(
|
||||
body("url", "Absolute https URL on an allowlisted host.", required=True),
|
||||
body("cache", "Allow a cached copy.", type="boolean"),
|
||||
),
|
||||
read_only=True,
|
||||
),
|
||||
Action(
|
||||
name="node_status",
|
||||
method="GET",
|
||||
path="/nodes/me",
|
||||
summary="Show your community donation status: consent, settings, devices and what you donated so far",
|
||||
description="Reports whether you may donate right now and, if not, the reason.",
|
||||
read_only=True,
|
||||
),
|
||||
Action(
|
||||
name="node_settings_set",
|
||||
method="POST",
|
||||
path="/nodes/settings",
|
||||
summary="Change your community donation settings (bandwidth, hours, platforms)",
|
||||
description=(
|
||||
"Donation can only be switched ON by the member in their own profile, never through this tool; "
|
||||
"'enabled': true is refused. Switching it off, and every other setting, is allowed."
|
||||
),
|
||||
params=(
|
||||
body("enabled", "Set false to stop donating.", type="boolean"),
|
||||
body("bandwidth_mb_day", "Daily bandwidth cap in megabytes.", type="integer"),
|
||||
body("max_concurrent", "Simultaneous requests.", type="integer"),
|
||||
body("active_from", "Start of the active window, HH:MM."),
|
||||
body("active_to", "End of the active window, HH:MM."),
|
||||
body("timezone", "IANA timezone for the active window, for example Europe/Amsterdam."),
|
||||
body("pause_on_battery", "Stop donating on battery power.", type="boolean"),
|
||||
body("public_display", "Show your name on the donor leaderboard.", type="boolean"),
|
||||
),
|
||||
ajax=True,
|
||||
),
|
||||
Action(
|
||||
name="node_device_list",
|
||||
method="GET",
|
||||
path="/nodes/devices",
|
||||
summary="List your donor devices",
|
||||
description="Shows each device with its status and when it last connected. Secrets are never shown.",
|
||||
read_only=True,
|
||||
),
|
||||
Action(
|
||||
name="node_device_revoke",
|
||||
method="POST",
|
||||
path="/nodes/devices/{device_uid}/delete",
|
||||
summary="Remove one of your donor devices",
|
||||
description="A connected device is disconnected within seconds and its secret stops working for good.",
|
||||
params=(DEVICE, confirm()),
|
||||
ajax=True,
|
||||
),
|
||||
Action(
|
||||
name="node_admin_status",
|
||||
method="GET",
|
||||
path="/admin/nodes",
|
||||
summary="Show the community node network: connected pool, queue, devices, open flags, settings and the kill switch",
|
||||
description=(
|
||||
"Administrator overview. Lists the 100 most recently seen devices with their owner and status, "
|
||||
"the open flags, the queue depth and the caps in force. Use the device uids it returns for "
|
||||
"node_admin_suspend and node_admin_unsuspend."
|
||||
),
|
||||
requires_admin=True,
|
||||
read_only=True,
|
||||
),
|
||||
Action(
|
||||
name="node_admin_suspend",
|
||||
method="POST",
|
||||
path="/admin/nodes/devices/{device_uid}/suspend",
|
||||
summary="Suspend one donor device for a fixed number of minutes with a stated reason",
|
||||
description=(
|
||||
"A suspended device is disconnected within seconds and cannot reconnect until the period ends or "
|
||||
"it is unsuspended. The account itself is untouched. Show the administrator the exact device and "
|
||||
"owner, get explicit confirmation, then call again with confirm=true."
|
||||
),
|
||||
requires_admin=True,
|
||||
params=(
|
||||
ADMIN_DEVICE,
|
||||
body("minutes", "Suspension length in minutes, 1 to 10080.", required=True, type="integer"),
|
||||
body("reason", "Reason recorded with the suspension."),
|
||||
confirm(),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="node_admin_unsuspend",
|
||||
method="POST",
|
||||
path="/admin/nodes/devices/{device_uid}/unsuspend",
|
||||
summary="Lift the suspension of one donor device",
|
||||
description="The device may reconnect immediately. Reversible, so no confirmation is needed.",
|
||||
requires_admin=True,
|
||||
params=(ADMIN_DEVICE,),
|
||||
),
|
||||
Action(
|
||||
name="node_admin_killswitch",
|
||||
method="POST",
|
||||
path="/admin/nodes/killswitch",
|
||||
summary="Switch community donation on or off for the whole platform",
|
||||
description=(
|
||||
"Off disconnects every donor node within seconds and stops new jobs; on lets consenting donors "
|
||||
"connect again. Omit 'enabled' to flip the current state. Show the administrator the current state "
|
||||
"and the effect, get explicit confirmation, then call again with confirm=true."
|
||||
),
|
||||
requires_admin=True,
|
||||
params=(
|
||||
body("enabled", "True to switch donation on, false to switch it off. Omit to flip it.", type="boolean"),
|
||||
confirm(),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,44 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..spec import Action
|
||||
from ._shared import body
|
||||
|
||||
ENVELOPE = (
|
||||
body("origin", "Stable id of the sending community, for example other.example.org.", required=True),
|
||||
body("records", "JSON array string of at most 50 records, for example [{\"origin_uid\": \"1\", ...}]. Field names are listed at /sync/spec.json.", required=True),
|
||||
body("dry_run", "Validate and report what would happen without writing.", type="boolean"),
|
||||
)
|
||||
|
||||
|
||||
def sync_action(kind: str, summary: str, description: str) -> Action:
|
||||
return Action(
|
||||
name=f"sync_{kind}",
|
||||
method="POST",
|
||||
path=f"/sync/{kind}",
|
||||
summary=summary,
|
||||
description=description,
|
||||
ajax=True,
|
||||
params=ENVELOPE,
|
||||
requires_admin=True,
|
||||
)
|
||||
|
||||
|
||||
SYNC_ACTIONS: tuple[Action, ...] = (
|
||||
sync_action(
|
||||
"users",
|
||||
"Upsert users copied from another community",
|
||||
"Creates or updates synced accounts keyed on each record's origin_uid. Counters, email and role are ignored. Send users before posts.",
|
||||
),
|
||||
sync_action(
|
||||
"posts",
|
||||
"Upsert posts copied from another community",
|
||||
"Creates or updates posts; author_uid is the author's origin_uid and must already be synced.",
|
||||
),
|
||||
sync_action(
|
||||
"comments",
|
||||
"Upsert comments copied from another community",
|
||||
"Creates or updates comments; author_uid, post_uid and any parent_uid are origin ids that must already be synced.",
|
||||
),
|
||||
)
|
||||
@@ -33,6 +33,10 @@ from .spec import Action, Catalog
|
||||
MUTATING_METHODS = ("POST", "DELETE", "PUT", "PATCH")
|
||||
|
||||
CONFIRM_REQUIRED = {
|
||||
"community_delete",
|
||||
"node_device_revoke",
|
||||
"node_admin_suspend",
|
||||
"node_admin_killswitch",
|
||||
"join_battle",
|
||||
"fight_battle",
|
||||
"delete_my_account",
|
||||
@@ -241,6 +245,31 @@ def confirmation_error(name: str, arguments: dict[str, Any]) -> ToolInputError |
|
||||
f"Deleting comment '{uid}' is permanent and cannot be undone. "
|
||||
"Ask the user to confirm explicitly, then call again with confirm=true."
|
||||
)
|
||||
if name == "community_delete":
|
||||
slug = str(arguments.get("slug", "")).strip() or "(unspecified)"
|
||||
return ToolInputError(
|
||||
f"Deleting community '{slug}' stops its synchronisation and moves it to the trash; content already "
|
||||
"synchronised stays. Show the user the community, get explicit confirmation, then call again with confirm=true."
|
||||
)
|
||||
if name == "node_device_revoke":
|
||||
uid = str(arguments.get("device_uid", "")).strip() or "(unspecified)"
|
||||
return ToolInputError(
|
||||
f"Removing donor device '{uid}' disconnects it and permanently invalidates its secret. "
|
||||
"Show the user the device, get explicit confirmation, then call again with confirm=true."
|
||||
)
|
||||
if name == "node_admin_suspend":
|
||||
uid = str(arguments.get("device_uid", "")).strip() or "(unspecified)"
|
||||
return ToolInputError(
|
||||
f"Suspending donor device '{uid}' disconnects it within seconds and blocks it from reconnecting for "
|
||||
"the whole period. Show the administrator the exact device and owner, get explicit confirmation, "
|
||||
"then call again with confirm=true."
|
||||
)
|
||||
if name == "node_admin_killswitch":
|
||||
return ToolInputError(
|
||||
"The community donation switch is global: turning it off disconnects every donor node within "
|
||||
"seconds and stops all node jobs. Show the administrator the current state and the effect, get "
|
||||
"explicit confirmation, then call again with confirm=true."
|
||||
)
|
||||
if name == "delete_gist":
|
||||
uid = str(arguments.get("uid", "")).strip() or "(unspecified)"
|
||||
return ToolInputError(
|
||||
|
||||
@@ -197,6 +197,15 @@ SYSTEM_PROMPT = (
|
||||
"information the platform cannot provide and the user wants it. Never use them to answer questions "
|
||||
"about this DevPlace instance, its users, posts, settings, or metrics - those have dedicated "
|
||||
"platform tools. When platform tools can serve the request, do not call rsearch.\n\n"
|
||||
"SOCIAL SEARCH AND THE DONOR NETWORK\n"
|
||||
"social_search looks up public developer discussion on Hacker News, Stack Overflow, GitHub, dev.to, "
|
||||
"Mastodon and, when community donors are online, Reddit. Use it only when the user asks what people "
|
||||
"outside this site say about a topic; platform tools and data come first. Read its 'unavailable' map "
|
||||
"and tell the user which platforms could not be reached; never claim results from a platform that is "
|
||||
"listed there and never invent a platform it does not cover (X, Instagram, LinkedIn and TikTok are not "
|
||||
"reachable). node_fetch reads one page from an allowlisted site through donors. node_status, "
|
||||
"node_settings_set, node_device_list and node_device_revoke manage the member's own donation; donation "
|
||||
"can only be switched on by the member in their profile, never by you.\n\n"
|
||||
"EMAIL TOOLS (email_*)\n"
|
||||
"The email_* tools connect to the user's OWN external mailbox over IMAP/SMTP, not this "
|
||||
"platform, and are only available to signed-in users. The connection must be configured first "
|
||||
|
||||
@@ -74,6 +74,9 @@ GROUP_LABELS: dict[str, str] = {
|
||||
"skills": "Skills",
|
||||
"battles": "Opinion Wars",
|
||||
"moderation": "Moderation",
|
||||
"sync": "Community Sync",
|
||||
"community": "Community Discovery (admin)",
|
||||
"nodes": "Community Donation and Social Search",
|
||||
"tasks": "Scheduled Tasks and Reminders",
|
||||
"agentic": "Self-Evaluation (delegate, eval)",
|
||||
"avatar": "Browser Avatar Tutorials",
|
||||
|
||||
@@ -15,6 +15,7 @@ from urllib.parse import urlparse
|
||||
import httpx
|
||||
|
||||
from devplacepy import stealth
|
||||
from devplacepy.config import INTERNAL_BASE_URL
|
||||
from devplacepy.net_guard import BlockedAddressError, guard_public_url, guarded_async_client
|
||||
|
||||
from .extract import extract_html, relevant_links
|
||||
@@ -155,6 +156,51 @@ async def search_queries(
|
||||
return _interleave(buckets)
|
||||
|
||||
|
||||
SOCIAL_QUERIES = 2
|
||||
SOCIAL_HITS = 4
|
||||
SOCIAL_TIMEOUT_SECONDS = 40.0
|
||||
|
||||
|
||||
async def social_candidates(
|
||||
queries: list[str], api_key: str, emit: Callable[[dict], None] = lambda frame: None
|
||||
) -> list[dict]:
|
||||
if not api_key:
|
||||
return []
|
||||
found: list[dict] = []
|
||||
timeout = httpx.Timeout(SOCIAL_TIMEOUT_SECONDS, connect=10.0)
|
||||
async with stealth.stealth_async_client(
|
||||
base_url=INTERNAL_BASE_URL,
|
||||
headers={"Authorization": f"Bearer {api_key}", "Accept": "application/json"},
|
||||
timeout=timeout,
|
||||
) as client:
|
||||
for query in queries[:SOCIAL_QUERIES]:
|
||||
try:
|
||||
response = await client.get(
|
||||
"/nodes/social/search", params={"q": query, "limit": SOCIAL_HITS}
|
||||
)
|
||||
emit({"type": "social", "success": response.status_code < 400})
|
||||
if response.status_code >= 400:
|
||||
continue
|
||||
data = response.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
emit({"type": "social", "success": False})
|
||||
logger.info("deepsearch social search failed for %r: %s", query, exc)
|
||||
continue
|
||||
for hit in data.get("hits") or []:
|
||||
url = (hit.get("url") or "").strip()
|
||||
if url:
|
||||
found.append(
|
||||
{
|
||||
"url": url,
|
||||
"title": hit.get("title") or "",
|
||||
"description": hit.get("snippet") or "",
|
||||
"content": hit.get("snippet") or "",
|
||||
"query": query,
|
||||
}
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
def _snippet_page(candidate: dict, depth: int) -> CrawledPage | None:
|
||||
snippet = _clean_snippet(candidate.get("content") or candidate.get("description") or "")
|
||||
if len(snippet) < SNIPPET_MIN_CHARS:
|
||||
|
||||
@@ -43,6 +43,14 @@ class DeepsearchService(JobService):
|
||||
help="Model sent to the internal AI gateway for query planning and report synthesis. Blank uses the gateway default model.",
|
||||
group="DeepSearch",
|
||||
),
|
||||
ConfigField(
|
||||
"deepsearch_social",
|
||||
"Include social search",
|
||||
type="bool",
|
||||
default=False,
|
||||
help="Add public discussion from Hacker News, Stack Overflow, GitHub, dev.to, Mastodon and (when donors are online) Reddit to the sources of a signed-in member's research. Off by default.",
|
||||
group="DeepSearch",
|
||||
),
|
||||
]
|
||||
|
||||
def session_dir(self, uid: str) -> Path:
|
||||
@@ -68,6 +76,9 @@ class DeepsearchService(JobService):
|
||||
payload["collection"] = self.collection_name(uid)
|
||||
payload["model"] = self.get_config()["deepsearch_model"] or INTERNAL_MODEL
|
||||
payload["cached_hashes"] = self._cached_hashes(database)
|
||||
payload["social"] = bool(
|
||||
job.get("owner_kind") == "user" and self.get_config()["deepsearch_social"]
|
||||
)
|
||||
payload_path = output_dir / "payload.json"
|
||||
await asyncio.to_thread(payload_path.write_text, json.dumps(payload), encoding="utf-8")
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from devplacepy.services.deepsearch.store import Chunk, VectorStore
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
from .chunking import chunk_text
|
||||
from .crawl import content_hash, crawl, search_queries, url_hash
|
||||
from .crawl import content_hash, crawl, search_queries, social_candidates, url_hash
|
||||
from .enhance import plan_followup_queries, plan_queries
|
||||
from .orchestrate import orchestrate
|
||||
from .phases import (
|
||||
@@ -174,6 +174,10 @@ async def _run(payload: dict, output_dir: Path) -> dict:
|
||||
|
||||
_stage("searching", "Searching the web", PHASE_SEARCHING)
|
||||
candidates = await search_queries(queries, _emit)
|
||||
if payload.get("social"):
|
||||
known = {item["url"] for item in candidates}
|
||||
extra = await social_candidates(queries, api_key, _emit)
|
||||
candidates += [item for item in extra if item["url"] not in known]
|
||||
_emit({"type": "candidates", "count": len(candidates)})
|
||||
|
||||
_stage("crawling", "Crawling sources", PHASE_CRAWLING)
|
||||
|
||||
@@ -64,6 +64,7 @@ OWNED_TABLES: tuple[tuple[str, str], ...] = (
|
||||
("node_settings", "user_uid"),
|
||||
("node_donors", "user_uid"),
|
||||
("node_usage_daily", "user_uid"),
|
||||
("node_caller_daily", "caller_id"),
|
||||
)
|
||||
|
||||
CHILD_TABLES: tuple[tuple[str, str, str, str], ...] = (
|
||||
|
||||
@@ -2,7 +2,7 @@ This file documents the community node network (`devplacepy/services/nodes/`, `d
|
||||
|
||||
## What exists today
|
||||
|
||||
Phase 1 (the specification), Phase 2 (the data layer and usage ledger) and Phase 3 (registry, session, consent gate, kill switch, device and ticket routes, the lock-owner socket) are built. No job traffic flows yet. Everything else is planned and listed under "Gaps" next to the phase that closes it. Nothing here is enabled for any account.
|
||||
The specification, data layer, registry, session, consent gate, kill switch, destination policy, queue, scheduler, dispatcher, circuit breakers, cache, the fabric facade, tunnel mode, social search adapters, the Go client, the browser extension, the Donate tab, the admin page, the Devii tools, the CLI and the docs are built and tested. The gaps below list what is unmeasured or missing. Nothing here is enabled for any account.
|
||||
|
||||
| Piece | File | State |
|
||||
|-------|------|-------|
|
||||
@@ -10,7 +10,7 @@ Phase 1 (the specification), Phase 2 (the data layer and usage ledger) and Phase
|
||||
| `GET /nodes/spec.json` | `routers/nodes/spec.py` | built, tested, public |
|
||||
| Generated protocol docs page | `templates/docs/nodes-protocol.html` | built, reads the `nodes_spec` Jinja global |
|
||||
| API docs group | `docs_api/groups/nodes.py` | built, one endpoint |
|
||||
| Tables and helpers | `database/nodes.py`, `NODES_COLUMNS` in `database/schema.py` | built, tested, no route uses them yet |
|
||||
| Tables and helpers | `database/nodes.py`, `NODES_COLUMNS` in `database/schema.py` | built, tested |
|
||||
| Usage ledger | `ledger.py` (`UsageLedger`, `DonationSummary`) | built, tested |
|
||||
| `bandwidth_sharing` consent kind | `CONSENT_KINDS` in `database/moderation.py` | registered, never granted by default, enforced by `ConsentGate` |
|
||||
| Frame codec | `frames.py` (`decode`, `encode`, `FrameError`) | built, tested, validates every frame against `spec.py` |
|
||||
@@ -20,6 +20,22 @@ Phase 1 (the specification), Phase 2 (the data layer and usage ledger) and Phase
|
||||
| Supervisor | `service.py` (`NodesService`) | built, registered in `main.py`, default enabled |
|
||||
| Device, ticket and socket routes | `routers/nodes/devices.py`, `ticket.py`, `ws.py` | built, tested |
|
||||
| nginx upgrade block for `/nodes/ws` | `nginx/nginx.conf.template` | added |
|
||||
| Destination policy | `policy.py` (`DestinationPolicy`, `Target`, `is_public_address`) | built, tested |
|
||||
| Result cache | `cache.py` (`ResultCache`) | built, tested, per process |
|
||||
| Circuit breaker | `circuit.py` (`CircuitBreaker`) | built, tested |
|
||||
| Scheduler | `scheduler.py` (`NodeScheduler`, `in_active_hours`) | built, tested |
|
||||
| Queue and broker | `broker.py` (`JobBroker`), `database/nodes.py` request helpers | built, tested |
|
||||
| Dispatcher | `dispatcher.py` (`NodeDispatcher`) | built, tested, runs in the lock owner |
|
||||
| Fabric facade | `fabric.py` (`NodeFabric`, `FabricResult`, `FabricError`) | built, tested, used by `social.py` and `POST /nodes/internal/fetch` |
|
||||
| Admin page and actions | `routers/admin/nodes.py`, `templates/admin_nodes.html`, `static/js/AdminNodes.js` | built |
|
||||
|
||||
## Administration (`/admin/nodes`)
|
||||
|
||||
`GET /admin/nodes` (HTML and JSON `AdminNodesOut`) is the administrator view: pool counters read from the `nodes` row in `service_state` (so any worker answers, at most one persist interval stale; `pool.online` is false when the heartbeat is stale), queue depth and counts by status, device counts by status, kind, address class and country, the current UTC day's totals, open flags and the 100 most recently seen devices (both capped), the kill switch and the six settings with defaults and ceilings. **Per-destination circuit state is not shown:** it lives in memory on the lock-owner worker and only the number of open circuits is persisted, so a per-destination view would need a new persisted snapshot. Device counts use the stored status, so a suspension that has expired still counts as `suspended` until the next write; the recent-devices list uses `effective_status`.
|
||||
|
||||
POST routes, all `require_admin`, all audited: `/killswitch` (`node.killswitch`, flips or sets `nodes_enabled`), `/settings` (`node.admin.settings`, only submitted fields change, values above the ceiling are clamped and reported under `clamped`), `/devices/{uid}/suspend` (`node.suspend`, 1 to 10080 minutes) and `/unsuspend` (`node.unsuspend`), `/flags/{uid}/resolve` (`node.flag.resolve`). The per-device and per-flag routes are gated by `is_senior_admin` against the device or flag owner and answer through `deny_senior` (audited `denied`). Suspension and the kill switch take effect within one `reconcile` tick on whichever worker holds the sockets. Ceilings: `nodes_max_total` and `nodes_cache_ttl_seconds` use `LimitSpec`; the quota ceilings are constants in the router because `LimitSpec` only carries defaults for them.
|
||||
|
||||
The page refreshes every 10 seconds by fetching its own HTML and swapping the `[data-live]` sections (`AdminNodes.js`, `app.adminNodes`); the settings form and the suspend modal are never swapped, so typing is not interrupted. Devii: `node_admin_status`, `node_admin_suspend` and `node_admin_killswitch` (the last two in `CONFIRM_REQUIRED`), `node_admin_unsuspend`, all `requires_admin`.
|
||||
|
||||
## The specification is the contract
|
||||
|
||||
@@ -65,6 +81,22 @@ Seven tables, all ensured with their full column set in `init_db` (`_ensure_node
|
||||
5. `hello` must arrive within `hello_timeout_seconds`, with a supported `protocol`, a `kind` equal to the registered device kind and a non-empty list of known `modes`. The server answers `welcome` with the full limits and allowlist from the spec. A second connection of the same device kicks the first with `replaced`.
|
||||
6. The session pings every `ping_seconds`; any frame from the client resets the missed counter and `missed_pings_before_evict` unanswered pings close it with `heartbeat`. `usage` frames are advisory and only summed on the session. Every other client frame is a protocol error until Phase 4 assigns jobs.
|
||||
|
||||
## Jobs (Phase 4)
|
||||
|
||||
A job is a row in `node_requests`, so any worker can ask and only the lock owner serves. Life cycle: `queued` -> `assigned` -> `done`, or `failed` / `expired`; every transition is one conditional `UPDATE`.
|
||||
|
||||
1. `NodeFabric.fetch(url, caller_kind=, caller_id=)` (any worker): policy check, result cache, kill switch and "any node online" (read from the `nodes` row in `service_state`, so it is at most one persist interval stale), then the **caller's daily quota**, then enqueue, then poll the row every `poll_interval_ms` until it is terminal or the caller's wait ends. One retry on `node_disconnected`, `timeout`, `send_failed`, `concurrency_exceeded`, `outside_hours`, `bandwidth_exhausted`; the retry does not spend a second quota unit. Bounded: queue depth (`max_queue_depth`, a conditional `INSERT`), `max_wait_seconds`, `queue_max_wait_seconds`.
|
||||
2. `NodeDispatcher.tick` (lock owner, every `dispatcher_tick_ms`, started by `ensure_dispatching()` from the socket route or `on_enable`): expire, sweep timed-out in-flight jobs, then for each queued row re-check the policy, ask the scheduler for a node, claim the row atomically, send the `job` frame.
|
||||
3. The node answers with `job_result` or `job_error`. `Session._handle` only accepts an answer for a job it has in flight; anything else is a protocol error and closes the socket.
|
||||
|
||||
**Scheduler eligibility** (every one must hold): socket open and not paused, speaks a transport the job allows (`tunnel` is preferred over `fetch` for `any`; a node that speaks neither gets nothing), is not the caller's own account, below `min(server, donor setting, hello limits)` concurrency, the destination's adapter is in the donor's `allowed_platforms`, inside the donor's active hours in the donor's timezone (windows may wrap midnight), below the donor's daily byte cap, at least `min_seconds_per_node_domain` since this node last started a job for the destination suffix, node circuit closed. Globally the destination needs `1/max_rps` seconds since its last job and a closed destination circuit. Choice: fewest in-flight, then least recently assigned, then random. Per-destination state is keyed by the allowlist suffix, so `a.reddit.com` ... `z.reddit.com` cannot grow it.
|
||||
|
||||
**Trust.** `via` is always `fetch` for node results. A result is checked before it is stored: status 100-599, strict base64, body capped at `max_body_bytes` (truncated and flagged, never silent), headers reduced to at most 64 short strings with `set-cookie`, `set-cookie2`, `authorization` and `proxy-authenticate` dropped. A malformed result is an **integrity failure**: the request fails, `integrity_failures` is counted on the device and the process, a `node_flags` row is raised for a human, and the node is NOT auto-banned. A donor refusing a destination the server allowed raises a `refusal_mismatch` flag. Donor limit codes (`bandwidth_exhausted`, `outside_hours`, `concurrency_exceeded`) pause the node instead of blaming it. HTTP 429 and 5xx count against the destination circuit (they are the signal that stops hammering a rate-limiting site); the result is still returned to the caller. Sampled redundancy between two donors is not built (Phase 6B).
|
||||
|
||||
**Accounting.** A successful job calls `record_job` (daily bucket and lifetime totals in one transaction). If the donor's cap would be crossed the result is still delivered but the node is stopped for the day. Failures call `record_failure`. Bytes up are estimated (`len(url) + 256`) because the server never sees the donor's real request.
|
||||
|
||||
**Measured** (scratch SQLite on this machine, 2000 synthetic sessions, 500 queued requests, batch 50): an idle tick costs 0.1 ms; a tick where no node is eligible because every node is outside its hours cost 225 ms before two fixes (the active-hours window is now cached per minute and a destination found saturated is scanned once per tick, not once per queued request) and 8.4 ms after; a destination held back by its rate ceiling costs 4.7 ms. Races with 16 to 60 real processes on fresh rows: the queue cap admitted exactly the remaining slots, one request was claimed by exactly one assignment, a caller quota of 5 admitted exactly 5. These are one machine and synthetic sessions, not a soak test.
|
||||
|
||||
## The consent choke point
|
||||
|
||||
`ConsentGate` is the only place that decides whether a member's node may exist. `allows_many(user_uids)` decides a whole set with a fixed number of queries and `allows(uid)` is the one-element case of it, so the single and batch answers cannot diverge (a test compares them). Order of reasons: `kill_switch`, `no_account`, `deletion_pending`, `account_inactive`, `suspended`, `not_adult`, `no_consent`, `not_enabled`. It is called at device registration, ticket issue, socket accept and every supervisor tick. An account whose `age_band` is empty (created before age bands existed) is refused as not adult, loudly, rather than assumed adult.
|
||||
@@ -77,13 +109,32 @@ Withdrawal is member-only: `MEMBER_ONLY_CONSENTS` in `database/moderation.py` li
|
||||
|
||||
`tunnel` (native client) relays raw TCP so TLS runs end to end between DevPlace and the destination: the donor cannot read or forge content. `fetch` (browser extension, which has no raw sockets) lets the donor read and forge the response, so every `fetch` result is verified and labelled. A plain web page or a service worker cannot do either (CORS, no sockets) and is out of scope for good.
|
||||
|
||||
## Gaps (what the next phases close)
|
||||
## Donate tab (member UI)
|
||||
|
||||
- Phase 2 leftovers: the `bandwidth_sharing` consent can currently be granted through `POST /profile/{username}/consent` and by Devii's `set_consent` (Devii calls that route as the user). Nothing reads it yet, so it is inert, but Phase 3 must make Devii unable to grant it (Devii may read and withdraw) at the same time as `ConsentGate` lands. No XP or badge is awarded yet (`UsageLedger.claim_xp` only reserves the daily budget); the privacy page and app-store declarations are updated with the first phase that lets a device connect, because until then no personal data is collected.
|
||||
- Phase 3 leftovers: `country`, `asn` and `asn_class` on a device are never filled (they stay `unknown`), so the datacenter exclusion (D6) is not enforced yet; it needs the offline ASN database from Phase 4. `last_seen_at` is written on connect only, not per heartbeat, so a device list cannot yet show who is online right now. The socket counters are per process and are shown in the service metrics only. The `devplace.net` front proxy idle timeout for a long-lived socket is unmeasured.
|
||||
- Phase 4: broker, scheduler, dispatcher, destination policy, circuit breakers, result cache.
|
||||
- Phase 5 onward: the reference client, adapters, the Go client, the extension, the profile UI, Devii tools, the CLI, the privacy and donor-terms pages, the legal gate.
|
||||
- Not measured yet: Reddit and X success rates per address class, WebSocket idle timeouts through `pravda.education` and `devplace.net`, dispatcher tick cost against SQLite, socket capacity of one worker. The caps in `LimitSpec` are starting ceilings, not measurements.
|
||||
The owner-only `Donate` tab on the profile (`?tab=donate`, `templates/_node_donation.html`, `static/js/NodeDonation.js` as `app.nodeDonation`, `static/css/nodes.css`) is a thin client of the routes above: it renders `/nodes/me` server-side (context key `node_me`, only for the owner on that tab, `noindex,follow`), then polls `GET /nodes/me` every 30 seconds with the shared `Poller`. Consent goes through `POST /profile/{username}/consent`, never a second consent route. The device secret exists only in the browser memory and DOM until "I have saved it" is pressed; the install lines are built client-side from `GET /nodes/client-version`. Prose for members is `templates/docs/nodes.html`; the data categories are in `templates/docs/privacy.html`.
|
||||
|
||||
## Tunnel mode (`tunnel.py`)
|
||||
|
||||
For a `tunnel` or `any` job the scheduler prefers a node whose `hello.modes` include `tunnel`. The dispatcher marks the transport on the row (`node_requests.mode` becomes the transport actually used, which is how `FabricResult.via` says `tunnel` or `fetch`), then runs the request itself in the lock owner: `TunnelBridge.fetch` opens a one-shot listener on `127.0.0.1:<ephemeral>`, points a stealth (curl_cffi, Chrome fingerprint) client at it as an HTTP proxy, and the `Relay` answers exactly one `CONNECT host:port` for the job's own host and port (anything else is `403`, a second `CONNECT` is `405`, a donor that refuses is `502`). Bytes are relayed as `tunnel_data` frames (48 KiB chunks, base64, both directions, `RELAY_CAP_BYTES` total, `queue_full` and `bad_base64` end the tunnel with a code). TLS runs between DevPlace and the destination, so the donor sees the host and ciphertext and cannot read or forge the page; the ledger counts the real bytes relayed. Late `tunnel_*` frames for a closed relay are ignored (the donor legitimately sends `tunnel_close` after the server did). Redirects are not followed; the caller gets the 3xx. In `fetch` mode the `job` frame now carries `JOB_HEADERS` (a descriptive bot user agent), because a Go or browser client would otherwise send its own.
|
||||
|
||||
## Social search (`social.py`, `adapters/`)
|
||||
|
||||
`SocialSearch.search` fans one query out to adapters in parallel (`asyncio.gather`, 20 s each) and merges hits round-robin. Direct adapters (`adapters/direct.py`: Hacker News, Stack Overflow, GitHub issues, dev.to by tag, Mastodon by hashtag) need no donors. `adapters/reddit.py` is node-backed and only used when `nodes_social_via_nodes` is `1`; otherwise it is reported as `nodes_not_used`. Every adapter parses a response structurally and raises `AdapterError` on a forged or wrong shape; platforms that fail are listed under `unavailable` with a code and never filled with invented results. **Measured on this machine (2026-10-15, datacenter address):** Hacker News Algolia, Stack Exchange, GitHub search, dev.to tag API and Mastodon hashtag timelines answer; Bluesky `searchPosts` returns 403 without auth, Mastodon `/api/v2/search` returns nothing without login, Lobsters search sits behind a bot wall, dev.to has no usable text search, and Reddit returns 403 to this address. Those are why Bluesky and Lobsters are not adapters. Surfaces: `GET /nodes/social/search` (daily per-member quota `nodes_daily_social_user`, default 300), Devii `social_search`, the DeepSearch `deepsearch_social` option (off by default, signed-in owners only, goes through the HTTP route with the owner's key).
|
||||
|
||||
## Clients
|
||||
|
||||
`nodes/client/` is the Go donor client (standard library only, `build.sh` cross-compiles five targets and writes `manifest.json`). `nodes/extension/` is the MV3 extension (Chrome and Firefox, fetch mode only, 49 `node --test` tests, one of which reads `spec.py` and fails on drift). `make node-client` builds both into `data/node_clients/`, which `GET /nodes/client-version` and `GET /nodes/download/{artifact}` serve (names must match a strict pattern, symlinks and the manifest are never served, the SHA-256 is computed from the file). **Verified end to end on 2026-10-15** against a scratch server on this machine: the Go client logged in, connected, answered a `tunnel` job for `reddit.com` (`via: tunnel`, status 403 because the donor was this same datacenter address) and stayed connected across repeated jobs. Not verified: a residential donor, the extension in a real browser, Windows and macOS builds running.
|
||||
|
||||
## Gaps (what is not built or not measured)
|
||||
|
||||
- **Reddit viability is unmeasured from a residential line.** The only real run used a datacenter donor and got 403, which proves the path, not the success rate. X, Instagram, LinkedIn, Facebook and TikTok stay unreachable (login walls); nothing here claims them.
|
||||
- `country`, `asn` and `asn_class` on a device are never filled, so the datacenter exclusion (D6) is not enforced: the scheduler treats every node as eligible. It needs an offline ASN database file.
|
||||
- `last_seen_at` is written on connect only. Socket counters are per process. The fabric cache is per process. Circuit breaker state is in memory. The same-process fast path is not built, callers poll at `poll_interval_ms`.
|
||||
- Fetch-mode results are not cross-checked (sampled redundancy between two donors is not built). They are structurally checked and labelled `via: fetch`, and `tunnel` is preferred whenever a node can do it.
|
||||
- `DeepSearch` still skips hostile domains with a static list; Reddit URLs from social search are used through their snippet. Routing a hostile URL fetch through `/nodes/internal/fetch` is not built.
|
||||
- The Go client has no battery-aware pause (`pause_on_battery` is stored and sent but ignored by it), and treats a timed suspension as permanent until restarted. The extension enforces its own local hours in the browser clock, not the server timezone.
|
||||
- Not measured: Reddit and X success rates per address class, WebSocket idle timeouts through `pravda.education` and `devplace.net`, socket capacity of one worker, a 24 hour soak. The caps in `LimitSpec` are starting ceilings, not measurements.
|
||||
- **Launch is blocked on a recorded legal decision** (platform terms, donor terms, ISP terms, jurisdiction). Nothing here is enabled for any account: `nodes_enabled` is `0` until an administrator switches it on.
|
||||
|
||||
## Rules for work in this directory
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,101 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
TITLE_MAX = 300
|
||||
SNIPPET_MAX = 500
|
||||
NAME_MAX = 80
|
||||
MAX_BODY_BYTES = 2 * 1024 * 1024
|
||||
TAG = re.compile(r"<[^>]+>")
|
||||
WS = re.compile(r"\s+")
|
||||
TAG_WORD = re.compile(r"[a-z0-9]{2,30}")
|
||||
|
||||
|
||||
class AdapterError(Exception):
|
||||
def __init__(self, code: str, message: str = ""):
|
||||
super().__init__(message or code)
|
||||
self.code = code
|
||||
self.message = message or code
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SocialHit:
|
||||
platform: str
|
||||
title: str
|
||||
url: str
|
||||
snippet: str = ""
|
||||
author: str = ""
|
||||
score: int = 0
|
||||
comments: int = 0
|
||||
created_at: str = ""
|
||||
via: str = "direct"
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return dict(self.__dict__)
|
||||
|
||||
|
||||
def clean(text: Any, limit: int) -> str:
|
||||
if not isinstance(text, str):
|
||||
return ""
|
||||
plain = html.unescape(TAG.sub(" ", text))
|
||||
return WS.sub(" ", plain).strip()[:limit]
|
||||
|
||||
|
||||
def number(value: Any) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return 0
|
||||
return max(0, int(value))
|
||||
|
||||
|
||||
def tag_for(query: str) -> str:
|
||||
words = TAG_WORD.findall(query.lower())
|
||||
return max(words, key=len) if words else ""
|
||||
|
||||
|
||||
def load_json(body: bytes) -> Any:
|
||||
if len(body) > MAX_BODY_BYTES:
|
||||
raise AdapterError("body_too_large")
|
||||
try:
|
||||
return json.loads(body)
|
||||
except ValueError:
|
||||
raise AdapterError("not_json", "the response is not JSON")
|
||||
|
||||
|
||||
def expect(value: Any, kind: type, what: str) -> Any:
|
||||
if not isinstance(value, kind):
|
||||
raise AdapterError("bad_structure", f"{what} has an unexpected shape")
|
||||
return value
|
||||
|
||||
|
||||
class SearchAdapter:
|
||||
name = ""
|
||||
direct = True
|
||||
|
||||
def request(self, query: str, limit: int) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def parse(self, body: bytes, limit: int) -> list[SocialHit]:
|
||||
raise NotImplementedError
|
||||
|
||||
def hit(self, **fields: Any) -> SocialHit | None:
|
||||
url = fields.get("url") or ""
|
||||
title = clean(fields.get("title"), TITLE_MAX)
|
||||
if not title or not isinstance(url, str) or not url.startswith(("http://", "https://")):
|
||||
return None
|
||||
return SocialHit(
|
||||
platform=self.name,
|
||||
title=title,
|
||||
url=url[:2048],
|
||||
snippet=clean(fields.get("snippet"), SNIPPET_MAX),
|
||||
author=clean(fields.get("author"), NAME_MAX),
|
||||
score=number(fields.get("score")),
|
||||
comments=number(fields.get("comments")),
|
||||
created_at=str(fields.get("created_at") or "")[:40],
|
||||
via=fields.get("via") or "direct",
|
||||
)
|
||||
@@ -0,0 +1,197 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy import stealth
|
||||
from devplacepy.services.nodes.adapters.base import (
|
||||
MAX_BODY_BYTES,
|
||||
AdapterError,
|
||||
SearchAdapter,
|
||||
SocialHit,
|
||||
clean,
|
||||
expect,
|
||||
load_json,
|
||||
number,
|
||||
tag_for,
|
||||
)
|
||||
|
||||
TIMEOUT_SECONDS = 12.0
|
||||
ACCEPT = {"Accept": "application/json"}
|
||||
|
||||
|
||||
async def get(url: str) -> bytes:
|
||||
timeout = httpx.Timeout(TIMEOUT_SECONDS, connect=6.0)
|
||||
try:
|
||||
async with stealth.stealth_async_client(
|
||||
headers=ACCEPT, follow_redirects=True, timeout=timeout
|
||||
) as client:
|
||||
response = await client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
raise AdapterError("network", type(exc).__name__)
|
||||
if response.status_code == 429:
|
||||
raise AdapterError("rate_limited")
|
||||
if response.status_code >= 400:
|
||||
raise AdapterError("upstream", f"HTTP {response.status_code}")
|
||||
body = response.content
|
||||
if len(body) > MAX_BODY_BYTES:
|
||||
raise AdapterError("body_too_large")
|
||||
return body
|
||||
|
||||
|
||||
class HackerNews(SearchAdapter):
|
||||
name = "hackernews"
|
||||
|
||||
def request(self, query: str, limit: int) -> str:
|
||||
params = urlencode({"query": query, "tags": "story", "hitsPerPage": limit})
|
||||
return f"https://hn.algolia.com/api/v1/search?{params}"
|
||||
|
||||
def parse(self, body: bytes, limit: int) -> list[SocialHit]:
|
||||
data = expect(load_json(body), dict, "response")
|
||||
hits = []
|
||||
for item in expect(data.get("hits"), list, "hits")[:limit]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
thread = f"https://news.ycombinator.com/item?id={quote(str(item.get('objectID') or ''))}"
|
||||
found = self.hit(
|
||||
title=item.get("title") or item.get("story_title"),
|
||||
url=thread,
|
||||
snippet=item.get("story_text") or item.get("url") or "",
|
||||
author=item.get("author"),
|
||||
score=item.get("points"),
|
||||
comments=item.get("num_comments"),
|
||||
created_at=item.get("created_at"),
|
||||
)
|
||||
if found and item.get("objectID"):
|
||||
hits.append(found)
|
||||
return hits
|
||||
|
||||
|
||||
class StackExchange(SearchAdapter):
|
||||
name = "stackexchange"
|
||||
|
||||
def request(self, query: str, limit: int) -> str:
|
||||
params = urlencode(
|
||||
{
|
||||
"q": query,
|
||||
"site": "stackoverflow",
|
||||
"pagesize": limit,
|
||||
"order": "desc",
|
||||
"sort": "relevance",
|
||||
}
|
||||
)
|
||||
return f"https://api.stackexchange.com/2.3/search/advanced?{params}"
|
||||
|
||||
def parse(self, body: bytes, limit: int) -> list[SocialHit]:
|
||||
data = expect(load_json(body), dict, "response")
|
||||
hits = []
|
||||
for item in expect(data.get("items"), list, "items")[:limit]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
owner = item.get("owner") if isinstance(item.get("owner"), dict) else {}
|
||||
tags = item.get("tags") if isinstance(item.get("tags"), list) else []
|
||||
found = self.hit(
|
||||
title=item.get("title"),
|
||||
url=item.get("link"),
|
||||
snippet=", ".join(str(tag) for tag in tags[:8]),
|
||||
author=owner.get("display_name"),
|
||||
score=item.get("score"),
|
||||
comments=item.get("answer_count"),
|
||||
created_at=item.get("creation_date"),
|
||||
)
|
||||
if found:
|
||||
hits.append(found)
|
||||
return hits
|
||||
|
||||
|
||||
class GitHub(SearchAdapter):
|
||||
name = "github"
|
||||
|
||||
def request(self, query: str, limit: int) -> str:
|
||||
params = urlencode({"q": query, "per_page": limit})
|
||||
return f"https://api.github.com/search/issues?{params}"
|
||||
|
||||
def parse(self, body: bytes, limit: int) -> list[SocialHit]:
|
||||
data = expect(load_json(body), dict, "response")
|
||||
hits = []
|
||||
for item in expect(data.get("items"), list, "items")[:limit]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
user = item.get("user") if isinstance(item.get("user"), dict) else {}
|
||||
reactions = item.get("reactions") if isinstance(item.get("reactions"), dict) else {}
|
||||
found = self.hit(
|
||||
title=item.get("title"),
|
||||
url=item.get("html_url"),
|
||||
snippet=item.get("body"),
|
||||
author=user.get("login"),
|
||||
score=reactions.get("total_count"),
|
||||
comments=item.get("comments"),
|
||||
created_at=item.get("created_at"),
|
||||
)
|
||||
if found:
|
||||
hits.append(found)
|
||||
return hits
|
||||
|
||||
|
||||
class DevTo(SearchAdapter):
|
||||
name = "devto"
|
||||
|
||||
def request(self, query: str, limit: int) -> str:
|
||||
tag = tag_for(query)
|
||||
if not tag:
|
||||
raise AdapterError("no_tag", "dev.to is searched by tag and the query has none")
|
||||
return f"https://dev.to/api/articles?{urlencode({'tag': tag, 'per_page': limit})}"
|
||||
|
||||
def parse(self, body: bytes, limit: int) -> list[SocialHit]:
|
||||
data = expect(load_json(body), list, "response")
|
||||
hits = []
|
||||
for item in data[:limit]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
user = item.get("user") if isinstance(item.get("user"), dict) else {}
|
||||
found = self.hit(
|
||||
title=item.get("title"),
|
||||
url=item.get("url"),
|
||||
snippet=item.get("description"),
|
||||
author=user.get("username"),
|
||||
score=item.get("public_reactions_count"),
|
||||
comments=item.get("comments_count"),
|
||||
created_at=item.get("published_at"),
|
||||
)
|
||||
if found:
|
||||
hits.append(found)
|
||||
return hits
|
||||
|
||||
|
||||
class Mastodon(SearchAdapter):
|
||||
name = "mastodon"
|
||||
|
||||
def request(self, query: str, limit: int) -> str:
|
||||
tag = tag_for(query)
|
||||
if not tag:
|
||||
raise AdapterError("no_tag", "Mastodon is searched by hashtag and the query has none")
|
||||
return f"https://mastodon.social/api/v1/timelines/tag/{quote(tag)}?{urlencode({'limit': min(limit, 40)})}"
|
||||
|
||||
def parse(self, body: bytes, limit: int) -> list[SocialHit]:
|
||||
data = expect(load_json(body), list, "response")
|
||||
hits = []
|
||||
for item in data[:limit]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
account = item.get("account") if isinstance(item.get("account"), dict) else {}
|
||||
content = clean(item.get("content"), 500)
|
||||
found = self.hit(
|
||||
title=content[:120],
|
||||
url=item.get("url"),
|
||||
snippet=content,
|
||||
author=account.get("acct"),
|
||||
score=number(item.get("favourites_count")) + number(item.get("reblogs_count")),
|
||||
comments=item.get("replies_count"),
|
||||
created_at=item.get("created_at"),
|
||||
)
|
||||
if found:
|
||||
hits.append(found)
|
||||
return hits
|
||||
@@ -0,0 +1,43 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from devplacepy.services.nodes.adapters.base import SearchAdapter, SocialHit, expect, load_json
|
||||
|
||||
BASE = "https://www.reddit.com"
|
||||
|
||||
|
||||
class Reddit(SearchAdapter):
|
||||
name = "reddit"
|
||||
direct = False
|
||||
|
||||
def request(self, query: str, limit: int) -> str:
|
||||
params = urlencode({"q": query, "limit": limit, "raw_json": 1, "sort": "relevance"})
|
||||
return f"{BASE}/search.json?{params}"
|
||||
|
||||
def parse(self, body: bytes, limit: int) -> list[SocialHit]:
|
||||
data = expect(load_json(body), dict, "response")
|
||||
listing = expect(data.get("data"), dict, "listing")
|
||||
hits = []
|
||||
for child in expect(listing.get("children"), list, "children")[:limit]:
|
||||
item = child.get("data") if isinstance(child, dict) else None
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
permalink = item.get("permalink")
|
||||
if not isinstance(permalink, str) or not permalink.startswith("/r/"):
|
||||
continue
|
||||
found = self.hit(
|
||||
title=item.get("title"),
|
||||
url=BASE + permalink,
|
||||
snippet=item.get("selftext"),
|
||||
author=item.get("author"),
|
||||
score=item.get("score"),
|
||||
comments=item.get("num_comments"),
|
||||
created_at=item.get("created_utc"),
|
||||
via="fetch",
|
||||
)
|
||||
if found:
|
||||
hits.append(found)
|
||||
return hits
|
||||
@@ -0,0 +1,12 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from devplacepy.services.nodes.adapters.base import SearchAdapter
|
||||
from devplacepy.services.nodes.adapters.direct import DevTo, GitHub, HackerNews, Mastodon, StackExchange
|
||||
from devplacepy.services.nodes.adapters.reddit import Reddit
|
||||
|
||||
ADAPTERS: dict[str, SearchAdapter] = {
|
||||
item.name: item for item in (HackerNews(), StackExchange(), GitHub(), DevTo(), Mastodon(), Reddit())
|
||||
}
|
||||
DEFAULT_PLATFORMS = ("hackernews", "stackexchange", "github", "devto", "mastodon", "reddit")
|
||||
@@ -0,0 +1,127 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from devplacepy.database import nodes as store
|
||||
from devplacepy.services.nodes.policy import Target
|
||||
from devplacepy.services.nodes.spec import SPEC
|
||||
|
||||
LIMITS = SPEC.limits
|
||||
|
||||
DONE = "done"
|
||||
FAILED = "failed"
|
||||
EXPIRED = "expired"
|
||||
TERMINAL = (DONE, FAILED, EXPIRED)
|
||||
|
||||
MAX_HEADERS = 64
|
||||
MAX_HEADER_CHARS = 1024
|
||||
DROPPED_HEADERS = frozenset({"set-cookie", "set-cookie2", "authorization", "proxy-authenticate"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BrokerResult:
|
||||
uid: str
|
||||
status: str
|
||||
http_status: int
|
||||
headers: dict
|
||||
body: bytes
|
||||
truncated: bool
|
||||
error: str
|
||||
via: str = "fetch"
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.status == DONE
|
||||
|
||||
|
||||
class BodyError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def clean_headers(raw) -> dict[str, str]:
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
kept: dict[str, str] = {}
|
||||
for key, value in raw.items():
|
||||
if len(kept) >= MAX_HEADERS:
|
||||
break
|
||||
if not isinstance(key, str) or not isinstance(value, str):
|
||||
continue
|
||||
name = key.strip().lower()
|
||||
if not name or name in DROPPED_HEADERS:
|
||||
continue
|
||||
kept[name[:MAX_HEADER_CHARS]] = value[:MAX_HEADER_CHARS]
|
||||
return kept
|
||||
|
||||
|
||||
def decode_body(body_b64: str) -> tuple[bytes, bool]:
|
||||
try:
|
||||
body = base64.b64decode(body_b64.encode(), validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
raise BodyError("bad_base64")
|
||||
if len(body) > LIMITS.max_body_bytes:
|
||||
return body[: LIMITS.max_body_bytes], True
|
||||
return body, False
|
||||
|
||||
|
||||
class JobBroker:
|
||||
def enqueue(self, caller_kind: str, caller_id: str, target: Target, mode: str) -> str:
|
||||
return store.enqueue_request(
|
||||
caller_kind, caller_id, target.url, target.method, target.adapter, mode
|
||||
)
|
||||
|
||||
def queued(self, limit: int = LIMITS.dispatch_batch) -> list[dict]:
|
||||
return store.list_queued(limit)
|
||||
|
||||
def claim(self, uid: str, device_uid: str) -> bool:
|
||||
return store.claim_request(uid, device_uid)
|
||||
|
||||
def complete(self, uid: str, http_status: int, headers: dict, body: bytes, truncated: bool) -> bool:
|
||||
return store.finish_request(
|
||||
uid,
|
||||
result_status=http_status,
|
||||
result_headers=json.dumps(headers),
|
||||
result_body=base64.b64encode(body).decode(),
|
||||
result_bytes=len(body),
|
||||
truncated=truncated,
|
||||
)
|
||||
|
||||
def fail(self, uid: str, error: str, *, expired: bool = False) -> bool:
|
||||
return store.fail_request(uid, error, expired=expired)
|
||||
|
||||
def expire(self) -> dict[str, int]:
|
||||
return store.expire_requests()
|
||||
|
||||
def read(self, uid: str) -> BrokerResult | None:
|
||||
row = store.get_request(uid)
|
||||
if row is None:
|
||||
return None
|
||||
return BrokerResult(
|
||||
uid=row["uid"],
|
||||
status=row["status"],
|
||||
http_status=int(row.get("result_status") or 0),
|
||||
headers=json.loads(row.get("result_headers") or "{}") if row.get("result_headers") else {},
|
||||
body=base64.b64decode(row["result_body"]) if row.get("result_body") else b"",
|
||||
truncated=bool(row.get("truncated")),
|
||||
error=row.get("error") or "",
|
||||
via=row.get("mode") if row.get("mode") in ("tunnel", "fetch") else "fetch",
|
||||
)
|
||||
|
||||
async def wait(self, uid: str, timeout: float) -> BrokerResult:
|
||||
deadline = time.monotonic() + timeout
|
||||
pause = LIMITS.poll_interval_ms / 1000
|
||||
while True:
|
||||
row = store.get_request(uid)
|
||||
if row is not None and row["status"] in TERMINAL:
|
||||
return self.read(uid)
|
||||
if time.monotonic() >= deadline:
|
||||
self.fail(uid, "caller_timeout")
|
||||
return self.read(uid)
|
||||
await asyncio.sleep(pause)
|
||||
@@ -0,0 +1,90 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
from devplacepy.services.nodes.spec import SPEC
|
||||
|
||||
LIMITS = SPEC.limits
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CachedResult:
|
||||
status: int
|
||||
headers: dict
|
||||
body: bytes
|
||||
truncated: bool
|
||||
stored_at: float
|
||||
|
||||
|
||||
def cache_key(method: str, url: str) -> str:
|
||||
return hashlib.sha256(f"{method.upper()} {url}".encode()).hexdigest()
|
||||
|
||||
|
||||
class ResultCache:
|
||||
def __init__(
|
||||
self,
|
||||
ttl_seconds: int = LIMITS.cache_ttl_seconds,
|
||||
max_bytes: int = LIMITS.cache_max_bytes,
|
||||
entry_max_bytes: int = LIMITS.cache_entry_max_bytes,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
self.ttl_seconds = max(0, min(ttl_seconds, LIMITS.cache_ttl_seconds))
|
||||
self.max_bytes = max_bytes
|
||||
self.entry_max_bytes = min(entry_max_bytes, LIMITS.cache_entry_max_bytes)
|
||||
self._clock = clock
|
||||
self._entries: OrderedDict[str, CachedResult] = OrderedDict()
|
||||
self._bytes = 0
|
||||
self.hits = 0
|
||||
self.misses = 0
|
||||
|
||||
@property
|
||||
def size_bytes(self) -> int:
|
||||
return self._bytes
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._entries)
|
||||
|
||||
def _drop(self, key: str) -> None:
|
||||
entry = self._entries.pop(key, None)
|
||||
if entry is not None:
|
||||
self._bytes -= len(entry.body)
|
||||
|
||||
def get(self, method: str, url: str) -> CachedResult | None:
|
||||
key = cache_key(method, url)
|
||||
entry = self._entries.get(key)
|
||||
if entry is None:
|
||||
self.misses += 1
|
||||
return None
|
||||
if self._clock() - entry.stored_at >= self.ttl_seconds:
|
||||
self._drop(key)
|
||||
self.misses += 1
|
||||
return None
|
||||
self._entries.move_to_end(key)
|
||||
self.hits += 1
|
||||
return entry
|
||||
|
||||
def put(
|
||||
self, method: str, url: str, status: int, headers: dict, body: bytes, truncated: bool
|
||||
) -> bool:
|
||||
if self.ttl_seconds <= 0 or status >= 500 or truncated:
|
||||
return False
|
||||
if len(body) > self.entry_max_bytes or len(body) > self.max_bytes:
|
||||
return False
|
||||
key = cache_key(method, url)
|
||||
self._drop(key)
|
||||
self._entries[key] = CachedResult(status, dict(headers), body, truncated, self._clock())
|
||||
self._bytes += len(body)
|
||||
while self._bytes > self.max_bytes and self._entries:
|
||||
oldest = next(iter(self._entries))
|
||||
self._drop(oldest)
|
||||
return True
|
||||
|
||||
def clear(self) -> None:
|
||||
self._entries.clear()
|
||||
self._bytes = 0
|
||||
@@ -0,0 +1,71 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Hashable
|
||||
|
||||
from devplacepy.services.nodes.spec import SPEC
|
||||
|
||||
LIMITS = SPEC.limits
|
||||
|
||||
CLOSED = "closed"
|
||||
OPEN = "open"
|
||||
HALF_OPEN = "half_open"
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Circuit:
|
||||
failures: int = 0
|
||||
opened_at: float = 0.0
|
||||
probing: bool = False
|
||||
|
||||
|
||||
class CircuitBreaker:
|
||||
def __init__(
|
||||
self,
|
||||
failures: int = LIMITS.breaker_failures,
|
||||
open_seconds: float = LIMITS.breaker_open_seconds,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
self.failures = max(1, failures)
|
||||
self.open_seconds = open_seconds
|
||||
self._clock = clock
|
||||
self._circuits: dict[Hashable, _Circuit] = {}
|
||||
|
||||
def state(self, key: Hashable) -> str:
|
||||
circuit = self._circuits.get(key)
|
||||
if circuit is None or circuit.failures < self.failures:
|
||||
return CLOSED
|
||||
if self._clock() - circuit.opened_at < self.open_seconds:
|
||||
return OPEN
|
||||
return HALF_OPEN
|
||||
|
||||
def allows(self, key: Hashable) -> bool:
|
||||
state = self.state(key)
|
||||
if state == CLOSED:
|
||||
return True
|
||||
if state == OPEN:
|
||||
return False
|
||||
return not self._circuits[key].probing
|
||||
|
||||
def begin(self, key: Hashable) -> None:
|
||||
if self.state(key) == HALF_OPEN:
|
||||
self._circuits[key].probing = True
|
||||
|
||||
def success(self, key: Hashable) -> None:
|
||||
self._circuits.pop(key, None)
|
||||
|
||||
def failure(self, key: Hashable) -> None:
|
||||
circuit = self._circuits.setdefault(key, _Circuit())
|
||||
circuit.probing = False
|
||||
circuit.failures += 1
|
||||
if circuit.failures >= self.failures:
|
||||
circuit.opened_at = self._clock()
|
||||
|
||||
def forget(self, key: Hashable) -> None:
|
||||
self._circuits.pop(key, None)
|
||||
|
||||
def open_count(self) -> int:
|
||||
return sum(1 for key in list(self._circuits) if self.state(key) != CLOSED)
|
||||
@@ -0,0 +1,273 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
from devplacepy.database import nodes as store
|
||||
from devplacepy.services.nodes.broker import BodyError, JobBroker, clean_headers, decode_body
|
||||
from devplacepy.services.nodes.circuit import CircuitBreaker
|
||||
from devplacepy.services.nodes.ledger import UsageLedger
|
||||
from devplacepy.services.nodes.policy import DestinationPolicy, PolicyError, Target
|
||||
from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry
|
||||
from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler
|
||||
from devplacepy.services.nodes.session import Inflight, NodeSession
|
||||
from devplacepy.services.nodes.spec import MODE_FETCH, MODE_TUNNEL, SPEC
|
||||
from devplacepy.services.nodes.tunnel import TunnelBridge, TunnelError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LIMITS = SPEC.limits
|
||||
|
||||
JOB_HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (compatible; DevPlaceBot/1.0; +https://devplace.net)",
|
||||
"Accept": "application/json, text/html;q=0.9, */*;q=0.5",
|
||||
}
|
||||
|
||||
REQUEST_OVERHEAD_BYTES = 256
|
||||
DONOR_LIMIT_PAUSE_SECONDS = 300
|
||||
CONCURRENCY_PAUSE_SECONDS = 5
|
||||
THROTTLED_STATUSES = (429,)
|
||||
DONOR_LIMIT_CODES = {
|
||||
"bandwidth_exhausted": None,
|
||||
"outside_hours": DONOR_LIMIT_PAUSE_SECONDS,
|
||||
"concurrency_exceeded": CONCURRENCY_PAUSE_SECONDS,
|
||||
}
|
||||
REFUSAL_CODES = frozenset(item.code for item in SPEC.refusals) - set(DONOR_LIMIT_CODES) - {
|
||||
"body_too_large",
|
||||
"timeout",
|
||||
}
|
||||
|
||||
|
||||
class NodeDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
registry: NodeRegistry,
|
||||
scheduler: NodeScheduler,
|
||||
broker: JobBroker,
|
||||
breaker: CircuitBreaker,
|
||||
ledger: UsageLedger,
|
||||
counters: NodeCounters,
|
||||
policy: DestinationPolicy | None = None,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
tunnels: TunnelBridge | None = None,
|
||||
) -> None:
|
||||
self.tunnels = tunnels or TunnelBridge()
|
||||
self._tunnel_tasks: dict[str, asyncio.Task] = {}
|
||||
self.registry = registry
|
||||
self.scheduler = scheduler
|
||||
self.broker = broker
|
||||
self.breaker = breaker
|
||||
self.ledger = ledger
|
||||
self.counters = counters
|
||||
self.policy = policy or DestinationPolicy()
|
||||
self._clock = clock
|
||||
|
||||
def attach(self, session: NodeSession) -> None:
|
||||
session.job_sink = self.on_job_frame
|
||||
session.tunnel_sink = self.tunnels.on_frame
|
||||
|
||||
async def tick(self) -> int:
|
||||
if store.count_open_requests() == 0:
|
||||
return 0
|
||||
self.broker.expire()
|
||||
await self.sweep_inflight()
|
||||
if self.registry.count() == 0:
|
||||
return 0
|
||||
assigned = 0
|
||||
saturated: set[tuple[str, str]] = set()
|
||||
for row in self.broker.queued():
|
||||
assigned += await self._assign(row, saturated)
|
||||
return assigned
|
||||
|
||||
async def _assign(self, row: dict, saturated: set) -> int:
|
||||
uid = row["uid"]
|
||||
try:
|
||||
target = self.policy.check(row["url"], row["method"], row["mode"])
|
||||
except PolicyError as exc:
|
||||
self.broker.fail(uid, exc.code)
|
||||
self.counters.inc("jobs_refused")
|
||||
return 0
|
||||
key = (target.destination.suffix, row["mode"])
|
||||
if key in saturated:
|
||||
return 0
|
||||
owner = row["caller_id"] if row["caller_kind"] == "user" else ""
|
||||
found = self.scheduler.candidates(target, row["mode"])
|
||||
if not found:
|
||||
saturated.add(key)
|
||||
return 0
|
||||
session = self.scheduler.choose(found, owner)
|
||||
if session is None or not self.broker.claim(uid, session.device_uid):
|
||||
return 0
|
||||
job = Inflight(
|
||||
job_id=uid,
|
||||
host=target.host,
|
||||
destination=target.destination.suffix,
|
||||
method=target.method,
|
||||
url=target.url,
|
||||
started=self._clock(),
|
||||
bytes_up=len(target.url.encode()) + REQUEST_OVERHEAD_BYTES,
|
||||
)
|
||||
session.inflight[uid] = job
|
||||
self.scheduler.mark_started(session, target)
|
||||
transport = self.scheduler.transport_of(session, target, row["mode"]) or MODE_FETCH
|
||||
store.set_request_transport(uid, transport)
|
||||
if transport == MODE_TUNNEL:
|
||||
self._tunnel_tasks[uid] = asyncio.create_task(self._run_tunnel(session, job, target))
|
||||
self.counters.inc("jobs_assigned")
|
||||
return 1
|
||||
try:
|
||||
await session.send(
|
||||
"job",
|
||||
job_id=uid,
|
||||
method=target.method,
|
||||
url=target.url,
|
||||
headers=JOB_HEADERS,
|
||||
timeout=LIMITS.job_timeout_seconds,
|
||||
)
|
||||
except Exception:
|
||||
session.inflight.pop(uid, None)
|
||||
self._failed(session, job, "send_failed")
|
||||
return 0
|
||||
self.counters.inc("jobs_assigned")
|
||||
return 1
|
||||
|
||||
def _failed(self, session: NodeSession, job: Inflight, code: str, node_fault: bool = True) -> None:
|
||||
self.broker.fail(job.job_id, code)
|
||||
self.counters.inc("jobs_failed")
|
||||
if node_fault:
|
||||
self.breaker.failure(("node", session.device_uid))
|
||||
store.add_event("job_failed", device_uid=session.device_uid, user_uid=session.user_uid, detail=code)
|
||||
self.ledger.record_failure(session.device_uid, session.user_uid)
|
||||
|
||||
async def on_job_frame(self, session: NodeSession, job: Inflight, frame: dict) -> None:
|
||||
session.inflight.pop(job.job_id, None)
|
||||
if frame["type"] == "job_error":
|
||||
self._on_error(session, job, frame["code"])
|
||||
return
|
||||
self._on_result(session, job, frame)
|
||||
|
||||
def _integrity_failure(self, session: NodeSession, job: Inflight, code: str) -> None:
|
||||
self.counters.inc("integrity_failures")
|
||||
self._failed(session, job, code)
|
||||
self.ledger.record_integrity_failure(session.device_uid, session.user_uid)
|
||||
store.raise_flag(
|
||||
"integrity", device_uid=session.device_uid, user_uid=session.user_uid, detail=code
|
||||
)
|
||||
|
||||
def _on_error(self, session: NodeSession, job: Inflight, code: str) -> None:
|
||||
if code in DONOR_LIMIT_CODES:
|
||||
pause = DONOR_LIMIT_CODES[code]
|
||||
if pause is None:
|
||||
session.used_today = int(session.settings.get("bandwidth_mb_day") or 0) * MEGABYTE
|
||||
else:
|
||||
session.paused_until = self._clock() + pause
|
||||
self._failed(session, job, code, node_fault=False)
|
||||
return
|
||||
if code in REFUSAL_CODES:
|
||||
self.counters.inc("jobs_refused")
|
||||
store.raise_flag(
|
||||
"refusal_mismatch",
|
||||
device_uid=session.device_uid,
|
||||
user_uid=session.user_uid,
|
||||
detail=f"{code} {job.destination}",
|
||||
)
|
||||
self._failed(session, job, code, node_fault=False)
|
||||
return
|
||||
self._failed(session, job, code)
|
||||
|
||||
def _on_result(self, session: NodeSession, job: Inflight, frame: dict) -> None:
|
||||
status = frame["status"]
|
||||
if not 100 <= status <= 599:
|
||||
self._integrity_failure(session, job, "bad_status")
|
||||
return
|
||||
try:
|
||||
body, over = decode_body(frame["body_b64"])
|
||||
except BodyError as exc:
|
||||
self._integrity_failure(session, job, str(exc))
|
||||
return
|
||||
truncated = bool(frame.get("truncated")) or over
|
||||
if truncated:
|
||||
self.counters.inc("results_truncated")
|
||||
headers = clean_headers(frame.get("headers"))
|
||||
if not self.broker.complete(job.job_id, status, headers, body, truncated):
|
||||
return
|
||||
self._account(session, job, status, job.bytes_up, len(body))
|
||||
|
||||
async def _run_tunnel(self, session: NodeSession, job: Inflight, target: Target) -> None:
|
||||
try:
|
||||
result = await self.tunnels.fetch(session, target, LIMITS.job_timeout_seconds)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except TunnelError as exc:
|
||||
self._tunnel_ended(session, job, exc.code)
|
||||
return
|
||||
except Exception:
|
||||
logger.exception("tunnel job %s failed", job.job_id)
|
||||
self._tunnel_ended(session, job, "tunnel_failed")
|
||||
return
|
||||
finally:
|
||||
self._tunnel_tasks.pop(job.job_id, None)
|
||||
if session.inflight.pop(job.job_id, None) is None:
|
||||
return
|
||||
if result.truncated:
|
||||
self.counters.inc("results_truncated")
|
||||
if not self.broker.complete(
|
||||
job.job_id, result.status, clean_headers(result.headers), result.body, result.truncated
|
||||
):
|
||||
return
|
||||
self._account(session, job, result.status, result.bytes_up, result.bytes_down)
|
||||
|
||||
def _tunnel_ended(self, session: NodeSession, job: Inflight, code: str) -> None:
|
||||
if session.inflight.pop(job.job_id, None) is not None:
|
||||
self._failed(session, job, code)
|
||||
|
||||
def _account(self, session: NodeSession, job: Inflight, status: int, up: int, down: int) -> None:
|
||||
self.counters.inc("jobs_done")
|
||||
spent = up + down
|
||||
accepted = self.ledger.record_success(
|
||||
session.device_uid, session.user_uid, bytes_up=up, bytes_down=down
|
||||
)
|
||||
session.used_today += spent
|
||||
if accepted:
|
||||
self.ledger.reward(session.user_uid)
|
||||
if not accepted:
|
||||
session.used_today = int(session.settings.get("bandwidth_mb_day") or 0) * MEGABYTE
|
||||
self.breaker.success(("node", session.device_uid))
|
||||
destination = ("destination", job.destination)
|
||||
if status >= 500 or status in THROTTLED_STATUSES:
|
||||
self.breaker.failure(destination)
|
||||
else:
|
||||
self.breaker.success(destination)
|
||||
|
||||
async def sweep_inflight(self) -> int:
|
||||
limit = LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds
|
||||
now = self._clock()
|
||||
swept = 0
|
||||
for session in self.registry.sessions():
|
||||
for job in list(session.inflight.values()):
|
||||
if now - job.started > limit:
|
||||
task = self._tunnel_tasks.pop(job.job_id, None)
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
session.inflight.pop(job.job_id, None)
|
||||
self._failed(session, job, "timeout")
|
||||
swept += 1
|
||||
return swept
|
||||
|
||||
def release_session(self, session: NodeSession) -> int:
|
||||
jobs = list(session.inflight.values())
|
||||
session.inflight.clear()
|
||||
self.tunnels.release_session(session)
|
||||
for job in jobs:
|
||||
task = self._tunnel_tasks.pop(job.job_id, None)
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
self._failed(session, job, "node_disconnected")
|
||||
return len(jobs)
|
||||
|
||||
def target_for(self, url: str, method: str = "GET", mode: str = "any") -> Target:
|
||||
return self.policy.check(url, method, mode)
|
||||
@@ -0,0 +1,204 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from devplacepy.database import get_int_setting, get_table
|
||||
from devplacepy.database import nodes as store
|
||||
from devplacepy.services.base import BaseService
|
||||
from devplacepy.services.nodes.broker import FAILED, JobBroker
|
||||
from devplacepy.services.nodes.cache import ResultCache
|
||||
from devplacepy.services.nodes.consent import ConsentGate
|
||||
from devplacepy.services.nodes.policy import DestinationPolicy, PolicyError, Target
|
||||
from devplacepy.services.nodes.spec import SPEC
|
||||
|
||||
LIMITS = SPEC.limits
|
||||
|
||||
VIA_FETCH = "fetch"
|
||||
VIA_CACHE = "cache"
|
||||
RETRYABLE = frozenset(
|
||||
{
|
||||
"node_disconnected",
|
||||
"timeout",
|
||||
"send_failed",
|
||||
"concurrency_exceeded",
|
||||
"outside_hours",
|
||||
"bandwidth_exhausted",
|
||||
}
|
||||
)
|
||||
MAX_ATTEMPTS = 2
|
||||
GUEST_KIND = "guest"
|
||||
USER_KIND = "user"
|
||||
|
||||
|
||||
class FabricError(Exception):
|
||||
def __init__(self, code: str, message: str = ""):
|
||||
super().__init__(message or code)
|
||||
self.code = code
|
||||
self.message = message or code
|
||||
|
||||
|
||||
class FabricRefused(FabricError):
|
||||
pass
|
||||
|
||||
|
||||
class FabricUnavailable(FabricError):
|
||||
pass
|
||||
|
||||
|
||||
class FabricQuotaExceeded(FabricError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FabricResult:
|
||||
url: str
|
||||
status: int
|
||||
headers: dict
|
||||
body: bytes
|
||||
via: str
|
||||
truncated: bool
|
||||
elapsed_ms: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FabricStatus:
|
||||
enabled: bool
|
||||
nodes_online: int
|
||||
|
||||
|
||||
class NodeFabric:
|
||||
def __init__(
|
||||
self,
|
||||
policy: DestinationPolicy | None = None,
|
||||
broker: JobBroker | None = None,
|
||||
cache: ResultCache | None = None,
|
||||
gate: ConsentGate | None = None,
|
||||
) -> None:
|
||||
self.policy = policy or DestinationPolicy()
|
||||
self.broker = broker or JobBroker()
|
||||
self.cache = cache or ResultCache()
|
||||
self.gate = gate or ConsentGate()
|
||||
|
||||
def available(self) -> FabricStatus:
|
||||
enabled = self.gate.switched_on()
|
||||
return FabricStatus(enabled, self._online() if enabled else 0)
|
||||
|
||||
@staticmethod
|
||||
def _online() -> int:
|
||||
row = get_table("service_state").find_one(name="nodes")
|
||||
if not row or not row.get("heartbeat"):
|
||||
return 0
|
||||
try:
|
||||
beat = datetime.fromisoformat(row["heartbeat"])
|
||||
except ValueError:
|
||||
return 0
|
||||
if beat.tzinfo is None:
|
||||
beat = beat.replace(tzinfo=timezone.utc)
|
||||
if (datetime.now(timezone.utc) - beat).total_seconds() > BaseService.STALE_SECONDS * 2:
|
||||
return 0
|
||||
try:
|
||||
stats = json.loads(row.get("metrics") or "{}").get("stats") or []
|
||||
except ValueError:
|
||||
return 0
|
||||
for item in stats:
|
||||
if item.get("label") == "connected":
|
||||
return int(item.get("value") or 0)
|
||||
return 0
|
||||
|
||||
def can_serve(self, url: str, method: str = "GET") -> bool:
|
||||
try:
|
||||
self.policy.check(url, method)
|
||||
except PolicyError:
|
||||
return False
|
||||
status = self.available()
|
||||
return status.enabled and status.nodes_online > 0
|
||||
|
||||
def _quota(self, caller_kind: str) -> int:
|
||||
if caller_kind == USER_KIND:
|
||||
return get_int_setting("nodes_daily_quota_user", LIMITS.default_daily_requests_user)
|
||||
return get_int_setting("nodes_daily_quota_guest", LIMITS.default_daily_requests_guest)
|
||||
|
||||
def _sync_cache_ttl(self) -> None:
|
||||
configured = get_int_setting("nodes_cache_ttl_seconds", LIMITS.cache_ttl_seconds)
|
||||
self.cache.ttl_seconds = max(0, min(configured, LIMITS.cache_ttl_seconds))
|
||||
|
||||
async def fetch(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
caller_kind: str,
|
||||
caller_id: str,
|
||||
method: str = "GET",
|
||||
mode: str = "any",
|
||||
use_cache: bool = True,
|
||||
timeout: float | None = None,
|
||||
) -> FabricResult:
|
||||
try:
|
||||
target = self.policy.check(url, method, mode)
|
||||
except PolicyError as exc:
|
||||
raise FabricRefused(exc.code, exc.reason)
|
||||
started = time.monotonic()
|
||||
self._sync_cache_ttl()
|
||||
if use_cache:
|
||||
cached = self.cache.get(target.method, target.url)
|
||||
if cached is not None:
|
||||
return FabricResult(
|
||||
target.url, cached.status, cached.headers, cached.body, VIA_CACHE, cached.truncated, 0
|
||||
)
|
||||
status = self.available()
|
||||
if not status.enabled:
|
||||
raise FabricUnavailable("kill_switch", "Community donation is switched off")
|
||||
if status.nodes_online <= 0:
|
||||
raise FabricUnavailable("no_nodes", "No donor node is connected")
|
||||
if not store.reserve_caller_request(caller_kind, caller_id, self._quota(caller_kind)):
|
||||
raise FabricQuotaExceeded("quota", "Daily node request quota reached")
|
||||
wait = min(float(timeout or LIMITS.default_wait_seconds), float(LIMITS.max_wait_seconds))
|
||||
result = await self._run(target, caller_kind, caller_id, mode, wait)
|
||||
if use_cache:
|
||||
self.cache.put(
|
||||
target.method, target.url, result.http_status, result.headers, result.body, result.truncated
|
||||
)
|
||||
return FabricResult(
|
||||
target.url,
|
||||
result.http_status,
|
||||
result.headers,
|
||||
result.body,
|
||||
result.via,
|
||||
result.truncated,
|
||||
int((time.monotonic() - started) * 1000),
|
||||
)
|
||||
|
||||
async def _run(self, target: Target, caller_kind: str, caller_id: str, mode: str, wait: float):
|
||||
deadline = time.monotonic() + wait
|
||||
error = ""
|
||||
for _ in range(MAX_ATTEMPTS):
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
try:
|
||||
uid = self.broker.enqueue(caller_kind, caller_id, target, mode)
|
||||
except store.QueueFull:
|
||||
raise FabricUnavailable("queue_full", "The node queue is full")
|
||||
result = await self.broker.wait(uid, remaining)
|
||||
if result.ok:
|
||||
return result
|
||||
error = result.error or result.status
|
||||
if result.status == FAILED and result.error in RETRYABLE:
|
||||
continue
|
||||
break
|
||||
raise FabricUnavailable(str(error) or "failed", f"Node fetch failed: {error}")
|
||||
|
||||
|
||||
_shared: NodeFabric | None = None
|
||||
|
||||
|
||||
def shared_fabric() -> NodeFabric:
|
||||
global _shared
|
||||
if _shared is None:
|
||||
_shared = NodeFabric()
|
||||
return _shared
|
||||
@@ -7,6 +7,17 @@ from datetime import datetime, timezone
|
||||
|
||||
from devplacepy.database import nodes as store
|
||||
|
||||
XP_PER_JOB = 1
|
||||
GIGABYTE = 1024 * 1024 * 1024
|
||||
BYTE_BADGES = (
|
||||
(GIGABYTE, "Gigabyte Donor"),
|
||||
(10 * GIGABYTE, "10 GB Donor"),
|
||||
(100 * GIGABYTE, "100 GB Donor"),
|
||||
)
|
||||
FIRST_BADGE = "Donor"
|
||||
STEADY_DAYS = 30
|
||||
STEADY_BADGE = "Steady Donor"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DonationSummary:
|
||||
@@ -49,6 +60,20 @@ class UsageLedger:
|
||||
def record_integrity_failure(self, device_uid: str, user_uid: str) -> None:
|
||||
store.record_integrity_failure(device_uid, user_uid)
|
||||
|
||||
def reward(self, user_uid: str) -> None:
|
||||
from devplacepy.utils import award_badge, award_rewards, notify_badge
|
||||
|
||||
if self.claim_xp(user_uid, XP_PER_JOB):
|
||||
award_rewards(user_uid, XP_PER_JOB)
|
||||
donor = store.get_donor(user_uid)
|
||||
earned = [FIRST_BADGE] if donor["requests_ok"] >= 1 else []
|
||||
earned += [name for size, name in BYTE_BADGES if donor["bytes_total"] >= size]
|
||||
if donor["requests_ok"] >= STEADY_DAYS and store.donation_days(user_uid) >= STEADY_DAYS:
|
||||
earned.append(STEADY_BADGE)
|
||||
for name in earned:
|
||||
if award_badge(user_uid, name):
|
||||
notify_badge(user_uid, name)
|
||||
|
||||
def claim_xp(self, user_uid: str, amount: int) -> bool:
|
||||
return store.claim_xp(user_uid, amount)
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from devplacepy.services.nodes.spec import ALLOWED_PORTS, MODES, SPEC, DestinationSpec
|
||||
|
||||
LIMITS = SPEC.limits
|
||||
DEFAULT_PORTS = {"http": 80, "https": 443}
|
||||
SCHEMES = tuple(DEFAULT_PORTS)
|
||||
|
||||
SHARED_ADDRESS_SPACE = ipaddress.ip_network("100.64.0.0/10")
|
||||
|
||||
|
||||
class PolicyError(ValueError):
|
||||
def __init__(self, code: str, reason: str = ""):
|
||||
super().__init__(reason or code)
|
||||
self.code = code
|
||||
self.reason = reason or code
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Target:
|
||||
url: str
|
||||
method: str
|
||||
host: str
|
||||
port: int
|
||||
destination: DestinationSpec
|
||||
|
||||
@property
|
||||
def adapter(self) -> str:
|
||||
return self.destination.adapter
|
||||
|
||||
|
||||
def is_public_address(address: str) -> bool:
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
except ValueError:
|
||||
return False
|
||||
if ip.version == 4 and ip in SHARED_ADDRESS_SPACE:
|
||||
return False
|
||||
return ip.is_global and not ip.is_multicast
|
||||
|
||||
|
||||
class DestinationPolicy:
|
||||
def check(self, url: str, method: str = "GET", mode: str = "any") -> Target:
|
||||
if mode not in MODES:
|
||||
raise PolicyError("bad_mode", f"unknown mode {mode!r}")
|
||||
if not isinstance(url, str) or not url or len(url) > LIMITS.url_max_chars:
|
||||
raise PolicyError("bad_url", "url is missing or too long")
|
||||
try:
|
||||
parts = urlsplit(url.strip())
|
||||
port = parts.port
|
||||
except ValueError:
|
||||
raise PolicyError("bad_url", "url cannot be parsed")
|
||||
if parts.scheme not in SCHEMES:
|
||||
raise PolicyError("bad_url", "only http and https are allowed")
|
||||
if parts.username is not None or parts.password is not None:
|
||||
raise PolicyError("bad_url", "credentials in a url are not allowed")
|
||||
host = (parts.hostname or "").lower().rstrip(".")
|
||||
if not host:
|
||||
raise PolicyError("bad_url", "url has no host")
|
||||
if _is_address_literal(host):
|
||||
raise PolicyError("host_not_allowed", "address literals are not allowed")
|
||||
destination = SPEC.destination(host)
|
||||
if destination is None:
|
||||
raise PolicyError("host_not_allowed", "destination is not on the allowlist")
|
||||
port = port or DEFAULT_PORTS[parts.scheme]
|
||||
if port not in ALLOWED_PORTS or port not in destination.ports:
|
||||
raise PolicyError("port_not_allowed", "port is not allowed")
|
||||
verb = (method or "").upper()
|
||||
if verb not in destination.methods:
|
||||
raise PolicyError("method_not_allowed", "method is not allowed here")
|
||||
return Target(parts.geturl(), verb, host, port, destination)
|
||||
|
||||
def check_redirect(self, location: str, method: str = "GET") -> Target:
|
||||
try:
|
||||
return self.check(location, method)
|
||||
except PolicyError as exc:
|
||||
raise PolicyError("redirect_not_allowed", exc.reason)
|
||||
|
||||
|
||||
def _is_address_literal(host: str) -> bool:
|
||||
try:
|
||||
ipaddress.ip_address(host.strip("[]"))
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
@@ -20,6 +20,12 @@ COUNTER_NAMES: tuple[str, ...] = (
|
||||
"evicted_heartbeat",
|
||||
"kicked",
|
||||
"replaced",
|
||||
"jobs_assigned",
|
||||
"jobs_done",
|
||||
"jobs_failed",
|
||||
"jobs_refused",
|
||||
"integrity_failures",
|
||||
"results_truncated",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import time
|
||||
from functools import lru_cache
|
||||
from datetime import datetime, timezone
|
||||
from typing import Callable
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from devplacepy.services.nodes.circuit import CircuitBreaker
|
||||
from devplacepy.services.nodes.policy import Target
|
||||
from devplacepy.services.nodes.registry import NodeRegistry
|
||||
from devplacepy.services.nodes.session import NodeSession
|
||||
from devplacepy.services.nodes.spec import MODE_ANY, MODE_FETCH, MODE_TUNNEL, SPEC
|
||||
|
||||
LIMITS = SPEC.limits
|
||||
MEGABYTE = 1024 * 1024
|
||||
MINUTES_PER_DAY = 24 * 60
|
||||
|
||||
SUPPORTED_MODES = (MODE_TUNNEL, MODE_FETCH)
|
||||
|
||||
|
||||
def _minutes(value: str) -> int | None:
|
||||
try:
|
||||
hours, minutes = value.split(":")
|
||||
return int(hours) * 60 + int(minutes)
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _zone(name: str):
|
||||
try:
|
||||
return ZoneInfo(name or "UTC")
|
||||
except (ZoneInfoNotFoundError, ValueError):
|
||||
return timezone.utc
|
||||
|
||||
|
||||
@lru_cache(maxsize=8192)
|
||||
def _window_open(start: str, end: str, zone_name: str, minute_epoch: int) -> bool:
|
||||
first, last = _minutes(start), _minutes(end)
|
||||
if first is None or last is None or first == last:
|
||||
return True
|
||||
moment = datetime.fromtimestamp(minute_epoch * 60, timezone.utc)
|
||||
local = moment.astimezone(_zone(zone_name))
|
||||
now = local.hour * 60 + local.minute
|
||||
if first < last:
|
||||
return first <= now < last
|
||||
return now >= first or now < last
|
||||
|
||||
|
||||
def in_active_hours(settings: dict, moment: datetime) -> bool:
|
||||
start, end = settings.get("active_from") or "", settings.get("active_to") or ""
|
||||
if not start and not end:
|
||||
return True
|
||||
return _window_open(start, end, settings.get("timezone") or "UTC", int(moment.timestamp() // 60))
|
||||
|
||||
|
||||
class NodeScheduler:
|
||||
def __init__(
|
||||
self,
|
||||
registry: NodeRegistry,
|
||||
breaker: CircuitBreaker,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
wall: Callable[[], datetime] = lambda: datetime.now(timezone.utc),
|
||||
rng: random.Random | None = None,
|
||||
) -> None:
|
||||
self.registry = registry
|
||||
self.breaker = breaker
|
||||
self._clock = clock
|
||||
self._wall = wall
|
||||
self._rng = rng or random.Random()
|
||||
self._destination_started: dict[str, float] = {}
|
||||
|
||||
@staticmethod
|
||||
def node_key(session: NodeSession) -> tuple[str, str]:
|
||||
return ("node", session.device_uid)
|
||||
|
||||
@staticmethod
|
||||
def destination_key(target: Target) -> tuple[str, str]:
|
||||
return ("destination", target.destination.suffix)
|
||||
|
||||
def transports_for(self, target: Target, mode: str) -> tuple[str, ...]:
|
||||
if mode not in (MODE_ANY, *SUPPORTED_MODES):
|
||||
return ()
|
||||
return tuple(
|
||||
item
|
||||
for item in SUPPORTED_MODES
|
||||
if item in target.destination.modes and mode in (MODE_ANY, item)
|
||||
)
|
||||
|
||||
def transport_of(self, session: NodeSession, target: Target, mode: str) -> str | None:
|
||||
for item in self.transports_for(target, mode):
|
||||
if item in session.modes:
|
||||
return item
|
||||
return None
|
||||
|
||||
def destination_ready(self, target: Target) -> bool:
|
||||
interval = 1.0 / max(target.destination.max_rps, 0.001)
|
||||
last = self._destination_started.get(target.destination.suffix)
|
||||
return last is None or self._clock() - last >= interval
|
||||
|
||||
def node_ready(self, session: NodeSession, target: Target, transport: str) -> bool:
|
||||
if session.closed or self._clock() < session.paused_until:
|
||||
return False
|
||||
if transport not in session.modes:
|
||||
return False
|
||||
if len(session.inflight) >= session.max_concurrent:
|
||||
return False
|
||||
if target.adapter not in (session.settings.get("allowed_platforms") or []):
|
||||
return False
|
||||
if not in_active_hours(session.settings, self._wall()):
|
||||
return False
|
||||
session.roll_day()
|
||||
cap = int(session.settings.get("bandwidth_mb_day") or 0) * MEGABYTE
|
||||
if session.used_today >= cap:
|
||||
return False
|
||||
last = session.host_started.get(target.destination.suffix)
|
||||
if last is not None and self._clock() - last < LIMITS.min_seconds_per_node_domain:
|
||||
return False
|
||||
return self.breaker.allows(self.node_key(session))
|
||||
|
||||
def candidates(self, target: Target, mode: str = MODE_ANY) -> list[NodeSession]:
|
||||
if not self.destination_ready(target) or not self.breaker.allows(self.destination_key(target)):
|
||||
return []
|
||||
for transport in self.transports_for(target, mode):
|
||||
found = [
|
||||
session
|
||||
for session in self.registry.sessions()
|
||||
if self.node_ready(session, target, transport)
|
||||
]
|
||||
if found:
|
||||
return found
|
||||
return []
|
||||
|
||||
def eligible(self, target: Target, mode: str = MODE_ANY, caller_owner: str = "") -> list[NodeSession]:
|
||||
found = self.candidates(target, mode)
|
||||
if not caller_owner:
|
||||
return found
|
||||
return [session for session in found if session.user_uid != caller_owner]
|
||||
|
||||
def choose(self, found: list[NodeSession], caller_owner: str = "") -> NodeSession | None:
|
||||
if caller_owner:
|
||||
found = [session for session in found if session.user_uid != caller_owner]
|
||||
if not found:
|
||||
return None
|
||||
return min(
|
||||
found,
|
||||
key=lambda item: (len(item.inflight), item.last_assigned, self._rng.random()),
|
||||
)
|
||||
|
||||
def pick(self, target: Target, mode: str = MODE_ANY, caller_owner: str = "") -> NodeSession | None:
|
||||
return self.choose(self.candidates(target, mode), caller_owner)
|
||||
|
||||
def mark_started(self, session: NodeSession, target: Target) -> None:
|
||||
now = self._clock()
|
||||
session.last_assigned = now
|
||||
session.host_started[target.destination.suffix] = now
|
||||
self._destination_started[target.destination.suffix] = now
|
||||
self.breaker.begin(self.node_key(session))
|
||||
self.breaker.begin(self.destination_key(target))
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
|
||||
@@ -10,11 +11,17 @@ from devplacepy.database.nodes import (
|
||||
add_event,
|
||||
effective_status,
|
||||
get_devices_by_uids,
|
||||
get_settings_by_users,
|
||||
)
|
||||
from devplacepy.services.base import BaseService, ConfigField
|
||||
from devplacepy.services.nodes.broker import JobBroker
|
||||
from devplacepy.services.nodes.circuit import CircuitBreaker
|
||||
from devplacepy.services.nodes.consent import ConsentGate
|
||||
from devplacepy.services.nodes.dispatcher import NodeDispatcher
|
||||
from devplacepy.services.nodes.fabric import NodeFabric
|
||||
from devplacepy.services.nodes.ledger import UsageLedger
|
||||
from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry
|
||||
from devplacepy.services.nodes.scheduler import NodeScheduler
|
||||
from devplacepy.services.nodes.spec import SPEC
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -64,7 +71,52 @@ class NodesService(BaseService):
|
||||
self.registry = NodeRegistry(self.counters)
|
||||
self.gate = ConsentGate()
|
||||
self.ledger = UsageLedger()
|
||||
self.breaker = CircuitBreaker()
|
||||
self.broker = JobBroker()
|
||||
self.scheduler = NodeScheduler(self.registry, self.breaker)
|
||||
self.dispatcher = NodeDispatcher(
|
||||
self.registry,
|
||||
self.scheduler,
|
||||
self.broker,
|
||||
self.breaker,
|
||||
self.ledger,
|
||||
self.counters,
|
||||
)
|
||||
self.fabric = NodeFabric(broker=self.broker, gate=self.gate)
|
||||
self._last_prune = 0.0
|
||||
self._dispatch_task: asyncio.Task | None = None
|
||||
|
||||
def ensure_dispatching(self) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
task = self._dispatch_task
|
||||
if task is not None and not task.done() and task.get_loop() is loop:
|
||||
return
|
||||
self._dispatch_task = loop.create_task(self._dispatch_loop())
|
||||
|
||||
async def _dispatch_loop(self) -> None:
|
||||
pause = LIMITS.dispatcher_tick_ms / 1000
|
||||
while True:
|
||||
try:
|
||||
await self.dispatcher.tick()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("node dispatch tick failed")
|
||||
await asyncio.sleep(pause)
|
||||
|
||||
async def on_enable(self) -> None:
|
||||
self.ensure_dispatching()
|
||||
|
||||
async def _refresh_settings(self, sessions) -> None:
|
||||
fresh = get_settings_by_users(item.user_uid for item in sessions)
|
||||
for session in sessions:
|
||||
current = fresh.get(session.user_uid)
|
||||
if current is not None and current != session.settings:
|
||||
session.settings = current
|
||||
try:
|
||||
await session.send("settings_update", settings=current)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def reconcile(self) -> int:
|
||||
sessions = self.registry.sessions()
|
||||
@@ -87,6 +139,7 @@ class NodesService(BaseService):
|
||||
detail=reason,
|
||||
)
|
||||
kicked += 1
|
||||
await self._refresh_settings(self.registry.sessions())
|
||||
return kicked
|
||||
|
||||
@staticmethod
|
||||
@@ -113,6 +166,9 @@ class NodesService(BaseService):
|
||||
self.log(f"pruned {pruned}")
|
||||
|
||||
async def on_disable(self) -> None:
|
||||
if self._dispatch_task is not None:
|
||||
self._dispatch_task.cancel()
|
||||
self._dispatch_task = None
|
||||
await self.registry.kick_all("service_stopped")
|
||||
|
||||
def collect_metrics(self) -> dict:
|
||||
@@ -123,6 +179,9 @@ class NodesService(BaseService):
|
||||
{"label": "capacity", "value": self.registry.capacity()},
|
||||
{"label": "accepted", "value": counts["connected_total"]},
|
||||
{"label": "kicked", "value": counts["kicked"]},
|
||||
{"label": "jobs done", "value": counts["jobs_done"]},
|
||||
{"label": "jobs failed", "value": counts["jobs_failed"]},
|
||||
{"label": "open circuits", "value": self.breaker.open_count()},
|
||||
],
|
||||
"table": {
|
||||
"columns": ["counter", "value"],
|
||||
|
||||
@@ -4,8 +4,11 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from devplacepy.database.nodes import get_node_settings
|
||||
from devplacepy.database.nodes import get_node_settings, usage_today
|
||||
from devplacepy.services.nodes.frames import FrameError, decode, encode
|
||||
from devplacepy.services.nodes.spec import (
|
||||
CLIENT_TO_SERVER,
|
||||
@@ -20,6 +23,21 @@ LIMITS = SPEC.limits
|
||||
|
||||
END_HEARTBEAT = "heartbeat"
|
||||
END_UNEXPECTED_FRAME = "unexpected_frame"
|
||||
TUNNEL_FRAMES = ("tunnel_opened", "tunnel_data", "tunnel_close")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Inflight:
|
||||
job_id: str
|
||||
host: str
|
||||
destination: str
|
||||
method: str
|
||||
url: str
|
||||
started: float
|
||||
bytes_up: int
|
||||
|
||||
|
||||
JobSink = Callable[["NodeSession", Inflight, dict], Awaitable[None]]
|
||||
|
||||
|
||||
class NodeSession:
|
||||
@@ -34,6 +52,37 @@ class NodeSession:
|
||||
self.reported_up = 0
|
||||
self.reported_down = 0
|
||||
self.closed = False
|
||||
self.settings: dict = {}
|
||||
self.inflight: dict[str, Inflight] = {}
|
||||
self.host_started: dict[str, float] = {}
|
||||
self.used_today = 0
|
||||
self.used_day = ""
|
||||
self.paused_until = 0.0
|
||||
self.last_assigned = 0.0
|
||||
self.job_sink: JobSink | None = None
|
||||
self.tunnel_sink: Callable[["NodeSession", dict], bool] | None = None
|
||||
self.tunnels: dict = {}
|
||||
|
||||
@property
|
||||
def modes(self) -> list[str]:
|
||||
return list(self.hello.get("modes") or [])
|
||||
|
||||
@property
|
||||
def max_concurrent(self) -> int:
|
||||
limit = min(
|
||||
LIMITS.max_concurrent_per_node,
|
||||
int(self.settings.get("max_concurrent") or LIMITS.max_concurrent_per_node),
|
||||
)
|
||||
declared = (self.hello.get("limits") or {}).get("max_concurrent")
|
||||
if isinstance(declared, int) and not isinstance(declared, bool) and declared >= 1:
|
||||
limit = min(limit, declared)
|
||||
return max(1, limit)
|
||||
|
||||
def roll_day(self) -> None:
|
||||
today = datetime.now(timezone.utc).date().isoformat()
|
||||
if self.used_day != today:
|
||||
self.used_day = today
|
||||
self.used_today = 0
|
||||
|
||||
async def send(self, frame_type: str, **fields) -> None:
|
||||
await self.websocket.send_text(encode(frame_type, SERVER_TO_CLIENT, **fields))
|
||||
@@ -76,16 +125,20 @@ class NodeSession:
|
||||
raise FrameError("expected_hello", "the first frame must be hello")
|
||||
self._check_hello(frame)
|
||||
self.hello = frame
|
||||
self.settings = get_node_settings(self.user_uid)
|
||||
self.roll_day()
|
||||
today = usage_today(self.device_uid)
|
||||
self.used_today = today["bytes_up"] + today["bytes_down"]
|
||||
await self.send(
|
||||
"welcome",
|
||||
device_uid=self.device_uid,
|
||||
limits=LIMITS.describe(),
|
||||
destinations=[item.describe() for item in SPEC.destinations],
|
||||
settings=get_node_settings(self.user_uid),
|
||||
settings=self.settings,
|
||||
)
|
||||
return frame
|
||||
|
||||
def _handle(self, frame: dict) -> None:
|
||||
async def _handle(self, frame: dict) -> None:
|
||||
kind = frame["type"]
|
||||
if kind == "pong":
|
||||
return
|
||||
@@ -93,7 +146,18 @@ class NodeSession:
|
||||
self.reported_up += max(0, frame["bytes_up"])
|
||||
self.reported_down += max(0, frame["bytes_down"])
|
||||
return
|
||||
raise FrameError(END_UNEXPECTED_FRAME, f"unexpected {kind} frame, no job is assigned")
|
||||
if kind in TUNNEL_FRAMES:
|
||||
if self.tunnel_sink is not None:
|
||||
self.tunnel_sink(self, frame)
|
||||
return
|
||||
if kind in ("job_result", "job_error"):
|
||||
job = self.inflight.get(frame["job_id"])
|
||||
if job is not None and self.job_sink is not None:
|
||||
await self.job_sink(self, job, frame)
|
||||
return
|
||||
raise FrameError(
|
||||
END_UNEXPECTED_FRAME, f"unexpected {kind} frame, no such job is assigned"
|
||||
)
|
||||
|
||||
async def run(self) -> str:
|
||||
while not self.closed:
|
||||
@@ -109,5 +173,5 @@ class NodeSession:
|
||||
await self.send("ping")
|
||||
continue
|
||||
self.missed_pings = 0
|
||||
self._handle(decode(raw, CLIENT_TO_SERVER))
|
||||
await self._handle(decode(raw, CLIENT_TO_SERVER))
|
||||
return "closed"
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import zip_longest
|
||||
|
||||
from devplacepy.database import get_setting
|
||||
from devplacepy.services.nodes.adapters.base import AdapterError, SearchAdapter, SocialHit
|
||||
from devplacepy.services.nodes.adapters.direct import get as direct_get
|
||||
from devplacepy.services.nodes.adapters.registry import ADAPTERS, DEFAULT_PLATFORMS
|
||||
from devplacepy.services.nodes.fabric import FabricError, NodeFabric, shared_fabric
|
||||
|
||||
MAX_LIMIT = 20
|
||||
DEFAULT_LIMIT = 8
|
||||
MAX_QUERY_CHARS = 200
|
||||
MAX_TOTAL_HITS = 60
|
||||
PLATFORM_TIMEOUT_SECONDS = 20.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class SocialOutcome:
|
||||
hits: list[SocialHit] = field(default_factory=list)
|
||||
unavailable: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return {
|
||||
"hits": [item.as_dict() for item in self.hits],
|
||||
"unavailable": dict(self.unavailable),
|
||||
}
|
||||
|
||||
|
||||
class SocialSearch:
|
||||
def __init__(self, fabric: NodeFabric | None = None) -> None:
|
||||
self.fabric = fabric or shared_fabric()
|
||||
|
||||
@staticmethod
|
||||
def platforms_for(requested: list[str] | None) -> list[str]:
|
||||
if not requested:
|
||||
return list(DEFAULT_PLATFORMS)
|
||||
return [name for name in dict.fromkeys(requested) if name in ADAPTERS]
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
caller_kind: str,
|
||||
caller_id: str,
|
||||
platforms: list[str] | None = None,
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
) -> SocialOutcome:
|
||||
query = " ".join(str(query or "").split())[:MAX_QUERY_CHARS]
|
||||
outcome = SocialOutcome()
|
||||
if not query:
|
||||
return outcome
|
||||
limit = max(1, min(int(limit or DEFAULT_LIMIT), MAX_LIMIT))
|
||||
names = self.platforms_for(platforms)
|
||||
results = await asyncio.gather(
|
||||
*(self._one(ADAPTERS[name], query, limit, caller_kind, caller_id) for name in names)
|
||||
)
|
||||
buckets = []
|
||||
for name, (hits, code) in zip(names, results):
|
||||
if code:
|
||||
outcome.unavailable[name] = code
|
||||
buckets.append(hits)
|
||||
for tier in zip_longest(*buckets):
|
||||
outcome.hits.extend(item for item in tier if item)
|
||||
outcome.hits = outcome.hits[:MAX_TOTAL_HITS]
|
||||
return outcome
|
||||
|
||||
async def _one(
|
||||
self, adapter: SearchAdapter, query: str, limit: int, caller_kind: str, caller_id: str
|
||||
) -> tuple[list[SocialHit], str]:
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
self._fetch(adapter, query, limit, caller_kind, caller_id),
|
||||
PLATFORM_TIMEOUT_SECONDS,
|
||||
), ""
|
||||
except asyncio.TimeoutError:
|
||||
return [], "timeout"
|
||||
except (AdapterError, FabricError) as exc:
|
||||
return [], exc.code
|
||||
|
||||
async def _fetch(
|
||||
self, adapter: SearchAdapter, query: str, limit: int, caller_kind: str, caller_id: str
|
||||
) -> list[SocialHit]:
|
||||
url = adapter.request(query, limit)
|
||||
if adapter.direct:
|
||||
return adapter.parse(await direct_get(url), limit)
|
||||
if get_setting("nodes_social_via_nodes", "0") != "1":
|
||||
raise AdapterError("nodes_not_used", "Routing through donor nodes is switched off")
|
||||
result = await self.fabric.fetch(url, caller_kind=caller_kind, caller_id=caller_id)
|
||||
if result.status >= 400:
|
||||
raise AdapterError("upstream", f"HTTP {result.status}")
|
||||
return adapter.parse(result.body, limit)
|
||||
@@ -97,6 +97,13 @@ class LimitSpec:
|
||||
hello_timeout_seconds: int = 10
|
||||
reconcile_seconds: int = 2
|
||||
prune_interval_seconds: int = 3600
|
||||
breaker_failures: int = 5
|
||||
breaker_open_seconds: int = 60
|
||||
queue_max_wait_seconds: int = 30
|
||||
job_grace_seconds: int = 5
|
||||
dispatch_batch: int = 50
|
||||
url_max_chars: int = 2048
|
||||
poll_interval_ms: int = 100
|
||||
|
||||
def describe(self) -> dict[str, int]:
|
||||
return dict(self.__dict__)
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
||||
from devplacepy import stealth
|
||||
from devplacepy.services.nodes.policy import Target
|
||||
from devplacepy.services.nodes.session import NodeSession
|
||||
from devplacepy.services.nodes.spec import SPEC
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LIMITS = SPEC.limits
|
||||
|
||||
LOOPBACK = "127.0.0.1"
|
||||
CHUNK_BYTES = 48 * 1024
|
||||
REQUEST_HEAD_MAX = 8 * 1024
|
||||
OPEN_TIMEOUT_SECONDS = 10.0
|
||||
POLL_SECONDS = 1.0
|
||||
TLS_OVERHEAD_BYTES = 512 * 1024
|
||||
RELAY_CAP_BYTES = LIMITS.max_body_bytes * 2 + TLS_OVERHEAD_BYTES
|
||||
CLOSED = object()
|
||||
|
||||
|
||||
class TunnelError(Exception):
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TunnelResult:
|
||||
status: int
|
||||
headers: dict
|
||||
body: bytes
|
||||
truncated: bool
|
||||
bytes_up: int
|
||||
bytes_down: int
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, session: NodeSession, target: Target) -> None:
|
||||
self.session = session
|
||||
self.target = target
|
||||
self.conn_id = generate_uid()
|
||||
self.frames: asyncio.Queue = asyncio.Queue(maxsize=256)
|
||||
self.up = 0
|
||||
self.down = 0
|
||||
self.error = ""
|
||||
self.used = False
|
||||
|
||||
def on_frame(self, frame: dict) -> None:
|
||||
try:
|
||||
self.frames.put_nowait(frame)
|
||||
except asyncio.QueueFull:
|
||||
self.error = "queue_full"
|
||||
|
||||
def drop(self) -> None:
|
||||
try:
|
||||
self.frames.put_nowait(CLOSED)
|
||||
except asyncio.QueueFull:
|
||||
pass
|
||||
|
||||
async def handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
||||
try:
|
||||
await self._serve(reader, writer)
|
||||
except (ConnectionError, asyncio.IncompleteReadError, asyncio.LimitOverrunError):
|
||||
pass
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
async def _serve(self, reader, writer) -> None:
|
||||
head = await reader.readuntil(b"\r\n\r\n")
|
||||
if len(head) > REQUEST_HEAD_MAX:
|
||||
return
|
||||
parts = head.split(b"\r\n", 1)[0].decode("latin-1", "replace").split()
|
||||
if len(parts) < 2 or parts[0].upper() != "CONNECT" or self.used:
|
||||
await self._refuse(writer, 405)
|
||||
return
|
||||
host, _, port = parts[1].rpartition(":")
|
||||
if host.lower() != self.target.host.lower() or port != str(self.target.port):
|
||||
self.error = "host_not_allowed"
|
||||
await self._refuse(writer, 403)
|
||||
return
|
||||
self.used = True
|
||||
self.session.tunnels[self.conn_id] = self
|
||||
await self.session.send("tunnel_open", conn_id=self.conn_id, host=self.target.host, port=self.target.port)
|
||||
if not await self._opened():
|
||||
await self._refuse(writer, 502)
|
||||
return
|
||||
writer.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
||||
await writer.drain()
|
||||
uplink = asyncio.create_task(self._uplink(reader))
|
||||
try:
|
||||
await self._downlink(writer)
|
||||
finally:
|
||||
uplink.cancel()
|
||||
await self._close_donor()
|
||||
|
||||
@staticmethod
|
||||
async def _refuse(writer, status: int) -> None:
|
||||
writer.write(f"HTTP/1.1 {status} Refused\r\nContent-Length: 0\r\n\r\n".encode())
|
||||
await writer.drain()
|
||||
|
||||
async def _opened(self) -> bool:
|
||||
try:
|
||||
frame = await asyncio.wait_for(self.frames.get(), OPEN_TIMEOUT_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
self.error = "tunnel_open_timeout"
|
||||
return False
|
||||
if frame is CLOSED or frame["type"] != "tunnel_opened" or not frame["ok"]:
|
||||
self.error = (frame.get("code") if isinstance(frame, dict) else "") or "tunnel_refused"
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _uplink(self, reader) -> None:
|
||||
while True:
|
||||
chunk = await reader.read(CHUNK_BYTES)
|
||||
if not chunk:
|
||||
return
|
||||
self.up += len(chunk)
|
||||
if self.up + self.down > RELAY_CAP_BYTES:
|
||||
self.error = "body_too_large"
|
||||
self.drop()
|
||||
return
|
||||
await self.session.send(
|
||||
"tunnel_data", conn_id=self.conn_id, b64=base64.b64encode(chunk).decode()
|
||||
)
|
||||
|
||||
async def _downlink(self, writer) -> None:
|
||||
while not self.error:
|
||||
try:
|
||||
frame = await asyncio.wait_for(self.frames.get(), POLL_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
if frame is CLOSED or frame["type"] == "tunnel_close":
|
||||
return
|
||||
if frame["type"] != "tunnel_data":
|
||||
self.error = "bad_frame"
|
||||
return
|
||||
try:
|
||||
data = base64.b64decode(frame["b64"].encode(), validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
self.error = "bad_base64"
|
||||
return
|
||||
self.down += len(data)
|
||||
if self.up + self.down > RELAY_CAP_BYTES:
|
||||
self.error = "body_too_large"
|
||||
return
|
||||
writer.write(data)
|
||||
await writer.drain()
|
||||
|
||||
async def _close_donor(self) -> None:
|
||||
self.session.tunnels.pop(self.conn_id, None)
|
||||
try:
|
||||
await self.session.send("tunnel_close", conn_id=self.conn_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class TunnelBridge:
|
||||
async def fetch(self, session: NodeSession, target: Target, timeout: float) -> TunnelResult:
|
||||
relay = Relay(session, target)
|
||||
server = await asyncio.start_server(relay.handle, LOOPBACK, 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
self._request(relay, target, f"http://{LOOPBACK}:{port}", timeout), timeout
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise TunnelError("timeout")
|
||||
finally:
|
||||
server.close()
|
||||
session.tunnels.pop(relay.conn_id, None)
|
||||
|
||||
async def _request(self, relay: Relay, target: Target, proxy: str, timeout: float) -> TunnelResult:
|
||||
limit = LIMITS.max_body_bytes
|
||||
try:
|
||||
async with stealth.stealth_async_client(
|
||||
proxy=proxy, follow_redirects=False, timeout=httpx.Timeout(timeout, connect=OPEN_TIMEOUT_SECONDS + 2)
|
||||
) as client:
|
||||
async with client.stream(target.method, target.url) as response:
|
||||
chunks = bytearray()
|
||||
truncated = False
|
||||
async for chunk in response.aiter_bytes():
|
||||
chunks.extend(chunk)
|
||||
if len(chunks) > limit:
|
||||
truncated = True
|
||||
del chunks[limit:]
|
||||
break
|
||||
return TunnelResult(
|
||||
response.status_code,
|
||||
dict(response.headers),
|
||||
bytes(chunks),
|
||||
truncated,
|
||||
relay.up,
|
||||
relay.down,
|
||||
)
|
||||
except httpx.HTTPError as exc:
|
||||
raise TunnelError(relay.error or f"tunnel_failed:{type(exc).__name__}")
|
||||
|
||||
def on_frame(self, session: NodeSession, frame: dict) -> bool:
|
||||
relay = session.tunnels.get(frame["conn_id"])
|
||||
if relay is None:
|
||||
return False
|
||||
relay.on_frame(frame)
|
||||
return True
|
||||
|
||||
def release_session(self, session: NodeSession) -> None:
|
||||
for relay in list(session.tunnels.values()):
|
||||
relay.drop()
|
||||
session.tunnels.clear()
|
||||
@@ -0,0 +1,87 @@
|
||||
This file documents community sync (`devplacepy/services/sync/`, `devplacepy/routers/sync/`, `devplacepy/database/sync.py`): copying users, posts and comments in from another DevPlace-like community. Claude Code auto-loads it whenever a file under `devplacepy/services/sync/` is read or edited.
|
||||
|
||||
## What exists today
|
||||
|
||||
| Piece | File | State |
|
||||
|-------|------|-------|
|
||||
| Specification (limits, kinds, statuses, reasons) | `spec.py` | built, `GET /sync/spec.json` adds the record JSON schemas |
|
||||
| Normalizers | `normalize.py` | built, tested, pure functions |
|
||||
| Pydantic record models and envelope | `records.py` | built, tested |
|
||||
| Link table helpers | `database/sync.py`, `SYNC_COLUMNS` in `database/schema.py` | built, tested |
|
||||
| Per-kind upsert logic | `processors.py` (`RecordProcessor`, `UserProcessor`, `PostProcessor`, `CommentProcessor`) | built, tested |
|
||||
| Batch runner | `batch.py` (`BatchRunner`) | built |
|
||||
| Routes | `routers/sync/` | built, tested |
|
||||
| Devii tools `sync_users`, `sync_posts`, `sync_comments` | `services/devii/actions/catalog/sync.py` | built, `requires_admin`; Devii sends forms, so `records` is a JSON array string (`SyncBatchForm.decode_records`) |
|
||||
| Admin settings fields `sync_enabled`, `sync_origins` | `AdminSettingsForm`, `admin_settings.html`, `admin_save_settings` | built, origins normalized on save |
|
||||
| Scheduled sync service, export endpoint, vote and reaction sync | none | not built, see Gaps |
|
||||
|
||||
## Model
|
||||
|
||||
Every synced row has a `sync_links` row `(origin, kind, origin_uid) -> local_uid`, unique on the triple. `origin` is the sender's stable id (a hostname, lower-cased, 2-63 characters of `a-z0-9.-`). `kind` is `users`, `posts` or `comments`. The link also stores a `fingerprint` of the normalized record (sha256 prefix, excludes `updated_at`) and `origin_updated_at`.
|
||||
|
||||
Per record, `RecordProcessor.apply` does: validate (Pydantic), resolve references (author, post, parent) through the links, content filter, then:
|
||||
|
||||
| Situation | Result |
|
||||
|-----------|--------|
|
||||
| no link, not deleted | `created`, link claimed first, row inserted second |
|
||||
| no link, `deleted: true` | `unchanged`, reason `tombstone_for_unknown`, nothing stored |
|
||||
| link, local row missing | `rejected`, reason `purged_locally`, never recreated |
|
||||
| incoming `updated_at` older than the link's | `stale` |
|
||||
| local row soft-deleted by anyone but `sync` | `unchanged`, reason `locally_deleted` (moderation wins) |
|
||||
| same fingerprint | `unchanged` |
|
||||
| otherwise | `updated`, link advanced |
|
||||
|
||||
A record that fails validation is `rejected` with reason `invalid` and up to `max_errors_reported` field messages. An unexpected exception is `failed` with reason `internal_error`, logged with the traceback. One bad record never fails the batch; the HTTP status is `200` unless the envelope itself is invalid (`422`), the caller is not an administrator (`401`/`403`), or the kill switch or allowlist refuses (`503`/`403`).
|
||||
|
||||
## Rules that are deliberate
|
||||
|
||||
- **Counters are not imported.** Votes, stars, XP and levels cannot be verified across communities. They are not fields on the models, so Pydantic drops them and the result lists them under `ignored`. Synced posts have `stars = 0`; users start at XP 0, level 1. Email, password, role and API key are never accepted either.
|
||||
- **Synced accounts cannot sign in.** The password hash is a valid pbkdf2 hash of a random secret nobody knows (`unusable_password_hash`, computed once per process), the email is the placeholder `<local uid>@sync.invalid`, role is always `Member`. `account_row` in `utils/accounts.py` is the shared row builder, so a new `users` column is added in one place. Synced accounts never go through `_create_account`, so an empty database cannot make a synced user the first administrator.
|
||||
- **No side effects.** No notifications, no XP, no badges, no mention fan-out, no AI correction or modifier, no SEO metadata job. The content filter runs on every non-deleted record (`screen_fields`, `ContentRefused` becomes `rejected` / `content_refused`) and `record_screening` files reports the same way a native post does.
|
||||
- **The origin is authoritative for a synced row, except where this community already acted.** Local soft deletes by moderators or owners are never undone and a locally purged row is never recreated.
|
||||
- **Tombstones.** `deleted: true` soft-deletes posts and comments with `deleted_by = "sync"` (so only sync can revive them) and sets `users.is_active = False` for users. There is no cascade: the origin sends a tombstone per record.
|
||||
- **Usernames** are normalized to `[A-Za-z0-9_-]`, 3-32 characters. A name taken by a different account gets `-<origin label>` and then a number (`username_candidates`), so the stored name may differ from the sent one. An update renames only to a free candidate.
|
||||
- **Timestamps** are parsed from ISO 8601 or epoch seconds, converted to UTC, clamped to `[2000-01-01, now + 5 minutes]`, and `updated_at` is never earlier than `created_at`.
|
||||
- **Caps are static** (`SyncLimits`): 50 records per batch, field lengths mirror the native forms. `GET /sync/spec.json` is generated from the same dataclass.
|
||||
- Off by default: `sync_enabled` (`0`) is the kill switch, `sync_origins` an optional comma separated allowlist. Both are site settings and apply live.
|
||||
|
||||
## Concurrency (measured)
|
||||
|
||||
The link is claimed with one `INSERT OR IGNORE` against the unique index and `rowcount` decides the winner; the row is inserted after. 16 separate OS processes racing one new record against a seeded database gave exactly 1 `created` and 15 `unchanged` in 5 of 5 runs, one link, one user. A loser can see the link before the winner's row exists, so `find_local` waits up to `CLAIM_WAIT_ATTEMPTS * CLAIM_WAIT_SECONDS` (2 s) on a link younger than that window, calling `refresh_snapshot()` before each read because a cached read transaction otherwise never sees the winner's commit. Without the wait, 3 of 4 runs returned a false `purged_locally` to one loser. A genuinely purged row is only waited for when its link is younger than 2 s.
|
||||
|
||||
A stateful fuzz (3000 random upserts, tombstones and edits over two origins and all three kinds, invariants checked every 100 steps: no dangling links, no duplicate usernames, no orphan comments, parents on the same post, `updated_at >= created_at`) produced no `failed` result and no violation.
|
||||
|
||||
## Gaps
|
||||
|
||||
- **No export endpoint and no scheduled sync service.** The sender today is a script reading the source community's own API. The service with one job per community is the next step; it should keep a per-origin watermark on `updated_at` and call these endpoints in the order users, posts, comments.
|
||||
- **Votes, reactions, follows, attachments, polls and project links are not synced.** Counters will not match the origin.
|
||||
- **A comment's post and parent must be sent in an earlier batch or earlier in the same batch.** There is no deferred resolution queue; the result reason tells the sender what to send first.
|
||||
- **Privacy disclosure.** Synced profile data (username, bio, location, links) comes from a third party. `/docs/privacy.html` has not been updated for it.
|
||||
- A user's own deletion on this side is final, but nothing tells the origin; the next sync of that user is `rejected` / `purged_locally` forever.
|
||||
|
||||
## Selection, identity aliasing and translation (community runs only)
|
||||
|
||||
The design record is `community_sync_additional.md`. Everything is optional and off by default, stored per community in `communities.sync_options` (JSON, `options.py` `SyncOptions`), edited only on the administrator page `/admin/communities/{slug}`. Ordinary users never see it: no public route, schema field, docs page or Devii tool for non-administrators exposes the options, the identity table or any original name. The master kill switch is the `community_alias_enabled` config field of the Community sync service (default off); while it is off an aliased community does not run and no identity is created.
|
||||
|
||||
Order inside `CommunityRunner.prepare`, before `SyncPusher`: `Selector` (`selection.py`) then `Aliaser` (`aliaser.py`).
|
||||
|
||||
| Piece | File | Rule |
|
||||
|-------|------|------|
|
||||
| Selection | `selection.py` | Deterministic: `sample_percent` buckets `sha256(origin:origin_uid) % 100`, so the same post always gets the same answer. `min_comments` counts comments collected in this run. `topics` allow-list. A post that already has a link is never dropped, so edits keep flowing. Comments of a dropped post are dropped. |
|
||||
| Jitter | `selection.jittered`, `CommunityStore.schedule_next` | `jitter_percent` (max 50) randomises the next run time; the 900 s floor still applies. |
|
||||
| Identity map | `identity.py`, table `sync_identities` | Unique on `(origin, source_key)` and on `our_key` (global). Created once per source name, never changed. The AI sees only a style fingerprint (casing, length, digit, separator) and a forbidden list, never the original name. A candidate is rejected when it shares 4 characters with the source, contains a banned fragment, mismatches the style, or collides with an identity or a local user. After 3 AI attempts a deterministic syllable generator is used. Claimed with `INSERT OR IGNORE`, the loser re-reads the winner. Hard cap 100 new identities per run. |
|
||||
| Mention rewriter | `rewriter.py` | One regex pass, longest name first, case insensitive, `@name` always, a bare name only when it is at least 4 characters (a 3 character handle such as `tef` is otherwise indistinguishable from a word). Fenced code, inline code and URLs are untouched. |
|
||||
| Aliaser | `aliaser.py` | The local clone is a user record keyed by the lower-cased source name (`origin_uid = source_key`), with the alias as username and no bio, location, links or avatar seed. Titles, bodies and translations all pass the rewriter. A mentioned name without an alias skips the item, as does an author without one. |
|
||||
| Translation | `translator.py` | Runs after the rewrite, then the rewriter runs again on the output. Up to 40 per run. Up to 3 samples of 200 characters per identity are stored (`style_samples`) and sent along to keep the voice. Skipped when the community language equals the target. |
|
||||
|
||||
Fail closed: any exception, missing identity, missing author or failed translation drops that item and counts it under `report.pushed["skipped"]`; the original name is never a fallback. Transient skips (`identity_failed`, `identity_cap`, `translation_failed`, `rewrite_failed`) hold the watermark, so the next run retries them and the run is `partial`. Dry runs project identities in memory and write none.
|
||||
|
||||
Enabling aliasing on an origin that already has links is refused (form and runner), because those rows hold original names. The ledger is `sync_links` (fingerprint is the content hash, so edits re-run the rewrite and update only that row); there is no second ledger.
|
||||
|
||||
Deliberately not built:
|
||||
|
||||
- **Outbound posting to the source community.** Only the reverse mapping exists (the identity map is bidirectional, and the rewriter works in both directions). Posting under invented identities into another community is not implemented.
|
||||
- **Visibility delay.** Items become visible immediately; a delayed publish needs a watermark that tolerates deferred items.
|
||||
- **Avatars** (explicitly out of scope).
|
||||
|
||||
Gaps: `min_comments` only sees comments collected in the same run; a clone renamed by an administrator no longer matches its alias; a username taken by a registration between alias generation and clone creation gets a suffix and diverges from the rewritten mentions.
|
||||
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,26 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from devplacepy.config import INTERNAL_MODEL
|
||||
from devplacepy.database import get_setting
|
||||
from devplacepy.database.settings import internal_gateway_key
|
||||
|
||||
logger = logging.getLogger("sync.ai")
|
||||
|
||||
|
||||
|
||||
async def complete(system: str, text: str, timeout: float) -> str | None:
|
||||
from devplacepy.services.correction import gateway_complete
|
||||
|
||||
model = get_setting("community_ai_model", "") or INTERNAL_MODEL
|
||||
reply, usage = await asyncio.to_thread(
|
||||
gateway_complete, internal_gateway_key(), system, text, timeout, None, model, True, False
|
||||
)
|
||||
if usage is None:
|
||||
logger.warning("the %s model is unreachable", model)
|
||||
return None
|
||||
return reply
|
||||
@@ -0,0 +1,213 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from devplacepy.database.sync import count_links
|
||||
from devplacepy.services.sync.identity import IdentityResolver, source_key
|
||||
from devplacepy.services.sync.options import SyncOptions
|
||||
from devplacepy.services.sync.rewriter import MentionRewriter
|
||||
from devplacepy.services.sync.spec import (
|
||||
ALIAS,
|
||||
SKIP_IDENTITY,
|
||||
SKIP_IDENTITY_CAP,
|
||||
SKIP_NO_AUTHOR,
|
||||
SKIP_PARENT,
|
||||
SKIP_REWRITE,
|
||||
SKIP_TRANSLATION,
|
||||
TRANSIENT_SKIPS,
|
||||
)
|
||||
from devplacepy.services.sync.translator import Translator
|
||||
|
||||
logger = logging.getLogger("sync.alias")
|
||||
|
||||
|
||||
class AliasRefused(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ItemSkipped(Exception):
|
||||
def __init__(self, reason: str):
|
||||
self.reason = reason
|
||||
super().__init__(reason)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AliasOutcome:
|
||||
records: dict[str, list[dict]]
|
||||
skipped: Counter = field(default_factory=Counter)
|
||||
substitutions: int = 0
|
||||
translated: int = 0
|
||||
identities_created: int = 0
|
||||
|
||||
def transient(self) -> int:
|
||||
return sum(self.skipped[reason] for reason in TRANSIENT_SKIPS)
|
||||
|
||||
|
||||
class Aliaser:
|
||||
def __init__(self, origin: str, options: SyncOptions, *, language: str = "", dry_run: bool = False, resolver: IdentityResolver | None = None, translator: Translator | None = None):
|
||||
self.origin = origin
|
||||
self.options = options
|
||||
self.language = language
|
||||
self.resolver = resolver or IdentityResolver(origin, persist=not dry_run)
|
||||
self.translator = translator or (Translator(options.translate_to) if options.translate_to else None)
|
||||
self.rewriter = MentionRewriter({})
|
||||
self.outcome = AliasOutcome({})
|
||||
self.authors: dict[str, str] = {}
|
||||
|
||||
def check_origin(self) -> None:
|
||||
if self.options.alias and not self.resolver.rows and count_links(self.origin):
|
||||
raise AliasRefused(
|
||||
f"{self.origin} already has synced records stored under their original names; "
|
||||
"aliasing cannot be switched on afterwards"
|
||||
)
|
||||
|
||||
async def apply(self, records: dict[str, list[dict]]) -> AliasOutcome:
|
||||
if self.options.alias:
|
||||
self.check_origin()
|
||||
self.outcome = AliasOutcome({"users": [], "posts": [], "comments": []})
|
||||
if self.options.alias:
|
||||
await self.map_names(records)
|
||||
self.rewriter = MentionRewriter(self.resolver.mapping())
|
||||
users = {str(user.get("origin_uid")): user for user in records.get("users", [])}
|
||||
kept_posts = set()
|
||||
for post in records.get("posts", []):
|
||||
kept = await self.process(post, users, ("title", "content"))
|
||||
if kept:
|
||||
self.outcome.records["posts"].append(kept)
|
||||
kept_posts.add(str(kept.get("origin_uid")))
|
||||
placed: set[str] = set()
|
||||
for comment in records.get("comments", []):
|
||||
parent = comment.get("parent_uid")
|
||||
if str(comment.get("post_uid")) not in kept_posts or (parent and str(parent) not in placed):
|
||||
self.outcome.skipped[SKIP_PARENT] += 1
|
||||
continue
|
||||
kept = await self.process(comment, users, ("content",))
|
||||
if kept:
|
||||
self.outcome.records["comments"].append(kept)
|
||||
placed.add(str(comment.get("origin_uid")))
|
||||
self.outcome.records["users"] = self.users_for(records, users)
|
||||
self.outcome.identities_created = self.resolver.created
|
||||
return self.outcome
|
||||
|
||||
def author_name(self, record: dict, users: dict[str, dict]) -> str | None:
|
||||
user = users.get(str(record.get("author_uid")))
|
||||
return str(user.get("username")) if user and user.get("username") else None
|
||||
|
||||
async def map_names(self, records: dict[str, list[dict]]) -> None:
|
||||
users = {str(user.get("origin_uid")): user for user in records.get("users", [])}
|
||||
scan = MentionRewriter({})
|
||||
names: dict[str, str] = {}
|
||||
for kind, fields in (("posts", ("title", "content")), ("comments", ("content",))):
|
||||
for record in records.get(kind, []):
|
||||
author = self.author_name(record, users)
|
||||
if author:
|
||||
names.setdefault(author.lower(), author)
|
||||
for name in fields:
|
||||
for mention in scan.mentioned(str(record.get(name) or "")):
|
||||
names.setdefault(mention.lower(), mention)
|
||||
for name in names.values():
|
||||
await self.resolver.ensure(name)
|
||||
|
||||
async def process(self, record: dict, users: dict[str, dict], fields: tuple[str, ...]) -> dict | None:
|
||||
try:
|
||||
return await self.transform(record, users, fields)
|
||||
except ItemSkipped as skipped:
|
||||
self.outcome.skipped[skipped.reason] += 1
|
||||
logger.info("skipped %s %s: %s", self.origin, record.get("origin_uid"), skipped.reason)
|
||||
except Exception:
|
||||
self.outcome.skipped[SKIP_REWRITE] += 1
|
||||
logger.exception("fail closed: %s %s skipped", self.origin, record.get("origin_uid"))
|
||||
return None
|
||||
|
||||
async def transform(self, record: dict, users: dict[str, dict], fields: tuple[str, ...]) -> dict:
|
||||
author = self.author_name(record, users)
|
||||
if not author:
|
||||
raise ItemSkipped(SKIP_NO_AUTHOR)
|
||||
result = dict(record)
|
||||
if self.options.alias:
|
||||
alias = self.resolver.known(author)
|
||||
if not alias:
|
||||
raise ItemSkipped(SKIP_IDENTITY_CAP if self.resolver.created >= self.resolver.cap else SKIP_IDENTITY)
|
||||
key = source_key(author)
|
||||
result["author_uid"] = key
|
||||
for name in fields:
|
||||
for mention in self.rewriter.mentioned(str(result.get(name) or "")):
|
||||
if self.resolver.known(mention) is None:
|
||||
raise ItemSkipped(SKIP_IDENTITY_CAP if self.resolver.created >= self.resolver.cap else SKIP_IDENTITY)
|
||||
result[name] = self.rewritten(str(result.get(name) or ""))
|
||||
if self.translator and self.translator.needed(self.language):
|
||||
await self.translate(result, fields, author)
|
||||
if self.options.alias:
|
||||
self.authors[source_key(author)] = author
|
||||
return result
|
||||
|
||||
def rewritten(self, text: str) -> str:
|
||||
rewrite = self.rewriter.rewrite(text)
|
||||
self.outcome.substitutions += len(rewrite.substitutions)
|
||||
return rewrite.text
|
||||
|
||||
async def translate(self, result: dict, fields: tuple[str, ...], author: str) -> None:
|
||||
if self.outcome.translated >= ALIAS.max_translations_per_run:
|
||||
raise ItemSkipped(SKIP_TRANSLATION)
|
||||
samples = self.resolver.samples(author) if self.options.alias else []
|
||||
for name in fields:
|
||||
text = str(result.get(name) or "")
|
||||
if not text:
|
||||
continue
|
||||
translated = await self.translator.translate(text, samples)
|
||||
if translated is None:
|
||||
raise ItemSkipped(SKIP_TRANSLATION)
|
||||
result[name] = self.rewritten(translated) if self.options.alias else translated
|
||||
self.outcome.translated += 1
|
||||
if self.options.alias:
|
||||
self.resolver.remember(author, str(result.get(fields[-1]) or ""))
|
||||
|
||||
def users_for(self, records: dict[str, list[dict]], users: dict[str, dict]) -> list[dict]:
|
||||
if not self.options.alias:
|
||||
needed = {str(item.get("author_uid")) for kind in ("posts", "comments") for item in self.outcome.records[kind]}
|
||||
return [user for uid, user in users.items() if uid in needed] if self.options.selective() else list(users.values())
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
created: dict[str, str] = {}
|
||||
removed: set[str] = set()
|
||||
for user in users.values():
|
||||
if user.get("username"):
|
||||
created.setdefault(str(user["username"]).lower(), str(user.get("created_at") or now))
|
||||
if user.get("deleted"):
|
||||
removed.add(str(user["username"]).lower())
|
||||
built = []
|
||||
for key, author in self.authors.items():
|
||||
alias = self.resolver.known(author)
|
||||
if alias:
|
||||
built.append(self.clone(key, alias, created.get(author.lower(), now), author.lower() in removed))
|
||||
for key, alias in self.resolver.mapping().items():
|
||||
if key not in self.authors and self.mentioned_in_output(alias):
|
||||
built.append(self.clone(key, alias, created.get(key) or self.first_mention(alias, now), key in removed))
|
||||
return built
|
||||
|
||||
@staticmethod
|
||||
def clone(key: str, alias: str, created_at: str, deleted: bool) -> dict:
|
||||
return {"origin_uid": key, "username": alias, "created_at": created_at, "deleted": deleted}
|
||||
|
||||
def first_mention(self, alias: str, fallback: str) -> str:
|
||||
needle = f"@{alias}".lower()
|
||||
stamps = [
|
||||
str(item.get("created_at"))
|
||||
for kind in ("posts", "comments")
|
||||
for item in self.outcome.records[kind]
|
||||
if needle in f"{item.get('title') or ''} {item.get('content') or ''}".lower()
|
||||
]
|
||||
return min(stamps) if stamps else fallback
|
||||
|
||||
def mentioned_in_output(self, alias: str) -> bool:
|
||||
needle = f"@{alias}".lower()
|
||||
return any(
|
||||
needle in str(item.get(name) or "").lower()
|
||||
for kind, names in (("posts", ("title", "content")), ("comments", ("content",)))
|
||||
for item in self.outcome.records[kind]
|
||||
for name in names
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
|
||||
from devplacepy.schemas.sync import SyncBatchOut, SyncRecordResultOut
|
||||
from devplacepy.services.sync.processors import PROCESSORS
|
||||
from devplacepy.services.sync.records import SyncBatchForm
|
||||
from devplacepy.services.sync.spec import STATUSES
|
||||
|
||||
|
||||
class BatchRunner:
|
||||
def __init__(self, kind: str, *, request=None):
|
||||
self.kind = kind
|
||||
self.request = request
|
||||
|
||||
def run(self, data: SyncBatchForm) -> SyncBatchOut:
|
||||
processor = PROCESSORS[self.kind](
|
||||
data.origin, dry_run=data.dry_run, request=self.request
|
||||
)
|
||||
results = [
|
||||
SyncRecordResultOut(**vars(processor.process(index, raw)))
|
||||
for index, raw in enumerate(data.records)
|
||||
]
|
||||
tally = Counter(result.status for result in results)
|
||||
return SyncBatchOut(
|
||||
origin=data.origin,
|
||||
kind=self.kind,
|
||||
dry_run=data.dry_run,
|
||||
total=len(results),
|
||||
counts={status: tally.get(status, 0) for status in STATUSES},
|
||||
results=results,
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user