From b616bce0eaa4a260164af540aae0b6f2d11ab02e Mon Sep 17 00:00:00 2001 From: retoor Date: Wed, 14 Oct 2026 08:32:17 +0200 Subject: [PATCH] Add community node data layer: tables, usage ledger, bandwidth_sharing consent Phase 2 of plan.md. Seven node_* tables with atomic device, bandwidth and XP caps, a transactional job ledger whose lifetime totals survive pruning, clamped donor settings, and the bandwidth_sharing consent kind. The acceptance service now has an explicit exclusion list so it can never grant that consent. Co-Authored-By: Claude Sonnet 5.5 --- devplacepy/database/moderation.py | 9 + devplacepy/database/nodes.py | 639 +++++++++++++++++++ devplacepy/database/schema.py | 161 +++++ devplacepy/database/soft_delete.py | 5 + devplacepy/services/acceptance/CLAUDE.md | 5 +- devplacepy/services/acceptance/agreements.py | 7 + devplacepy/services/moderation/deletion.py | 4 + devplacepy/services/nodes/CLAUDE.md | 18 +- devplacepy/services/nodes/ledger.py | 77 +++ pyproject.toml | 2 +- tests/unit/database/nodes.py | 227 +++++++ tests/unit/services/acceptance/agreements.py | 6 +- 12 files changed, 1154 insertions(+), 6 deletions(-) create mode 100644 devplacepy/database/nodes.py create mode 100644 devplacepy/services/nodes/ledger.py create mode 100644 tests/unit/database/nodes.py diff --git a/devplacepy/database/moderation.py b/devplacepy/database/moderation.py index 07999ecf..bae9cfef 100644 --- a/devplacepy/database/moderation.py +++ b/devplacepy/database/moderation.py @@ -49,6 +49,11 @@ UNREPORTABLE_TABLES: dict[str, str] = { "bookmarks": "private to the owner", "notes": "private to the owner", "skills": "private to the owner, never shown to others", + "node_devices": "private to the owner, the label is never shown to others", + "node_settings": "private to the owner", + "node_usage_daily": "derived donation counters, carry no authored content", + "node_donors": "derived donation totals, carry no authored content", + "node_flags": "operational flags for administrators, carry no authored content", "follows": "relationship rows, carry no authored content", "poll_votes": "private ballots", "opinion_war_fighters": "membership and damage counters, carry no authored content", @@ -133,6 +138,10 @@ CONSENT_KINDS: dict[str, str] = { "Sharing your DevPlace credentials with software another member runs " "in a container" ), + "bandwidth_sharing": ( + "Sharing part of your internet connection with the DevPlace community " + "so search can reach sites that block datacenters" + ), } CONSENT_STATES: tuple[str, ...] = ("granted", "withdrawn") diff --git a/devplacepy/database/nodes.py b/devplacepy/database/nodes.py new file mode 100644 index 00000000..bc282c71 --- /dev/null +++ b/devplacepy/database/nodes.py @@ -0,0 +1,639 @@ +# retoor + +import hashlib +import hmac +import json +import re +import secrets +from datetime import datetime, timedelta, timezone + +from sqlalchemy import text + +from devplacepy.config import SECRET_KEY +from devplacepy.services.nodes.spec import DEVICE_KINDS, SPEC + +from .core import _now_iso, db, get_table +from .soft_delete import soft_delete + +LIMITS = SPEC.limits + +SECRET_PREFIX = "dpn_" +SECRET_PREFIX_LENGTH = 12 +SECRET_BYTES = 32 +LABEL_MAX = 60 +EVENT_DETAIL_MAX = 500 +MEGABYTE = 1024 * 1024 +HOURS_PATTERN = re.compile(r"^([01]\d|2[0-3]):[0-5]\d$") +PLATFORMS = tuple(sorted({item.adapter for item in SPEC.destinations})) +FLAG_OPEN = "open" +FLAG_RESOLVED = "resolved" +STATUS_ACTIVE = "active" +STATUS_SUSPENDED = "suspended" +STATUS_REVOKED = "revoked" + +DEFAULT_SETTINGS = { + "enabled": False, + "bandwidth_mb_day": LIMITS.default_donor_mb_per_day, + "max_concurrent": LIMITS.max_concurrent_per_node, + "active_from": "", + "active_to": "", + "timezone": "UTC", + "allowed_platforms": list(PLATFORMS), + "pause_on_battery": True, + "public_display": False, +} + + +class NodeLimitError(ValueError): + pass + + +def _uid() -> str: + from devplacepy.utils import generate_uid + + return generate_uid() + + +def _day(moment: datetime | None = None) -> str: + return (moment or datetime.now(timezone.utc)).date().isoformat() + + +def hash_ip(address: str) -> str: + if not address: + return "" + return hmac.new( + SECRET_KEY.encode(), address.encode(), hashlib.sha256 + ).hexdigest()[:32] + + +def _hash_secret(secret: str) -> str: + return hashlib.sha256(secret.encode()).hexdigest() + + +def new_secret() -> str: + return SECRET_PREFIX + secrets.token_urlsafe(SECRET_BYTES) + + +def _execute(sql: str, params: dict) -> int: + with db: + return db.executable.execute(text(sql), params).rowcount + + +def create_device( + user_uid: str, + label: str, + kind: str, + *, + platform: str = "", + client_version: str = "", +) -> tuple[dict, str]: + if kind not in DEVICE_KINDS: + raise NodeLimitError("Unknown device kind") + secret = new_secret() + uid = _uid() + now = _now_iso() + created = _execute( + "INSERT INTO node_devices (uid, user_uid, label, kind, secret_hash, " + "secret_prefix, platform, client_version, protocol_version, status, " + "suspended_until, suspend_reason, country, asn, asn_class, ip_hash, " + "last_seen_at, created_at, updated_at) " + "SELECT :uid, :user_uid, :label, :kind, :hash, :prefix, :platform, " + ":version, 0, 'active', '', '', '', '', 'unknown', '', '', :now, :now " + "WHERE (SELECT COUNT(*) FROM node_devices " + "WHERE user_uid = :user_uid AND deleted_at IS NULL) < :max_devices", + { + "uid": uid, + "user_uid": user_uid, + "label": (label or "").strip()[:LABEL_MAX], + "kind": kind, + "hash": _hash_secret(secret), + "prefix": secret[:SECRET_PREFIX_LENGTH], + "platform": (platform or "")[:40], + "version": (client_version or "")[:40], + "now": now, + "max_devices": LIMITS.max_devices_per_user, + }, + ) + if created != 1: + raise NodeLimitError( + f"A member can register at most {LIMITS.max_devices_per_user} devices" + ) + return get_device(uid), secret + + +def get_device(device_uid: str) -> dict | None: + return get_table("node_devices").find_one(uid=device_uid, deleted_at=None) + + +def list_devices(user_uid: str) -> list[dict]: + return list( + get_table("node_devices").find( + user_uid=user_uid, deleted_at=None, order_by=["created_at"] + ) + ) + + +def count_devices(user_uid: str) -> int: + return get_table("node_devices").count(user_uid=user_uid, deleted_at=None) + + +def find_device_by_secret(secret: str) -> dict | None: + if not secret or not secret.startswith(SECRET_PREFIX): + return None + digest = _hash_secret(secret) + for row in get_table("node_devices").find( + secret_prefix=secret[:SECRET_PREFIX_LENGTH], deleted_at=None + ): + if hmac.compare_digest(row.get("secret_hash") or "", digest): + return row + return None + + +def rotate_device_secret(device_uid: str, user_uid: str) -> str | None: + secret = new_secret() + changed = _execute( + "UPDATE node_devices SET secret_hash = :hash, secret_prefix = :prefix, " + "updated_at = :now WHERE uid = :uid AND user_uid = :user_uid " + "AND deleted_at IS NULL", + { + "hash": _hash_secret(secret), + "prefix": secret[:SECRET_PREFIX_LENGTH], + "now": _now_iso(), + "uid": device_uid, + "user_uid": user_uid, + }, + ) + return secret if changed == 1 else None + + +def revoke_device(device_uid: str, actor_uid: str) -> int: + get_table("node_devices").update( + {"uid": device_uid, "status": STATUS_REVOKED, "updated_at": _now_iso()}, + ["uid"], + ) + return soft_delete("node_devices", actor_uid, uid=device_uid) + + +def suspend_device(device_uid: str, reason: str, minutes: int) -> bool: + until = (datetime.now(timezone.utc) + timedelta(minutes=minutes)).isoformat() + return ( + _execute( + "UPDATE node_devices SET status = 'suspended', suspended_until = :until, " + "suspend_reason = :reason, updated_at = :now " + "WHERE uid = :uid AND deleted_at IS NULL AND status != 'revoked'", + { + "until": until, + "reason": (reason or "")[:EVENT_DETAIL_MAX], + "now": _now_iso(), + "uid": device_uid, + }, + ) + == 1 + ) + + +def unsuspend_device(device_uid: str) -> bool: + return ( + _execute( + "UPDATE node_devices SET status = 'active', suspended_until = '', " + "suspend_reason = '', updated_at = :now " + "WHERE uid = :uid AND deleted_at IS NULL AND status = 'suspended'", + {"now": _now_iso(), "uid": device_uid}, + ) + == 1 + ) + + +def effective_status(device: dict, moment: datetime | None = None) -> str: + status = device.get("status") or STATUS_ACTIVE + if status != STATUS_SUSPENDED: + return status + until = (device.get("suspended_until") or "").strip() + try: + expiry = datetime.fromisoformat(until) + except ValueError: + return STATUS_SUSPENDED + if expiry.tzinfo is None: + expiry = expiry.replace(tzinfo=timezone.utc) + now = moment or datetime.now(timezone.utc) + return STATUS_ACTIVE if expiry <= now else STATUS_SUSPENDED + + +def record_device_connection( + device_uid: str, + *, + platform: str = "", + client_version: str = "", + protocol_version: int = 0, + country: str = "", + asn: str = "", + asn_class: str = "unknown", + ip_hash: str = "", +) -> None: + get_table("node_devices").update( + { + "uid": device_uid, + "platform": platform[:40], + "client_version": client_version[:40], + "protocol_version": int(protocol_version), + "country": country[:2].upper(), + "asn": asn[:20], + "asn_class": asn_class, + "ip_hash": ip_hash, + "last_seen_at": _now_iso(), + }, + ["uid"], + ) + + +def _clamped(value, low: int, high: int, default: int) -> int: + try: + number = int(value) + except (TypeError, ValueError): + return default + return max(low, min(high, number)) + + +def _hours(value) -> str: + value = (value or "").strip() + return value if HOURS_PATTERN.match(value) else "" + + +def _platforms(value) -> list[str]: + if isinstance(value, str): + try: + value = json.loads(value) + except ValueError: + return list(PLATFORMS) + if not isinstance(value, (list, tuple)): + return list(PLATFORMS) + return [item for item in PLATFORMS if item in set(value)] + + +def normalise_settings(values: dict) -> dict: + return { + "enabled": bool(values.get("enabled")), + "bandwidth_mb_day": _clamped( + values.get("bandwidth_mb_day"), + 1, + LIMITS.max_donor_mb_per_day, + LIMITS.default_donor_mb_per_day, + ), + "max_concurrent": _clamped( + values.get("max_concurrent"), + 1, + LIMITS.max_concurrent_per_node, + LIMITS.max_concurrent_per_node, + ), + "active_from": _hours(values.get("active_from")), + "active_to": _hours(values.get("active_to")), + "timezone": (values.get("timezone") or "UTC").strip()[:64] or "UTC", + "allowed_platforms": _platforms(values.get("allowed_platforms")), + "pause_on_battery": bool(values.get("pause_on_battery")), + "public_display": bool(values.get("public_display")), + } + + +def get_node_settings(user_uid: str) -> dict: + row = get_table("node_settings").find_one(user_uid=user_uid, deleted_at=None) + if not row: + return dict(DEFAULT_SETTINGS) + return normalise_settings(row) + + +def save_node_settings(user_uid: str, values: dict) -> dict: + merged = {**get_node_settings(user_uid), **values} + clean = normalise_settings(merged) + now = _now_iso() + _execute( + "INSERT INTO node_settings (uid, user_uid, enabled, bandwidth_mb_day, " + "max_concurrent, active_from, active_to, timezone, allowed_platforms, " + "pause_on_battery, public_display, created_at, updated_at) " + "VALUES (:uid, :user_uid, :enabled, :mb, :concurrent, :hours_from, " + ":hours_to, :zone, :platforms, :battery, :public, :now, :now) " + "ON CONFLICT(user_uid) DO UPDATE SET enabled = excluded.enabled, " + "bandwidth_mb_day = excluded.bandwidth_mb_day, " + "max_concurrent = excluded.max_concurrent, " + "active_from = excluded.active_from, active_to = excluded.active_to, " + "timezone = excluded.timezone, " + "allowed_platforms = excluded.allowed_platforms, " + "pause_on_battery = excluded.pause_on_battery, " + "public_display = excluded.public_display, " + "updated_at = excluded.updated_at", + { + "uid": _uid(), + "user_uid": user_uid, + "enabled": int(clean["enabled"]), + "mb": clean["bandwidth_mb_day"], + "concurrent": clean["max_concurrent"], + "hours_from": clean["active_from"], + "hours_to": clean["active_to"], + "zone": clean["timezone"], + "platforms": json.dumps(clean["allowed_platforms"]), + "battery": int(clean["pause_on_battery"]), + "public": int(clean["public_display"]), + "now": now, + }, + ) + return clean + + +def daily_cap_bytes(user_uid: str) -> int: + return get_node_settings(user_uid)["bandwidth_mb_day"] * MEGABYTE + + +def record_job( + device_uid: str, + user_uid: str, + *, + ok: bool, + bytes_up: int = 0, + bytes_down: int = 0, + cache_hit: bool = False, + cap_bytes: int | None = None, + moment: datetime | None = None, +) -> bool: + now = (moment or datetime.now(timezone.utc)).isoformat() + day = _day(moment) + up = max(0, int(bytes_up)) + down = max(0, int(bytes_down)) + cap = daily_cap_bytes(user_uid) if cap_bytes is None else int(cap_bytes) + with db: + db.executable.execute( + text( + "INSERT OR IGNORE INTO node_usage_daily (uid, device_uid, user_uid, " + "day, requests_ok, requests_failed, bytes_up, bytes_down, " + "cache_hits_served, integrity_failures, created_at, updated_at) " + "VALUES (:uid, :device, :user, :day, 0, 0, 0, 0, 0, 0, :now, :now)" + ), + {"uid": _uid(), "device": device_uid, "user": user_uid, "day": day, "now": now}, + ) + if not ok: + db.executable.execute( + text( + "UPDATE node_usage_daily SET " + "requests_failed = COALESCE(requests_failed, 0) + 1, " + "updated_at = :now WHERE device_uid = :device AND day = :day" + ), + {"now": now, "device": device_uid, "day": day}, + ) + return True + counted = db.executable.execute( + text( + "UPDATE node_usage_daily SET " + "requests_ok = COALESCE(requests_ok, 0) + 1, " + "bytes_up = COALESCE(bytes_up, 0) + :up, " + "bytes_down = COALESCE(bytes_down, 0) + :down, " + "cache_hits_served = COALESCE(cache_hits_served, 0) + :hit, " + "updated_at = :now " + "WHERE device_uid = :device AND day = :day " + "AND COALESCE(bytes_up, 0) + COALESCE(bytes_down, 0) + :up + :down " + "<= :cap" + ), + { + "up": up, + "down": down, + "hit": 1 if cache_hit else 0, + "now": now, + "device": device_uid, + "day": day, + "cap": cap, + }, + ).rowcount + if counted != 1: + return False + db.executable.execute( + text( + "INSERT INTO node_donors (uid, user_uid, requests_ok, bytes_total, " + "first_donated_at, last_donated_at, xp_awarded_today, xp_day, " + "created_at, updated_at) " + "VALUES (:uid, :user, 1, :bytes, :now, :now, 0, '', :now, :now) " + "ON CONFLICT(user_uid) DO UPDATE SET " + "requests_ok = COALESCE(requests_ok, 0) + 1, " + "bytes_total = COALESCE(bytes_total, 0) + :bytes, " + "first_donated_at = CASE WHEN COALESCE(first_donated_at, '') = '' " + "THEN :now ELSE first_donated_at END, " + "last_donated_at = :now, updated_at = :now" + ), + {"uid": _uid(), "user": user_uid, "bytes": up + down, "now": now}, + ) + return True + + +def record_integrity_failure(device_uid: str, user_uid: str) -> None: + now = _now_iso() + with db: + db.executable.execute( + text( + "INSERT OR IGNORE INTO node_usage_daily (uid, device_uid, user_uid, " + "day, requests_ok, requests_failed, bytes_up, bytes_down, " + "cache_hits_served, integrity_failures, created_at, updated_at) " + "VALUES (:uid, :device, :user, :day, 0, 0, 0, 0, 0, 0, :now, :now)" + ), + {"uid": _uid(), "device": device_uid, "user": user_uid, "day": _day(), "now": now}, + ) + db.executable.execute( + text( + "UPDATE node_usage_daily SET " + "integrity_failures = COALESCE(integrity_failures, 0) + 1, " + "updated_at = :now WHERE device_uid = :device AND day = :day" + ), + {"now": now, "device": device_uid, "day": _day()}, + ) + + +def claim_xp(user_uid: str, amount: int, *, moment: datetime | None = None) -> bool: + if amount <= 0: + return False + day = _day(moment) + return ( + _execute( + "UPDATE node_donors SET " + "xp_awarded_today = CASE WHEN COALESCE(xp_day, '') = :day " + "THEN COALESCE(xp_awarded_today, 0) + :amount ELSE :amount END, " + "xp_day = :day WHERE user_uid = :user AND deleted_at IS NULL AND " + "(CASE WHEN COALESCE(xp_day, '') = :day " + "THEN COALESCE(xp_awarded_today, 0) ELSE 0 END) + :amount <= :cap", + {"day": day, "amount": amount, "user": user_uid, "cap": LIMITS.xp_daily_cap}, + ) + == 1 + ) + + +def get_donor(user_uid: str) -> dict: + row = get_table("node_donors").find_one(user_uid=user_uid, deleted_at=None) + return { + "requests_ok": int((row or {}).get("requests_ok") or 0), + "bytes_total": int((row or {}).get("bytes_total") or 0), + "first_donated_at": (row or {}).get("first_donated_at") or "", + "last_donated_at": (row or {}).get("last_donated_at") or "", + } + + +def usage_since(user_uid: str, since_day: str) -> dict: + rows = db.query( + "SELECT COALESCE(SUM(requests_ok), 0) AS requests_ok, " + "COALESCE(SUM(requests_failed), 0) AS requests_failed, " + "COALESCE(SUM(bytes_up), 0) AS bytes_up, " + "COALESCE(SUM(bytes_down), 0) AS bytes_down " + "FROM node_usage_daily WHERE user_uid = :user AND day >= :since " + "AND deleted_at IS NULL", + user=user_uid, + since=since_day, + ) + return dict(next(iter(rows))) + + +def usage_today(device_uid: str) -> dict: + row = get_table("node_usage_daily").find_one( + device_uid=device_uid, day=_day(), deleted_at=None + ) + row = row or {} + return { + key: int(row.get(key) or 0) + for key in ( + "requests_ok", + "requests_failed", + "bytes_up", + "bytes_down", + "cache_hits_served", + "integrity_failures", + ) + } + + +def community_stats() -> dict: + rows = db.query( + "SELECT COALESCE(SUM(requests_ok), 0) AS requests, " + "COALESCE(SUM(bytes_total), 0) AS bytes, COUNT(*) AS donors " + "FROM node_donors WHERE deleted_at IS NULL AND COALESCE(requests_ok, 0) > 0" + ) + row = next(iter(rows)) + return { + "requests": int(row["requests"]), + "bytes": int(row["bytes"]), + "donors": int(row["donors"]), + } + + +def add_event( + kind: str, *, device_uid: str = "", user_uid: str = "", detail: str = "" +) -> None: + get_table("node_events").insert( + { + "uid": _uid(), + "device_uid": device_uid, + "user_uid": user_uid, + "kind": kind[:40], + "detail": (detail or "")[:EVENT_DETAIL_MAX], + "created_at": _now_iso(), + } + ) + + +def raise_flag(kind: str, *, device_uid: str = "", user_uid: str = "", detail: str = "") -> dict: + table = get_table("node_flags") + existing = table.find_one( + kind=kind, device_uid=device_uid, status=FLAG_OPEN, deleted_at=None + ) + if existing: + return existing + uid = _uid() + now = _now_iso() + table.insert( + { + "uid": uid, + "device_uid": device_uid, + "user_uid": user_uid, + "kind": kind[:40], + "detail": (detail or "")[:EVENT_DETAIL_MAX], + "status": FLAG_OPEN, + "resolved_by": "", + "resolved_at": "", + "created_at": now, + "updated_at": now, + "deleted_at": None, + "deleted_by": None, + } + ) + return table.find_one(uid=uid) + + +def list_flags(status: str = FLAG_OPEN) -> list[dict]: + return list( + get_table("node_flags").find( + status=status, deleted_at=None, order_by=["-created_at"] + ) + ) + + +def resolve_flag(flag_uid: str, actor_uid: str) -> bool: + return ( + _execute( + "UPDATE node_flags SET status = 'resolved', resolved_by = :actor, " + "resolved_at = :now, updated_at = :now " + "WHERE uid = :uid AND status = 'open' AND deleted_at IS NULL", + {"actor": actor_uid, "now": _now_iso(), "uid": flag_uid}, + ) + == 1 + ) + + +def prune_usage_daily(*, dry_run: bool = False, moment: datetime | None = None) -> int: + cutoff = _day((moment or datetime.now(timezone.utc)) - timedelta(days=LIMITS.usage_daily_retention_days)) + count = next( + iter( + db.query( + "SELECT COUNT(*) AS c FROM node_usage_daily WHERE day < :cutoff", + cutoff=cutoff, + ) + ) + )["c"] + if not dry_run and count: + _execute("DELETE FROM node_usage_daily WHERE day < :cutoff", {"cutoff": cutoff}) + return count + + +def prune_events(*, dry_run: bool = False, moment: datetime | None = None) -> int: + cutoff = ( + (moment or datetime.now(timezone.utc)) + - timedelta(days=LIMITS.event_retention_days) + ).isoformat() + aged = next( + iter( + db.query( + "SELECT COUNT(*) AS c FROM node_events WHERE created_at < :cutoff", + cutoff=cutoff, + ) + ) + )["c"] + total = get_table("node_events").count() + excess = max(0, total - aged - LIMITS.event_max_rows) + if not dry_run: + if aged: + _execute("DELETE FROM node_events WHERE created_at < :cutoff", {"cutoff": cutoff}) + if excess: + _execute( + "DELETE FROM node_events WHERE id IN " + "(SELECT id FROM node_events ORDER BY created_at, id LIMIT :n)", + {"n": excess}, + ) + return aged + excess + + +def prune_requests(*, dry_run: bool = False, moment: datetime | None = None) -> int: + cutoff = ( + (moment or datetime.now(timezone.utc)) + - timedelta(seconds=LIMITS.request_retention_seconds) + ).isoformat() + count = next( + iter( + db.query( + "SELECT COUNT(*) AS c FROM node_requests WHERE created_at < :cutoff", + cutoff=cutoff, + ) + ) + )["c"] + if not dry_run and count: + _execute("DELETE FROM node_requests WHERE created_at < :cutoff", {"cutoff": cutoff}) + return count diff --git a/devplacepy/database/schema.py b/devplacepy/database/schema.py index 023e70c3..b3c8465b 100644 --- a/devplacepy/database/schema.py +++ b/devplacepy/database/schema.py @@ -1,5 +1,6 @@ # retoor +from devplacepy.services.nodes.spec import SPEC as NODES_SPEC from .core import AQUALITY_NEWS_GRADING_MODEL, AQUALITY_NEWS_GRADING_URL, DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger from .settings import get_setting, set_setting from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns @@ -1850,6 +1851,7 @@ def init_db(): _ensure_moderation_tables() _ensure_skills_table() + _ensure_nodes_tables() for table in db.tables: _uid_index(db, table) @@ -2004,6 +2006,12 @@ def init_db(): "guidelines_version": "1", "ai_third_party_provider": "", "account_deletion_grace_hours": "24", + "nodes_enabled": "0", + "nodes_max_total": str(NODES_SPEC.limits.max_connected_nodes), + "nodes_daily_quota_user": str(NODES_SPEC.limits.default_daily_requests_user), + "nodes_daily_quota_guest": str(NODES_SPEC.limits.default_daily_requests_guest), + "nodes_cache_ttl_seconds": str(NODES_SPEC.limits.cache_ttl_seconds), + "nodes_social_via_nodes": "0", } for key, value in operational_defaults.items(): existing = db["site_settings"].find_one(key=key) @@ -2191,6 +2199,159 @@ def _ensure_skills_table() -> None: ) +NODES_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = { + "node_devices": ( + ("uid", ""), + ("user_uid", ""), + ("label", ""), + ("kind", ""), + ("secret_hash", ""), + ("secret_prefix", ""), + ("platform", ""), + ("client_version", ""), + ("protocol_version", 0), + ("status", ""), + ("suspended_until", ""), + ("suspend_reason", ""), + ("country", ""), + ("asn", ""), + ("asn_class", ""), + ("ip_hash", ""), + ("last_seen_at", ""), + ("created_at", ""), + ("updated_at", ""), + ("deleted_at", ""), + ("deleted_by", ""), + ), + "node_settings": ( + ("uid", ""), + ("user_uid", ""), + ("enabled", 0), + ("bandwidth_mb_day", 0), + ("max_concurrent", 0), + ("active_from", ""), + ("active_to", ""), + ("timezone", ""), + ("allowed_platforms", ""), + ("pause_on_battery", 0), + ("public_display", 0), + ("created_at", ""), + ("updated_at", ""), + ("deleted_at", ""), + ("deleted_by", ""), + ), + "node_usage_daily": ( + ("uid", ""), + ("device_uid", ""), + ("user_uid", ""), + ("day", ""), + ("requests_ok", 0), + ("requests_failed", 0), + ("bytes_up", 0), + ("bytes_down", 0), + ("cache_hits_served", 0), + ("integrity_failures", 0), + ("created_at", ""), + ("updated_at", ""), + ("deleted_at", ""), + ("deleted_by", ""), + ), + "node_donors": ( + ("uid", ""), + ("user_uid", ""), + ("requests_ok", 0), + ("bytes_total", 0), + ("first_donated_at", ""), + ("last_donated_at", ""), + ("xp_awarded_today", 0), + ("xp_day", ""), + ("created_at", ""), + ("updated_at", ""), + ("deleted_at", ""), + ("deleted_by", ""), + ), + "node_requests": ( + ("uid", ""), + ("caller_kind", ""), + ("caller_id", ""), + ("url", ""), + ("method", ""), + ("adapter", ""), + ("mode", ""), + ("status", ""), + ("device_uid", ""), + ("result_status", 0), + ("result_bytes", 0), + ("result_body", ""), + ("error", ""), + ("created_at", ""), + ("assigned_at", ""), + ("finished_at", ""), + ), + "node_events": ( + ("uid", ""), + ("device_uid", ""), + ("user_uid", ""), + ("kind", ""), + ("detail", ""), + ("created_at", ""), + ), + "node_flags": ( + ("uid", ""), + ("device_uid", ""), + ("user_uid", ""), + ("kind", ""), + ("detail", ""), + ("status", ""), + ("resolved_by", ""), + ("resolved_at", ""), + ("created_at", ""), + ("updated_at", ""), + ("deleted_at", ""), + ("deleted_by", ""), + ), +} + + +def _ensure_nodes_tables() -> None: + for table_name, columns in NODES_COLUMNS.items(): + table = get_table(table_name) + for column, example in columns: + if not table.has_column(column): + table.create_column_by_example(column, example) + _index(db, "node_devices", "idx_node_devices_user", ["user_uid", "deleted_at"]) + _index(db, "node_devices", "idx_node_devices_status", ["status", "last_seen_at"]) + _index(db, "node_devices", "idx_node_devices_prefix", ["secret_prefix"]) + _index(db, "node_settings", "idx_node_settings_user", ["user_uid"], unique=True) + _index(db, "node_donors", "idx_node_donors_user", ["user_uid"], unique=True) + _index( + db, + "node_usage_daily", + "idx_node_usage_device_day", + ["device_uid", "day"], + unique=True, + ) + _index(db, "node_usage_daily", "idx_node_usage_day", ["day"]) + _index(db, "node_usage_daily", "idx_node_usage_user", ["user_uid", "day"]) + _index( + db, + "node_requests", + "idx_node_requests_open", + ["status", "created_at"], + where="status IN ('queued','assigned')", + ) + _index(db, "node_requests", "idx_node_requests_created", ["created_at"]) + _index(db, "node_events", "idx_node_events_created", ["created_at"]) + _index(db, "node_events", "idx_node_events_device", ["device_uid", "created_at"]) + _index( + db, + "node_flags", + "idx_node_flags_open", + ["status", "created_at"], + where="deleted_at IS NULL", + ) + + def _ensure_moderation_tables() -> None: for table_name, columns in MODERATION_COLUMNS.items(): table = get_table(table_name) diff --git a/devplacepy/database/soft_delete.py b/devplacepy/database/soft_delete.py index 8efc0788..268777f4 100644 --- a/devplacepy/database/soft_delete.py +++ b/devplacepy/database/soft_delete.py @@ -18,6 +18,11 @@ SOFT_DELETE_TABLES = [ "bookmarks", "notes", "skills", + "node_devices", + "node_settings", + "node_usage_daily", + "node_donors", + "node_flags", "follows", "poll_votes", "polls", diff --git a/devplacepy/services/acceptance/CLAUDE.md b/devplacepy/services/acceptance/CLAUDE.md index 31868922..1fd155a1 100644 --- a/devplacepy/services/acceptance/CLAUDE.md +++ b/devplacepy/services/acceptance/CLAUDE.md @@ -29,7 +29,7 @@ It is off by default and it is never appropriate on a real production host. `AGREEMENTS` annotates `database.CONSENT_KINDS` with two facts: which `site_settings` key holds the policy version, and which `users` column the application's own gate reads. `terms` is the only agreement with a gate column, because `needs_acceptance` reads `users.terms_version` and not the ledger; `privacy` is versioned but ledger-only, matching `VERSION_KEYS` in `routers/profile/consent.py`. -A unit test asserts `{a.kind for a in AGREEMENTS} == set(CONSENT_KINDS)`. **A sixth consent fails the suite until it is classified here**, and it then appears in the admin form with no edit to the service, because the per-agreement config fields are built from the registry rather than written out by hand. +A unit test asserts that `AGREEMENTS` and `EXCLUDED_CONSENTS` are disjoint and together equal `set(CONSENT_KINDS)`. **A new consent fails the suite until it is classified here**, and it then appears in the admin form with no edit to the service, because the per-agreement config fields are built from the registry rather than written out by hand. ## Rules that must not regress @@ -49,6 +49,7 @@ A unit test asserts `{a.kind for a in AGREEMENTS} == set(CONSENT_KINDS)`. **A si | `users.age_band` | A declaration of fact, not an agreement. Fabricating a declared age would silently unlock `restricted` content for an account that declared 13-15. Sign the test account up with an adult date of birth instead. | | `users.mature_opt_in` | A preference gated by the age band, with no ledger row and therefore no decline register. The site setting `moderation_mature_default_hidden` already turns the interstitial off instance-wide. | | Guest consents | Nothing writes a consent row with `owner_kind = "guest"`, and the gateway resolves a guest to owner kind `anonymous`, which `consent_denied` exempts by design. | +| `bandwidth_sharing` consent | Lets other people use the member's own internet connection. Granting it on someone's behalf would be real harm, not a test convenience. It is listed in `EXCLUDED_CONSENTS` with its reason and the service never touches it. | | Preferences (`interactions_enabled`, notifications, customization) | None of them blocks anything. | ## The terms asymmetry, and the one operator step @@ -59,7 +60,7 @@ Withdrawing the `terms` consent does not clear `users.terms_version`, so a teste ## Rules for extending this -- A new consent kind: classify it in `AGREEMENTS` (the test forces this), and nothing else. +- A new consent kind: classify it in `AGREEMENTS`, or in `EXCLUDED_CONSENTS` with a reason when it is not a policy agreement (the test forces one of the two), and nothing else. - Never add a route, a schema, a template, a Devii tool or a `docs_api` entry. The surface is the generic services admin, exactly as for `NotificationRelayService` and `AuditService`. - Never add a second decline register, a second ordering, or an environment check. - Never write a persisted test that enables the service and asserts convergence: the suite is serial against one seeded database, and granting `terms` to every account would poison the moderation tests that assert the gate refuses. Convergence is covered by unit tests calling `converge_user` directly. diff --git a/devplacepy/services/acceptance/agreements.py b/devplacepy/services/acceptance/agreements.py index 5bb909b9..3c36b58b 100644 --- a/devplacepy/services/acceptance/agreements.py +++ b/devplacepy/services/acceptance/agreements.py @@ -21,6 +21,13 @@ AGREEMENTS: tuple[Agreement, ...] = ( Agreement("container_credentials", "", "", ""), ) +EXCLUDED_CONSENTS: dict[str, str] = { + "bandwidth_sharing": ( + "grants the use of the member's own internet connection by other people, " + "so only the member can give it" + ), +} + def setting_key(kind: str) -> str: return f"acceptance_grant_{kind}" diff --git a/devplacepy/services/moderation/deletion.py b/devplacepy/services/moderation/deletion.py index e233d305..806d2a98 100644 --- a/devplacepy/services/moderation/deletion.py +++ b/devplacepy/services/moderation/deletion.py @@ -60,6 +60,10 @@ OWNED_TABLES: tuple[tuple[str, str], ...] = ( ("deepsearch_sessions", "owner_id"), ("isslop_analyses", "owner_id"), ("skills", "user_uid"), + ("node_devices", "user_uid"), + ("node_settings", "user_uid"), + ("node_donors", "user_uid"), + ("node_usage_daily", "user_uid"), ) CHILD_TABLES: tuple[tuple[str, str, str, str], ...] = ( diff --git a/devplacepy/services/nodes/CLAUDE.md b/devplacepy/services/nodes/CLAUDE.md index 0b5f58e1..ce704dd8 100644 --- a/devplacepy/services/nodes/CLAUDE.md +++ b/devplacepy/services/nodes/CLAUDE.md @@ -2,7 +2,7 @@ This file documents the community node network (`devplacepy/services/nodes/`, `d ## What exists today -Only Phase 1 (the specification) is built. Everything else is planned and listed under "Gaps" next to the phase that closes it. Nothing here is enabled for any account. +Phase 1 (the specification) and Phase 2 (the data layer and usage ledger) are built. Everything else is planned and listed under "Gaps" next to the phase that closes it. Nothing here is enabled for any account. | Piece | File | State | |-------|------|-------| @@ -10,6 +10,9 @@ Only Phase 1 (the specification) is built. Everything else is planned and listed | `GET /nodes/spec.json` | `routers/nodes/spec.py` | built, tested, public | | Generated protocol docs page | `templates/docs/nodes-protocol.html` | built, reads the `nodes_spec` Jinja global | | API docs group | `docs_api/groups/nodes.py` | built, one endpoint | +| Tables and helpers | `database/nodes.py`, `NODES_COLUMNS` in `database/schema.py` | built, tested, no route uses them yet | +| Usage ledger | `ledger.py` (`UsageLedger`, `DonationSummary`) | built, tested | +| `bandwidth_sharing` consent kind | `CONSENT_KINDS` in `database/moderation.py` | registered, never granted by default, enforced nowhere yet | ## The specification is the contract @@ -35,13 +38,24 @@ The docs page `nodes-protocol.html` renders its tables from the `nodes_spec` Jin | D5 | donors must be adults (`age_band`) | | D6 | a datacenter address is not a donor: residential and mobile only, admin override | +## Data layer (Phase 2) + +Seven tables, all ensured with their full column set in `init_db` (`_ensure_nodes_tables`): `node_devices`, `node_settings`, `node_usage_daily`, `node_donors`, `node_flags` (soft-deletable, in `SOFT_DELETE_TABLES` and `UNREPORTABLE_TABLES` with a reason), and `node_requests`, `node_events` (ephemeral, hard garbage collection only). Account deletion cascades through `OWNED_TABLES` for devices, settings, donors and daily usage. + +- **Atomic by one statement.** The device cap is an `INSERT ... SELECT ... WHERE (SELECT COUNT(*)...) < max`, the daily bandwidth cap is an `UPDATE ... WHERE COALESCE(bytes_up,0)+COALESCE(bytes_down,0)+:n <= :cap`, and the daily XP cap is a conditional `UPDATE` that rolls the day over inside the same statement. Verified with real separate processes against fresh, unseeded rows (40 racers on a 10-job cap gave exactly 10, 24 device creations gave exactly 5, 30 XP claims gave exactly the cap). +- **`record_job` is one transaction**: the daily bucket (`INSERT OR IGNORE` then the conditional `UPDATE`) and the lifetime `node_donors` upsert commit together or not at all. A job refused by the cap counts nothing. A failed job counts `requests_failed` and never moves bytes or lifetime totals. +- **Lifetime totals are maintained at write time**, so pruning the daily buckets (`usage_daily_retention_days`) can never lower them. Prune functions take `dry_run` and use only the constants in `LimitSpec`; there is no argument that lowers a retention floor. +- **Secrets**: `dpn_` plus 32 random bytes, shown once at create or rotate, stored as sha256 plus a 12 character lookup prefix, compared with `hmac.compare_digest`. IP addresses are stored only as a keyed hash (`hash_ip`). +- **Settings are clamped on write and on read** (`normalise_settings`): bandwidth to `1..max_donor_mb_per_day`, concurrency to `1..max_concurrent_per_node`, hours must be `HH:MM`, platforms are intersected with the spec's adapters. A member with no row is opted out. +- Site settings seeded in `init_db`: `nodes_enabled` (`0`), `nodes_max_total`, `nodes_daily_quota_user`, `nodes_daily_quota_guest`, `nodes_cache_ttl_seconds`, `nodes_social_via_nodes` (`0`). Like every seed, consumers pass the default to `get_setting` because the seed does not run on an empty database. + ## Two transports, two trust levels `tunnel` (native client) relays raw TCP so TLS runs end to end between DevPlace and the destination: the donor cannot read or forge content. `fetch` (browser extension, which has no raw sockets) lets the donor read and forge the response, so every `fetch` result is verified and labelled. A plain web page or a service worker cannot do either (CORS, no sockets) and is out of scope for good. ## Gaps (what the next phases close) -- Phase 2: tables, indexes, the `bandwidth_sharing` consent kind, `OWNED_TABLES` and `UNREPORTABLE_TABLES` entries, the usage ledger. +- Phase 2 leftovers: the `bandwidth_sharing` consent can currently be granted through `POST /profile/{username}/consent` and by Devii's `set_consent` (Devii calls that route as the user). Nothing reads it yet, so it is inert, but Phase 3 must make Devii unable to grant it (Devii may read and withdraw) at the same time as `ConsentGate` lands. No XP or badge is awarded yet (`UsageLedger.claim_xp` only reserves the daily budget); the privacy page and app-store declarations are updated with the first phase that lets a device connect, because until then no personal data is collected. - Phase 3: registry, session, the consent choke point, the kill switch, device and ticket routes, the lock-owner WebSocket, the nginx upgrade location. - Phase 4: broker, scheduler, dispatcher, destination policy, circuit breakers, result cache. - Phase 5 onward: the reference client, adapters, the Go client, the extension, the profile UI, Devii tools, the CLI, the privacy and donor-terms pages, the legal gate. diff --git a/devplacepy/services/nodes/ledger.py b/devplacepy/services/nodes/ledger.py new file mode 100644 index 00000000..475f444d --- /dev/null +++ b/devplacepy/services/nodes/ledger.py @@ -0,0 +1,77 @@ +# retoor + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone + +from devplacepy.database import nodes as store + + +@dataclass(frozen=True) +class DonationSummary: + lifetime_requests: int + lifetime_bytes: int + first_donated_at: str + last_donated_at: str + month_requests: int + month_bytes_up: int + month_bytes_down: int + today_requests: int + today_bytes: int + + def describe(self) -> dict: + return dict(self.__dict__) + + +class UsageLedger: + def record_success( + self, + device_uid: str, + user_uid: str, + *, + bytes_up: int, + bytes_down: int, + cache_hit: bool = False, + ) -> bool: + return store.record_job( + device_uid, + user_uid, + ok=True, + bytes_up=bytes_up, + bytes_down=bytes_down, + cache_hit=cache_hit, + ) + + def record_failure(self, device_uid: str, user_uid: str) -> None: + store.record_job(device_uid, user_uid, ok=False) + + def record_integrity_failure(self, device_uid: str, user_uid: str) -> None: + store.record_integrity_failure(device_uid, user_uid) + + def claim_xp(self, user_uid: str, amount: int) -> bool: + return store.claim_xp(user_uid, amount) + + def summary(self, user_uid: str) -> DonationSummary: + now = datetime.now(timezone.utc) + donor = store.get_donor(user_uid) + month = store.usage_since(user_uid, now.replace(day=1).date().isoformat()) + today = store.usage_since(user_uid, now.date().isoformat()) + return DonationSummary( + lifetime_requests=donor["requests_ok"], + lifetime_bytes=donor["bytes_total"], + first_donated_at=donor["first_donated_at"], + last_donated_at=donor["last_donated_at"], + month_requests=month["requests_ok"], + month_bytes_up=month["bytes_up"], + month_bytes_down=month["bytes_down"], + today_requests=today["requests_ok"], + today_bytes=today["bytes_up"] + today["bytes_down"], + ) + + def prune(self, *, dry_run: bool = False) -> dict: + return { + "usage_daily": store.prune_usage_daily(dry_run=dry_run), + "events": store.prune_events(dry_run=dry_run), + "requests": store.prune_requests(dry_run=dry_run), + } diff --git a/pyproject.toml b/pyproject.toml index 00cfff39..cab640e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "devplacepy" -version = "1.0.18" +version = "1.0.19" description = "DevPlace - The Developer Social Network" requires-python = ">=3.12" dependencies = [ diff --git a/tests/unit/database/nodes.py b/tests/unit/database/nodes.py new file mode 100644 index 00000000..115dc845 --- /dev/null +++ b/tests/unit/database/nodes.py @@ -0,0 +1,227 @@ +# retoor + +from datetime import datetime, timedelta, timezone + +import pytest + +from devplacepy.database import ( + CONSENT_KINDS, + SOFT_DELETE_TABLES, + UNREPORTABLE_TABLES, + consent_granted, + get_setting, + get_table, +) +from devplacepy.database import nodes as store +from devplacepy.services.moderation.deletion import OWNED_TABLES +from devplacepy.services.nodes.ledger import UsageLedger +from devplacepy.services.nodes.spec import SPEC +from devplacepy.utils import generate_uid + +LIMITS = SPEC.limits +SOFT_TABLES = ("node_devices", "node_settings", "node_usage_daily", "node_donors", "node_flags") + + +@pytest.fixture +def owner(local_db): + return generate_uid() + + +@pytest.fixture +def device(owner): + row, _ = store.create_device(owner, "laptop", "native") + return row + + +def test_node_tables_are_registered_everywhere_they_must_be(): + for table in SOFT_TABLES: + assert table in SOFT_DELETE_TABLES + assert table in UNREPORTABLE_TABLES + for table in ("node_devices", "node_settings", "node_donors", "node_usage_daily"): + assert (table, "user_uid") in OWNED_TABLES + + +def test_bandwidth_sharing_consent_exists_and_is_never_seeded(owner): + assert "bandwidth_sharing" in CONSENT_KINDS + assert not consent_granted("user", owner, "bandwidth_sharing") + assert get_setting("nodes_enabled", "0") == "0" + + +def test_secret_is_shown_once_and_stored_hashed(owner): + row, secret = store.create_device(owner, " desk ", "extension") + assert secret.startswith(store.SECRET_PREFIX) + assert row["label"] == "desk" + assert secret not in str(row) + assert store.find_device_by_secret(secret)["uid"] == row["uid"] + assert store.find_device_by_secret(secret + "x") is None + assert store.find_device_by_secret("") is None + + +def test_rotate_invalidates_the_old_secret_and_checks_the_owner(owner): + row, old = store.create_device(owner, "a", "native") + assert store.rotate_device_secret(row["uid"], generate_uid()) is None + new = store.rotate_device_secret(row["uid"], owner) + assert new and new != old + assert store.find_device_by_secret(old) is None + assert store.find_device_by_secret(new)["uid"] == row["uid"] + + +def test_device_cap_per_user_is_enforced(owner): + for _ in range(LIMITS.max_devices_per_user): + store.create_device(owner, "x", "native") + with pytest.raises(store.NodeLimitError): + store.create_device(owner, "x", "native") + assert store.count_devices(owner) == LIMITS.max_devices_per_user + + +def test_unknown_device_kind_is_rejected(owner): + with pytest.raises(store.NodeLimitError): + store.create_device(owner, "x", "browser") + + +def test_revoke_is_a_soft_delete_and_frees_the_slot(owner): + row, secret = store.create_device(owner, "a", "native") + assert store.revoke_device(row["uid"], owner) == 1 + assert store.get_device(row["uid"]) is None + assert store.find_device_by_secret(secret) is None + assert store.count_devices(owner) == 0 + assert get_table("node_devices").find_one(uid=row["uid"])["deleted_by"] == owner + + +def test_suspension_expires_on_its_own_and_revoked_never_suspends(owner): + row, _ = store.create_device(owner, "a", "native") + assert store.suspend_device(row["uid"], "integrity", 30) + suspended = store.get_device(row["uid"]) + assert store.effective_status(suspended) == "suspended" + later = datetime.now(timezone.utc) + timedelta(minutes=31) + assert store.effective_status(suspended, later) == "active" + assert store.unsuspend_device(row["uid"]) + assert store.get_device(row["uid"])["status"] == "active" + store.revoke_device(row["uid"], owner) + assert not store.suspend_device(row["uid"], "x", 5) + + +def test_settings_default_to_opted_out(owner): + settings = store.get_node_settings(owner) + assert settings["enabled"] is False + assert settings["bandwidth_mb_day"] == LIMITS.default_donor_mb_per_day + + +@pytest.mark.parametrize("value", [-5, 0, 1, 500, 10**9, "x", None, 3.7]) +def test_settings_are_clamped_to_the_declared_ceilings(owner, value): + saved = store.save_node_settings( + owner, {"bandwidth_mb_day": value, "max_concurrent": value} + ) + assert 1 <= saved["bandwidth_mb_day"] <= LIMITS.max_donor_mb_per_day + assert 1 <= saved["max_concurrent"] <= LIMITS.max_concurrent_per_node + assert store.get_node_settings(owner) == saved + + +def test_settings_reject_bad_hours_and_unknown_platforms(owner): + saved = store.save_node_settings( + owner, + {"enabled": True, "active_from": "25:00", "active_to": "07:30", + "allowed_platforms": ["reddit", "evil"]}, + ) + assert saved["active_from"] == "" + assert saved["active_to"] == "07:30" + assert saved["allowed_platforms"] == ["reddit"] + again = store.save_node_settings(owner, {"enabled": False}) + assert again["active_to"] == "07:30" + assert again["enabled"] is False + + +def test_daily_cap_blocks_the_job_that_would_cross_it(owner, device): + cap = 10_000 + results = [ + store.record_job(device["uid"], owner, ok=True, bytes_up=100, bytes_down=900, cap_bytes=cap) + for _ in range(14) + ] + assert results.count(True) == 10 + today = store.usage_today(device["uid"]) + assert today["requests_ok"] == 10 + assert today["bytes_up"] + today["bytes_down"] == cap + donor = store.get_donor(owner) + assert donor["requests_ok"] == 10 and donor["bytes_total"] == cap + + +def test_failed_jobs_count_but_move_no_bytes_or_lifetime_totals(owner, device): + store.record_job(device["uid"], owner, ok=False, bytes_down=999) + today = store.usage_today(device["uid"]) + assert today["requests_failed"] == 1 + assert today["bytes_down"] == 0 + assert store.get_donor(owner)["requests_ok"] == 0 + + +def test_lifetime_totals_survive_pruning_the_daily_buckets(owner, device): + store.record_job(device["uid"], owner, ok=True, bytes_down=10, cap_bytes=1000) + far = datetime.now(timezone.utc) + timedelta(days=LIMITS.usage_daily_retention_days + 2) + assert store.prune_usage_daily(dry_run=True, moment=far) >= 1 + assert store.usage_today(device["uid"])["requests_ok"] == 1 + assert store.prune_usage_daily(moment=far) >= 1 + assert store.usage_today(device["uid"])["requests_ok"] == 0 + assert store.get_donor(owner)["requests_ok"] == 1 + + +def test_recent_buckets_are_not_pruned(owner, device): + store.record_job(device["uid"], owner, ok=True, cap_bytes=1000) + assert store.prune_usage_daily() == 0 + + +def test_xp_is_capped_per_day_and_resets_the_next_day(owner, device): + store.record_job(device["uid"], owner, ok=True, cap_bytes=1000) + granted = sum(store.claim_xp(owner, 7) for _ in range(30)) + assert granted == LIMITS.xp_daily_cap // 7 + tomorrow = datetime.now(timezone.utc) + timedelta(days=1) + assert store.claim_xp(owner, 7, moment=tomorrow) + assert not store.claim_xp(owner, 0) + assert not store.claim_xp(generate_uid(), 7) + + +def test_integrity_failures_are_counted(owner, device): + store.record_integrity_failure(device["uid"], owner) + store.record_integrity_failure(device["uid"], owner) + assert store.usage_today(device["uid"])["integrity_failures"] == 2 + + +def test_ledger_summary_reports_lifetime_month_and_today(owner, device): + ledger = UsageLedger() + assert ledger.record_success(device["uid"], owner, bytes_up=10, bytes_down=90) + ledger.record_failure(device["uid"], owner) + summary = ledger.summary(owner) + assert summary.lifetime_requests == 1 + assert summary.lifetime_bytes == 100 + assert summary.month_requests == 1 + assert summary.today_bytes == 100 + assert summary.first_donated_at + + +def test_community_stats_are_aggregate_only(owner, device): + store.record_job(device["uid"], owner, ok=True, bytes_down=50, cap_bytes=1000) + stats = store.community_stats() + assert set(stats) == {"requests", "bytes", "donors"} + assert stats["requests"] >= 1 and stats["donors"] >= 1 + + +def test_flags_deduplicate_while_open_and_resolve_once(owner): + flag = store.raise_flag("integrity", device_uid="d1", user_uid=owner, detail="x") + assert store.raise_flag("integrity", device_uid="d1")["uid"] == flag["uid"] + assert store.resolve_flag(flag["uid"], "admin") + assert not store.resolve_flag(flag["uid"], "admin") + assert flag["uid"] not in [item["uid"] for item in store.list_flags()] + + +def test_event_log_is_bounded_by_age(owner): + store.add_event("connect", user_uid=owner, detail="x" * 1000) + row = get_table("node_events").find_one(user_uid=owner) + assert len(row["detail"]) == store.EVENT_DETAIL_MAX + far = datetime.now(timezone.utc) + timedelta(days=LIMITS.event_retention_days + 1) + assert store.prune_events(dry_run=True, moment=far) >= 1 + assert store.prune_events(moment=far) >= 1 + + +def test_ip_is_only_ever_stored_hashed(): + digest = store.hash_ip("203.0.113.9") + assert "203" not in digest and len(digest) == 32 + assert store.hash_ip("203.0.113.9") == digest + assert store.hash_ip("") == "" diff --git a/tests/unit/services/acceptance/agreements.py b/tests/unit/services/acceptance/agreements.py index 21c6dae8..c1f433f1 100644 --- a/tests/unit/services/acceptance/agreements.py +++ b/tests/unit/services/acceptance/agreements.py @@ -3,6 +3,7 @@ from devplacepy.database import CONSENT_KINDS from devplacepy.services.acceptance.agreements import ( AGREEMENTS, + EXCLUDED_CONSENTS, agreement_for, label_for, setting_key, @@ -11,7 +12,10 @@ from devplacepy.services.acceptance.service import AcceptanceService def test_every_consent_kind_is_classified_as_an_agreement(): - assert {agreement.kind for agreement in AGREEMENTS} == set(CONSENT_KINDS) + classified = {agreement.kind for agreement in AGREEMENTS} + assert classified.isdisjoint(EXCLUDED_CONSENTS) + assert classified | set(EXCLUDED_CONSENTS) == set(CONSENT_KINDS) + assert all(EXCLUDED_CONSENTS.values()) def test_setting_keys_are_unique_and_namespaced():