Add community node data layer: tables, usage ledger, bandwidth_sharing consent
Phase 2 of plan.md. Seven node_* tables with atomic device, bandwidth and XP caps, a transactional job ledger whose lifetime totals survive pruning, clamped donor settings, and the bandwidth_sharing consent kind. The acceptance service now has an explicit exclusion list so it can never grant that consent. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -49,6 +49,11 @@ UNREPORTABLE_TABLES: dict[str, str] = {
|
||||
"bookmarks": "private to the owner",
|
||||
"notes": "private to the owner",
|
||||
"skills": "private to the owner, never shown to others",
|
||||
"node_devices": "private to the owner, the label is never shown to others",
|
||||
"node_settings": "private to the owner",
|
||||
"node_usage_daily": "derived donation counters, carry no authored content",
|
||||
"node_donors": "derived donation totals, carry no authored content",
|
||||
"node_flags": "operational flags for administrators, carry no authored content",
|
||||
"follows": "relationship rows, carry no authored content",
|
||||
"poll_votes": "private ballots",
|
||||
"opinion_war_fighters": "membership and damage counters, carry no authored content",
|
||||
@@ -133,6 +138,10 @@ CONSENT_KINDS: dict[str, str] = {
|
||||
"Sharing your DevPlace credentials with software another member runs "
|
||||
"in a container"
|
||||
),
|
||||
"bandwidth_sharing": (
|
||||
"Sharing part of your internet connection with the DevPlace community "
|
||||
"so search can reach sites that block datacenters"
|
||||
),
|
||||
}
|
||||
CONSENT_STATES: tuple[str, ...] = ("granted", "withdrawn")
|
||||
|
||||
|
||||
@@ -0,0 +1,639 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from devplacepy.config import SECRET_KEY
|
||||
from devplacepy.services.nodes.spec import DEVICE_KINDS, SPEC
|
||||
|
||||
from .core import _now_iso, db, get_table
|
||||
from .soft_delete import soft_delete
|
||||
|
||||
LIMITS = SPEC.limits
|
||||
|
||||
SECRET_PREFIX = "dpn_"
|
||||
SECRET_PREFIX_LENGTH = 12
|
||||
SECRET_BYTES = 32
|
||||
LABEL_MAX = 60
|
||||
EVENT_DETAIL_MAX = 500
|
||||
MEGABYTE = 1024 * 1024
|
||||
HOURS_PATTERN = re.compile(r"^([01]\d|2[0-3]):[0-5]\d$")
|
||||
PLATFORMS = tuple(sorted({item.adapter for item in SPEC.destinations}))
|
||||
FLAG_OPEN = "open"
|
||||
FLAG_RESOLVED = "resolved"
|
||||
STATUS_ACTIVE = "active"
|
||||
STATUS_SUSPENDED = "suspended"
|
||||
STATUS_REVOKED = "revoked"
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
"enabled": False,
|
||||
"bandwidth_mb_day": LIMITS.default_donor_mb_per_day,
|
||||
"max_concurrent": LIMITS.max_concurrent_per_node,
|
||||
"active_from": "",
|
||||
"active_to": "",
|
||||
"timezone": "UTC",
|
||||
"allowed_platforms": list(PLATFORMS),
|
||||
"pause_on_battery": True,
|
||||
"public_display": False,
|
||||
}
|
||||
|
||||
|
||||
class NodeLimitError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _uid() -> str:
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
return generate_uid()
|
||||
|
||||
|
||||
def _day(moment: datetime | None = None) -> str:
|
||||
return (moment or datetime.now(timezone.utc)).date().isoformat()
|
||||
|
||||
|
||||
def hash_ip(address: str) -> str:
|
||||
if not address:
|
||||
return ""
|
||||
return hmac.new(
|
||||
SECRET_KEY.encode(), address.encode(), hashlib.sha256
|
||||
).hexdigest()[:32]
|
||||
|
||||
|
||||
def _hash_secret(secret: str) -> str:
|
||||
return hashlib.sha256(secret.encode()).hexdigest()
|
||||
|
||||
|
||||
def new_secret() -> str:
|
||||
return SECRET_PREFIX + secrets.token_urlsafe(SECRET_BYTES)
|
||||
|
||||
|
||||
def _execute(sql: str, params: dict) -> int:
|
||||
with db:
|
||||
return db.executable.execute(text(sql), params).rowcount
|
||||
|
||||
|
||||
def create_device(
|
||||
user_uid: str,
|
||||
label: str,
|
||||
kind: str,
|
||||
*,
|
||||
platform: str = "",
|
||||
client_version: str = "",
|
||||
) -> tuple[dict, str]:
|
||||
if kind not in DEVICE_KINDS:
|
||||
raise NodeLimitError("Unknown device kind")
|
||||
secret = new_secret()
|
||||
uid = _uid()
|
||||
now = _now_iso()
|
||||
created = _execute(
|
||||
"INSERT INTO node_devices (uid, user_uid, label, kind, secret_hash, "
|
||||
"secret_prefix, platform, client_version, protocol_version, status, "
|
||||
"suspended_until, suspend_reason, country, asn, asn_class, ip_hash, "
|
||||
"last_seen_at, created_at, updated_at) "
|
||||
"SELECT :uid, :user_uid, :label, :kind, :hash, :prefix, :platform, "
|
||||
":version, 0, 'active', '', '', '', '', 'unknown', '', '', :now, :now "
|
||||
"WHERE (SELECT COUNT(*) FROM node_devices "
|
||||
"WHERE user_uid = :user_uid AND deleted_at IS NULL) < :max_devices",
|
||||
{
|
||||
"uid": uid,
|
||||
"user_uid": user_uid,
|
||||
"label": (label or "").strip()[:LABEL_MAX],
|
||||
"kind": kind,
|
||||
"hash": _hash_secret(secret),
|
||||
"prefix": secret[:SECRET_PREFIX_LENGTH],
|
||||
"platform": (platform or "")[:40],
|
||||
"version": (client_version or "")[:40],
|
||||
"now": now,
|
||||
"max_devices": LIMITS.max_devices_per_user,
|
||||
},
|
||||
)
|
||||
if created != 1:
|
||||
raise NodeLimitError(
|
||||
f"A member can register at most {LIMITS.max_devices_per_user} devices"
|
||||
)
|
||||
return get_device(uid), secret
|
||||
|
||||
|
||||
def get_device(device_uid: str) -> dict | None:
|
||||
return get_table("node_devices").find_one(uid=device_uid, deleted_at=None)
|
||||
|
||||
|
||||
def list_devices(user_uid: str) -> list[dict]:
|
||||
return list(
|
||||
get_table("node_devices").find(
|
||||
user_uid=user_uid, deleted_at=None, order_by=["created_at"]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def count_devices(user_uid: str) -> int:
|
||||
return get_table("node_devices").count(user_uid=user_uid, deleted_at=None)
|
||||
|
||||
|
||||
def find_device_by_secret(secret: str) -> dict | None:
|
||||
if not secret or not secret.startswith(SECRET_PREFIX):
|
||||
return None
|
||||
digest = _hash_secret(secret)
|
||||
for row in get_table("node_devices").find(
|
||||
secret_prefix=secret[:SECRET_PREFIX_LENGTH], deleted_at=None
|
||||
):
|
||||
if hmac.compare_digest(row.get("secret_hash") or "", digest):
|
||||
return row
|
||||
return None
|
||||
|
||||
|
||||
def rotate_device_secret(device_uid: str, user_uid: str) -> str | None:
|
||||
secret = new_secret()
|
||||
changed = _execute(
|
||||
"UPDATE node_devices SET secret_hash = :hash, secret_prefix = :prefix, "
|
||||
"updated_at = :now WHERE uid = :uid AND user_uid = :user_uid "
|
||||
"AND deleted_at IS NULL",
|
||||
{
|
||||
"hash": _hash_secret(secret),
|
||||
"prefix": secret[:SECRET_PREFIX_LENGTH],
|
||||
"now": _now_iso(),
|
||||
"uid": device_uid,
|
||||
"user_uid": user_uid,
|
||||
},
|
||||
)
|
||||
return secret if changed == 1 else None
|
||||
|
||||
|
||||
def revoke_device(device_uid: str, actor_uid: str) -> int:
|
||||
get_table("node_devices").update(
|
||||
{"uid": device_uid, "status": STATUS_REVOKED, "updated_at": _now_iso()},
|
||||
["uid"],
|
||||
)
|
||||
return soft_delete("node_devices", actor_uid, uid=device_uid)
|
||||
|
||||
|
||||
def suspend_device(device_uid: str, reason: str, minutes: int) -> bool:
|
||||
until = (datetime.now(timezone.utc) + timedelta(minutes=minutes)).isoformat()
|
||||
return (
|
||||
_execute(
|
||||
"UPDATE node_devices SET status = 'suspended', suspended_until = :until, "
|
||||
"suspend_reason = :reason, updated_at = :now "
|
||||
"WHERE uid = :uid AND deleted_at IS NULL AND status != 'revoked'",
|
||||
{
|
||||
"until": until,
|
||||
"reason": (reason or "")[:EVENT_DETAIL_MAX],
|
||||
"now": _now_iso(),
|
||||
"uid": device_uid,
|
||||
},
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def unsuspend_device(device_uid: str) -> bool:
|
||||
return (
|
||||
_execute(
|
||||
"UPDATE node_devices SET status = 'active', suspended_until = '', "
|
||||
"suspend_reason = '', updated_at = :now "
|
||||
"WHERE uid = :uid AND deleted_at IS NULL AND status = 'suspended'",
|
||||
{"now": _now_iso(), "uid": device_uid},
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def effective_status(device: dict, moment: datetime | None = None) -> str:
|
||||
status = device.get("status") or STATUS_ACTIVE
|
||||
if status != STATUS_SUSPENDED:
|
||||
return status
|
||||
until = (device.get("suspended_until") or "").strip()
|
||||
try:
|
||||
expiry = datetime.fromisoformat(until)
|
||||
except ValueError:
|
||||
return STATUS_SUSPENDED
|
||||
if expiry.tzinfo is None:
|
||||
expiry = expiry.replace(tzinfo=timezone.utc)
|
||||
now = moment or datetime.now(timezone.utc)
|
||||
return STATUS_ACTIVE if expiry <= now else STATUS_SUSPENDED
|
||||
|
||||
|
||||
def record_device_connection(
|
||||
device_uid: str,
|
||||
*,
|
||||
platform: str = "",
|
||||
client_version: str = "",
|
||||
protocol_version: int = 0,
|
||||
country: str = "",
|
||||
asn: str = "",
|
||||
asn_class: str = "unknown",
|
||||
ip_hash: str = "",
|
||||
) -> None:
|
||||
get_table("node_devices").update(
|
||||
{
|
||||
"uid": device_uid,
|
||||
"platform": platform[:40],
|
||||
"client_version": client_version[:40],
|
||||
"protocol_version": int(protocol_version),
|
||||
"country": country[:2].upper(),
|
||||
"asn": asn[:20],
|
||||
"asn_class": asn_class,
|
||||
"ip_hash": ip_hash,
|
||||
"last_seen_at": _now_iso(),
|
||||
},
|
||||
["uid"],
|
||||
)
|
||||
|
||||
|
||||
def _clamped(value, low: int, high: int, default: int) -> int:
|
||||
try:
|
||||
number = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return max(low, min(high, number))
|
||||
|
||||
|
||||
def _hours(value) -> str:
|
||||
value = (value or "").strip()
|
||||
return value if HOURS_PATTERN.match(value) else ""
|
||||
|
||||
|
||||
def _platforms(value) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except ValueError:
|
||||
return list(PLATFORMS)
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return list(PLATFORMS)
|
||||
return [item for item in PLATFORMS if item in set(value)]
|
||||
|
||||
|
||||
def normalise_settings(values: dict) -> dict:
|
||||
return {
|
||||
"enabled": bool(values.get("enabled")),
|
||||
"bandwidth_mb_day": _clamped(
|
||||
values.get("bandwidth_mb_day"),
|
||||
1,
|
||||
LIMITS.max_donor_mb_per_day,
|
||||
LIMITS.default_donor_mb_per_day,
|
||||
),
|
||||
"max_concurrent": _clamped(
|
||||
values.get("max_concurrent"),
|
||||
1,
|
||||
LIMITS.max_concurrent_per_node,
|
||||
LIMITS.max_concurrent_per_node,
|
||||
),
|
||||
"active_from": _hours(values.get("active_from")),
|
||||
"active_to": _hours(values.get("active_to")),
|
||||
"timezone": (values.get("timezone") or "UTC").strip()[:64] or "UTC",
|
||||
"allowed_platforms": _platforms(values.get("allowed_platforms")),
|
||||
"pause_on_battery": bool(values.get("pause_on_battery")),
|
||||
"public_display": bool(values.get("public_display")),
|
||||
}
|
||||
|
||||
|
||||
def get_node_settings(user_uid: str) -> dict:
|
||||
row = get_table("node_settings").find_one(user_uid=user_uid, deleted_at=None)
|
||||
if not row:
|
||||
return dict(DEFAULT_SETTINGS)
|
||||
return normalise_settings(row)
|
||||
|
||||
|
||||
def save_node_settings(user_uid: str, values: dict) -> dict:
|
||||
merged = {**get_node_settings(user_uid), **values}
|
||||
clean = normalise_settings(merged)
|
||||
now = _now_iso()
|
||||
_execute(
|
||||
"INSERT INTO node_settings (uid, user_uid, enabled, bandwidth_mb_day, "
|
||||
"max_concurrent, active_from, active_to, timezone, allowed_platforms, "
|
||||
"pause_on_battery, public_display, created_at, updated_at) "
|
||||
"VALUES (:uid, :user_uid, :enabled, :mb, :concurrent, :hours_from, "
|
||||
":hours_to, :zone, :platforms, :battery, :public, :now, :now) "
|
||||
"ON CONFLICT(user_uid) DO UPDATE SET enabled = excluded.enabled, "
|
||||
"bandwidth_mb_day = excluded.bandwidth_mb_day, "
|
||||
"max_concurrent = excluded.max_concurrent, "
|
||||
"active_from = excluded.active_from, active_to = excluded.active_to, "
|
||||
"timezone = excluded.timezone, "
|
||||
"allowed_platforms = excluded.allowed_platforms, "
|
||||
"pause_on_battery = excluded.pause_on_battery, "
|
||||
"public_display = excluded.public_display, "
|
||||
"updated_at = excluded.updated_at",
|
||||
{
|
||||
"uid": _uid(),
|
||||
"user_uid": user_uid,
|
||||
"enabled": int(clean["enabled"]),
|
||||
"mb": clean["bandwidth_mb_day"],
|
||||
"concurrent": clean["max_concurrent"],
|
||||
"hours_from": clean["active_from"],
|
||||
"hours_to": clean["active_to"],
|
||||
"zone": clean["timezone"],
|
||||
"platforms": json.dumps(clean["allowed_platforms"]),
|
||||
"battery": int(clean["pause_on_battery"]),
|
||||
"public": int(clean["public_display"]),
|
||||
"now": now,
|
||||
},
|
||||
)
|
||||
return clean
|
||||
|
||||
|
||||
def daily_cap_bytes(user_uid: str) -> int:
|
||||
return get_node_settings(user_uid)["bandwidth_mb_day"] * MEGABYTE
|
||||
|
||||
|
||||
def record_job(
|
||||
device_uid: str,
|
||||
user_uid: str,
|
||||
*,
|
||||
ok: bool,
|
||||
bytes_up: int = 0,
|
||||
bytes_down: int = 0,
|
||||
cache_hit: bool = False,
|
||||
cap_bytes: int | None = None,
|
||||
moment: datetime | None = None,
|
||||
) -> bool:
|
||||
now = (moment or datetime.now(timezone.utc)).isoformat()
|
||||
day = _day(moment)
|
||||
up = max(0, int(bytes_up))
|
||||
down = max(0, int(bytes_down))
|
||||
cap = daily_cap_bytes(user_uid) if cap_bytes is None else int(cap_bytes)
|
||||
with db:
|
||||
db.executable.execute(
|
||||
text(
|
||||
"INSERT OR IGNORE INTO node_usage_daily (uid, device_uid, user_uid, "
|
||||
"day, requests_ok, requests_failed, bytes_up, bytes_down, "
|
||||
"cache_hits_served, integrity_failures, created_at, updated_at) "
|
||||
"VALUES (:uid, :device, :user, :day, 0, 0, 0, 0, 0, 0, :now, :now)"
|
||||
),
|
||||
{"uid": _uid(), "device": device_uid, "user": user_uid, "day": day, "now": now},
|
||||
)
|
||||
if not ok:
|
||||
db.executable.execute(
|
||||
text(
|
||||
"UPDATE node_usage_daily SET "
|
||||
"requests_failed = COALESCE(requests_failed, 0) + 1, "
|
||||
"updated_at = :now WHERE device_uid = :device AND day = :day"
|
||||
),
|
||||
{"now": now, "device": device_uid, "day": day},
|
||||
)
|
||||
return True
|
||||
counted = db.executable.execute(
|
||||
text(
|
||||
"UPDATE node_usage_daily SET "
|
||||
"requests_ok = COALESCE(requests_ok, 0) + 1, "
|
||||
"bytes_up = COALESCE(bytes_up, 0) + :up, "
|
||||
"bytes_down = COALESCE(bytes_down, 0) + :down, "
|
||||
"cache_hits_served = COALESCE(cache_hits_served, 0) + :hit, "
|
||||
"updated_at = :now "
|
||||
"WHERE device_uid = :device AND day = :day "
|
||||
"AND COALESCE(bytes_up, 0) + COALESCE(bytes_down, 0) + :up + :down "
|
||||
"<= :cap"
|
||||
),
|
||||
{
|
||||
"up": up,
|
||||
"down": down,
|
||||
"hit": 1 if cache_hit else 0,
|
||||
"now": now,
|
||||
"device": device_uid,
|
||||
"day": day,
|
||||
"cap": cap,
|
||||
},
|
||||
).rowcount
|
||||
if counted != 1:
|
||||
return False
|
||||
db.executable.execute(
|
||||
text(
|
||||
"INSERT INTO node_donors (uid, user_uid, requests_ok, bytes_total, "
|
||||
"first_donated_at, last_donated_at, xp_awarded_today, xp_day, "
|
||||
"created_at, updated_at) "
|
||||
"VALUES (:uid, :user, 1, :bytes, :now, :now, 0, '', :now, :now) "
|
||||
"ON CONFLICT(user_uid) DO UPDATE SET "
|
||||
"requests_ok = COALESCE(requests_ok, 0) + 1, "
|
||||
"bytes_total = COALESCE(bytes_total, 0) + :bytes, "
|
||||
"first_donated_at = CASE WHEN COALESCE(first_donated_at, '') = '' "
|
||||
"THEN :now ELSE first_donated_at END, "
|
||||
"last_donated_at = :now, updated_at = :now"
|
||||
),
|
||||
{"uid": _uid(), "user": user_uid, "bytes": up + down, "now": now},
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def record_integrity_failure(device_uid: str, user_uid: str) -> None:
|
||||
now = _now_iso()
|
||||
with db:
|
||||
db.executable.execute(
|
||||
text(
|
||||
"INSERT OR IGNORE INTO node_usage_daily (uid, device_uid, user_uid, "
|
||||
"day, requests_ok, requests_failed, bytes_up, bytes_down, "
|
||||
"cache_hits_served, integrity_failures, created_at, updated_at) "
|
||||
"VALUES (:uid, :device, :user, :day, 0, 0, 0, 0, 0, 0, :now, :now)"
|
||||
),
|
||||
{"uid": _uid(), "device": device_uid, "user": user_uid, "day": _day(), "now": now},
|
||||
)
|
||||
db.executable.execute(
|
||||
text(
|
||||
"UPDATE node_usage_daily SET "
|
||||
"integrity_failures = COALESCE(integrity_failures, 0) + 1, "
|
||||
"updated_at = :now WHERE device_uid = :device AND day = :day"
|
||||
),
|
||||
{"now": now, "device": device_uid, "day": _day()},
|
||||
)
|
||||
|
||||
|
||||
def claim_xp(user_uid: str, amount: int, *, moment: datetime | None = None) -> bool:
|
||||
if amount <= 0:
|
||||
return False
|
||||
day = _day(moment)
|
||||
return (
|
||||
_execute(
|
||||
"UPDATE node_donors SET "
|
||||
"xp_awarded_today = CASE WHEN COALESCE(xp_day, '') = :day "
|
||||
"THEN COALESCE(xp_awarded_today, 0) + :amount ELSE :amount END, "
|
||||
"xp_day = :day WHERE user_uid = :user AND deleted_at IS NULL AND "
|
||||
"(CASE WHEN COALESCE(xp_day, '') = :day "
|
||||
"THEN COALESCE(xp_awarded_today, 0) ELSE 0 END) + :amount <= :cap",
|
||||
{"day": day, "amount": amount, "user": user_uid, "cap": LIMITS.xp_daily_cap},
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def get_donor(user_uid: str) -> dict:
|
||||
row = get_table("node_donors").find_one(user_uid=user_uid, deleted_at=None)
|
||||
return {
|
||||
"requests_ok": int((row or {}).get("requests_ok") or 0),
|
||||
"bytes_total": int((row or {}).get("bytes_total") or 0),
|
||||
"first_donated_at": (row or {}).get("first_donated_at") or "",
|
||||
"last_donated_at": (row or {}).get("last_donated_at") or "",
|
||||
}
|
||||
|
||||
|
||||
def usage_since(user_uid: str, since_day: str) -> dict:
|
||||
rows = db.query(
|
||||
"SELECT COALESCE(SUM(requests_ok), 0) AS requests_ok, "
|
||||
"COALESCE(SUM(requests_failed), 0) AS requests_failed, "
|
||||
"COALESCE(SUM(bytes_up), 0) AS bytes_up, "
|
||||
"COALESCE(SUM(bytes_down), 0) AS bytes_down "
|
||||
"FROM node_usage_daily WHERE user_uid = :user AND day >= :since "
|
||||
"AND deleted_at IS NULL",
|
||||
user=user_uid,
|
||||
since=since_day,
|
||||
)
|
||||
return dict(next(iter(rows)))
|
||||
|
||||
|
||||
def usage_today(device_uid: str) -> dict:
|
||||
row = get_table("node_usage_daily").find_one(
|
||||
device_uid=device_uid, day=_day(), deleted_at=None
|
||||
)
|
||||
row = row or {}
|
||||
return {
|
||||
key: int(row.get(key) or 0)
|
||||
for key in (
|
||||
"requests_ok",
|
||||
"requests_failed",
|
||||
"bytes_up",
|
||||
"bytes_down",
|
||||
"cache_hits_served",
|
||||
"integrity_failures",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def community_stats() -> dict:
|
||||
rows = db.query(
|
||||
"SELECT COALESCE(SUM(requests_ok), 0) AS requests, "
|
||||
"COALESCE(SUM(bytes_total), 0) AS bytes, COUNT(*) AS donors "
|
||||
"FROM node_donors WHERE deleted_at IS NULL AND COALESCE(requests_ok, 0) > 0"
|
||||
)
|
||||
row = next(iter(rows))
|
||||
return {
|
||||
"requests": int(row["requests"]),
|
||||
"bytes": int(row["bytes"]),
|
||||
"donors": int(row["donors"]),
|
||||
}
|
||||
|
||||
|
||||
def add_event(
|
||||
kind: str, *, device_uid: str = "", user_uid: str = "", detail: str = ""
|
||||
) -> None:
|
||||
get_table("node_events").insert(
|
||||
{
|
||||
"uid": _uid(),
|
||||
"device_uid": device_uid,
|
||||
"user_uid": user_uid,
|
||||
"kind": kind[:40],
|
||||
"detail": (detail or "")[:EVENT_DETAIL_MAX],
|
||||
"created_at": _now_iso(),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def raise_flag(kind: str, *, device_uid: str = "", user_uid: str = "", detail: str = "") -> dict:
|
||||
table = get_table("node_flags")
|
||||
existing = table.find_one(
|
||||
kind=kind, device_uid=device_uid, status=FLAG_OPEN, deleted_at=None
|
||||
)
|
||||
if existing:
|
||||
return existing
|
||||
uid = _uid()
|
||||
now = _now_iso()
|
||||
table.insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"device_uid": device_uid,
|
||||
"user_uid": user_uid,
|
||||
"kind": kind[:40],
|
||||
"detail": (detail or "")[:EVENT_DETAIL_MAX],
|
||||
"status": FLAG_OPEN,
|
||||
"resolved_by": "",
|
||||
"resolved_at": "",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
)
|
||||
return table.find_one(uid=uid)
|
||||
|
||||
|
||||
def list_flags(status: str = FLAG_OPEN) -> list[dict]:
|
||||
return list(
|
||||
get_table("node_flags").find(
|
||||
status=status, deleted_at=None, order_by=["-created_at"]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def resolve_flag(flag_uid: str, actor_uid: str) -> bool:
|
||||
return (
|
||||
_execute(
|
||||
"UPDATE node_flags SET status = 'resolved', resolved_by = :actor, "
|
||||
"resolved_at = :now, updated_at = :now "
|
||||
"WHERE uid = :uid AND status = 'open' AND deleted_at IS NULL",
|
||||
{"actor": actor_uid, "now": _now_iso(), "uid": flag_uid},
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def prune_usage_daily(*, dry_run: bool = False, moment: datetime | None = None) -> int:
|
||||
cutoff = _day((moment or datetime.now(timezone.utc)) - timedelta(days=LIMITS.usage_daily_retention_days))
|
||||
count = next(
|
||||
iter(
|
||||
db.query(
|
||||
"SELECT COUNT(*) AS c FROM node_usage_daily WHERE day < :cutoff",
|
||||
cutoff=cutoff,
|
||||
)
|
||||
)
|
||||
)["c"]
|
||||
if not dry_run and count:
|
||||
_execute("DELETE FROM node_usage_daily WHERE day < :cutoff", {"cutoff": cutoff})
|
||||
return count
|
||||
|
||||
|
||||
def prune_events(*, dry_run: bool = False, moment: datetime | None = None) -> int:
|
||||
cutoff = (
|
||||
(moment or datetime.now(timezone.utc))
|
||||
- timedelta(days=LIMITS.event_retention_days)
|
||||
).isoformat()
|
||||
aged = next(
|
||||
iter(
|
||||
db.query(
|
||||
"SELECT COUNT(*) AS c FROM node_events WHERE created_at < :cutoff",
|
||||
cutoff=cutoff,
|
||||
)
|
||||
)
|
||||
)["c"]
|
||||
total = get_table("node_events").count()
|
||||
excess = max(0, total - aged - LIMITS.event_max_rows)
|
||||
if not dry_run:
|
||||
if aged:
|
||||
_execute("DELETE FROM node_events WHERE created_at < :cutoff", {"cutoff": cutoff})
|
||||
if excess:
|
||||
_execute(
|
||||
"DELETE FROM node_events WHERE id IN "
|
||||
"(SELECT id FROM node_events ORDER BY created_at, id LIMIT :n)",
|
||||
{"n": excess},
|
||||
)
|
||||
return aged + excess
|
||||
|
||||
|
||||
def prune_requests(*, dry_run: bool = False, moment: datetime | None = None) -> int:
|
||||
cutoff = (
|
||||
(moment or datetime.now(timezone.utc))
|
||||
- timedelta(seconds=LIMITS.request_retention_seconds)
|
||||
).isoformat()
|
||||
count = next(
|
||||
iter(
|
||||
db.query(
|
||||
"SELECT COUNT(*) AS c FROM node_requests WHERE created_at < :cutoff",
|
||||
cutoff=cutoff,
|
||||
)
|
||||
)
|
||||
)["c"]
|
||||
if not dry_run and count:
|
||||
_execute("DELETE FROM node_requests WHERE created_at < :cutoff", {"cutoff": cutoff})
|
||||
return count
|
||||
@@ -1,5 +1,6 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from devplacepy.services.nodes.spec import SPEC as NODES_SPEC
|
||||
from .core import AQUALITY_NEWS_GRADING_MODEL, AQUALITY_NEWS_GRADING_URL, DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger
|
||||
from .settings import get_setting, set_setting
|
||||
from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns
|
||||
@@ -1850,6 +1851,7 @@ def init_db():
|
||||
|
||||
_ensure_moderation_tables()
|
||||
_ensure_skills_table()
|
||||
_ensure_nodes_tables()
|
||||
|
||||
for table in db.tables:
|
||||
_uid_index(db, table)
|
||||
@@ -2004,6 +2006,12 @@ def init_db():
|
||||
"guidelines_version": "1",
|
||||
"ai_third_party_provider": "",
|
||||
"account_deletion_grace_hours": "24",
|
||||
"nodes_enabled": "0",
|
||||
"nodes_max_total": str(NODES_SPEC.limits.max_connected_nodes),
|
||||
"nodes_daily_quota_user": str(NODES_SPEC.limits.default_daily_requests_user),
|
||||
"nodes_daily_quota_guest": str(NODES_SPEC.limits.default_daily_requests_guest),
|
||||
"nodes_cache_ttl_seconds": str(NODES_SPEC.limits.cache_ttl_seconds),
|
||||
"nodes_social_via_nodes": "0",
|
||||
}
|
||||
for key, value in operational_defaults.items():
|
||||
existing = db["site_settings"].find_one(key=key)
|
||||
@@ -2191,6 +2199,159 @@ def _ensure_skills_table() -> None:
|
||||
)
|
||||
|
||||
|
||||
NODES_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
|
||||
"node_devices": (
|
||||
("uid", ""),
|
||||
("user_uid", ""),
|
||||
("label", ""),
|
||||
("kind", ""),
|
||||
("secret_hash", ""),
|
||||
("secret_prefix", ""),
|
||||
("platform", ""),
|
||||
("client_version", ""),
|
||||
("protocol_version", 0),
|
||||
("status", ""),
|
||||
("suspended_until", ""),
|
||||
("suspend_reason", ""),
|
||||
("country", ""),
|
||||
("asn", ""),
|
||||
("asn_class", ""),
|
||||
("ip_hash", ""),
|
||||
("last_seen_at", ""),
|
||||
("created_at", ""),
|
||||
("updated_at", ""),
|
||||
("deleted_at", ""),
|
||||
("deleted_by", ""),
|
||||
),
|
||||
"node_settings": (
|
||||
("uid", ""),
|
||||
("user_uid", ""),
|
||||
("enabled", 0),
|
||||
("bandwidth_mb_day", 0),
|
||||
("max_concurrent", 0),
|
||||
("active_from", ""),
|
||||
("active_to", ""),
|
||||
("timezone", ""),
|
||||
("allowed_platforms", ""),
|
||||
("pause_on_battery", 0),
|
||||
("public_display", 0),
|
||||
("created_at", ""),
|
||||
("updated_at", ""),
|
||||
("deleted_at", ""),
|
||||
("deleted_by", ""),
|
||||
),
|
||||
"node_usage_daily": (
|
||||
("uid", ""),
|
||||
("device_uid", ""),
|
||||
("user_uid", ""),
|
||||
("day", ""),
|
||||
("requests_ok", 0),
|
||||
("requests_failed", 0),
|
||||
("bytes_up", 0),
|
||||
("bytes_down", 0),
|
||||
("cache_hits_served", 0),
|
||||
("integrity_failures", 0),
|
||||
("created_at", ""),
|
||||
("updated_at", ""),
|
||||
("deleted_at", ""),
|
||||
("deleted_by", ""),
|
||||
),
|
||||
"node_donors": (
|
||||
("uid", ""),
|
||||
("user_uid", ""),
|
||||
("requests_ok", 0),
|
||||
("bytes_total", 0),
|
||||
("first_donated_at", ""),
|
||||
("last_donated_at", ""),
|
||||
("xp_awarded_today", 0),
|
||||
("xp_day", ""),
|
||||
("created_at", ""),
|
||||
("updated_at", ""),
|
||||
("deleted_at", ""),
|
||||
("deleted_by", ""),
|
||||
),
|
||||
"node_requests": (
|
||||
("uid", ""),
|
||||
("caller_kind", ""),
|
||||
("caller_id", ""),
|
||||
("url", ""),
|
||||
("method", ""),
|
||||
("adapter", ""),
|
||||
("mode", ""),
|
||||
("status", ""),
|
||||
("device_uid", ""),
|
||||
("result_status", 0),
|
||||
("result_bytes", 0),
|
||||
("result_body", ""),
|
||||
("error", ""),
|
||||
("created_at", ""),
|
||||
("assigned_at", ""),
|
||||
("finished_at", ""),
|
||||
),
|
||||
"node_events": (
|
||||
("uid", ""),
|
||||
("device_uid", ""),
|
||||
("user_uid", ""),
|
||||
("kind", ""),
|
||||
("detail", ""),
|
||||
("created_at", ""),
|
||||
),
|
||||
"node_flags": (
|
||||
("uid", ""),
|
||||
("device_uid", ""),
|
||||
("user_uid", ""),
|
||||
("kind", ""),
|
||||
("detail", ""),
|
||||
("status", ""),
|
||||
("resolved_by", ""),
|
||||
("resolved_at", ""),
|
||||
("created_at", ""),
|
||||
("updated_at", ""),
|
||||
("deleted_at", ""),
|
||||
("deleted_by", ""),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _ensure_nodes_tables() -> None:
|
||||
for table_name, columns in NODES_COLUMNS.items():
|
||||
table = get_table(table_name)
|
||||
for column, example in columns:
|
||||
if not table.has_column(column):
|
||||
table.create_column_by_example(column, example)
|
||||
_index(db, "node_devices", "idx_node_devices_user", ["user_uid", "deleted_at"])
|
||||
_index(db, "node_devices", "idx_node_devices_status", ["status", "last_seen_at"])
|
||||
_index(db, "node_devices", "idx_node_devices_prefix", ["secret_prefix"])
|
||||
_index(db, "node_settings", "idx_node_settings_user", ["user_uid"], unique=True)
|
||||
_index(db, "node_donors", "idx_node_donors_user", ["user_uid"], unique=True)
|
||||
_index(
|
||||
db,
|
||||
"node_usage_daily",
|
||||
"idx_node_usage_device_day",
|
||||
["device_uid", "day"],
|
||||
unique=True,
|
||||
)
|
||||
_index(db, "node_usage_daily", "idx_node_usage_day", ["day"])
|
||||
_index(db, "node_usage_daily", "idx_node_usage_user", ["user_uid", "day"])
|
||||
_index(
|
||||
db,
|
||||
"node_requests",
|
||||
"idx_node_requests_open",
|
||||
["status", "created_at"],
|
||||
where="status IN ('queued','assigned')",
|
||||
)
|
||||
_index(db, "node_requests", "idx_node_requests_created", ["created_at"])
|
||||
_index(db, "node_events", "idx_node_events_created", ["created_at"])
|
||||
_index(db, "node_events", "idx_node_events_device", ["device_uid", "created_at"])
|
||||
_index(
|
||||
db,
|
||||
"node_flags",
|
||||
"idx_node_flags_open",
|
||||
["status", "created_at"],
|
||||
where="deleted_at IS NULL",
|
||||
)
|
||||
|
||||
|
||||
def _ensure_moderation_tables() -> None:
|
||||
for table_name, columns in MODERATION_COLUMNS.items():
|
||||
table = get_table(table_name)
|
||||
|
||||
@@ -18,6 +18,11 @@ SOFT_DELETE_TABLES = [
|
||||
"bookmarks",
|
||||
"notes",
|
||||
"skills",
|
||||
"node_devices",
|
||||
"node_settings",
|
||||
"node_usage_daily",
|
||||
"node_donors",
|
||||
"node_flags",
|
||||
"follows",
|
||||
"poll_votes",
|
||||
"polls",
|
||||
|
||||
@@ -29,7 +29,7 @@ It is off by default and it is never appropriate on a real production host.
|
||||
|
||||
`AGREEMENTS` annotates `database.CONSENT_KINDS` with two facts: which `site_settings` key holds the policy version, and which `users` column the application's own gate reads. `terms` is the only agreement with a gate column, because `needs_acceptance` reads `users.terms_version` and not the ledger; `privacy` is versioned but ledger-only, matching `VERSION_KEYS` in `routers/profile/consent.py`.
|
||||
|
||||
A unit test asserts `{a.kind for a in AGREEMENTS} == set(CONSENT_KINDS)`. **A sixth consent fails the suite until it is classified here**, and it then appears in the admin form with no edit to the service, because the per-agreement config fields are built from the registry rather than written out by hand.
|
||||
A unit test asserts that `AGREEMENTS` and `EXCLUDED_CONSENTS` are disjoint and together equal `set(CONSENT_KINDS)`. **A new consent fails the suite until it is classified here**, and it then appears in the admin form with no edit to the service, because the per-agreement config fields are built from the registry rather than written out by hand.
|
||||
|
||||
## Rules that must not regress
|
||||
|
||||
@@ -49,6 +49,7 @@ A unit test asserts `{a.kind for a in AGREEMENTS} == set(CONSENT_KINDS)`. **A si
|
||||
| `users.age_band` | A declaration of fact, not an agreement. Fabricating a declared age would silently unlock `restricted` content for an account that declared 13-15. Sign the test account up with an adult date of birth instead. |
|
||||
| `users.mature_opt_in` | A preference gated by the age band, with no ledger row and therefore no decline register. The site setting `moderation_mature_default_hidden` already turns the interstitial off instance-wide. |
|
||||
| Guest consents | Nothing writes a consent row with `owner_kind = "guest"`, and the gateway resolves a guest to owner kind `anonymous`, which `consent_denied` exempts by design. |
|
||||
| `bandwidth_sharing` consent | Lets other people use the member's own internet connection. Granting it on someone's behalf would be real harm, not a test convenience. It is listed in `EXCLUDED_CONSENTS` with its reason and the service never touches it. |
|
||||
| Preferences (`interactions_enabled`, notifications, customization) | None of them blocks anything. |
|
||||
|
||||
## The terms asymmetry, and the one operator step
|
||||
@@ -59,7 +60,7 @@ Withdrawing the `terms` consent does not clear `users.terms_version`, so a teste
|
||||
|
||||
## Rules for extending this
|
||||
|
||||
- A new consent kind: classify it in `AGREEMENTS` (the test forces this), and nothing else.
|
||||
- A new consent kind: classify it in `AGREEMENTS`, or in `EXCLUDED_CONSENTS` with a reason when it is not a policy agreement (the test forces one of the two), and nothing else.
|
||||
- Never add a route, a schema, a template, a Devii tool or a `docs_api` entry. The surface is the generic services admin, exactly as for `NotificationRelayService` and `AuditService`.
|
||||
- Never add a second decline register, a second ordering, or an environment check.
|
||||
- Never write a persisted test that enables the service and asserts convergence: the suite is serial against one seeded database, and granting `terms` to every account would poison the moderation tests that assert the gate refuses. Convergence is covered by unit tests calling `converge_user` directly.
|
||||
|
||||
@@ -21,6 +21,13 @@ AGREEMENTS: tuple[Agreement, ...] = (
|
||||
Agreement("container_credentials", "", "", ""),
|
||||
)
|
||||
|
||||
EXCLUDED_CONSENTS: dict[str, str] = {
|
||||
"bandwidth_sharing": (
|
||||
"grants the use of the member's own internet connection by other people, "
|
||||
"so only the member can give it"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def setting_key(kind: str) -> str:
|
||||
return f"acceptance_grant_{kind}"
|
||||
|
||||
@@ -60,6 +60,10 @@ OWNED_TABLES: tuple[tuple[str, str], ...] = (
|
||||
("deepsearch_sessions", "owner_id"),
|
||||
("isslop_analyses", "owner_id"),
|
||||
("skills", "user_uid"),
|
||||
("node_devices", "user_uid"),
|
||||
("node_settings", "user_uid"),
|
||||
("node_donors", "user_uid"),
|
||||
("node_usage_daily", "user_uid"),
|
||||
)
|
||||
|
||||
CHILD_TABLES: tuple[tuple[str, str, str, str], ...] = (
|
||||
|
||||
@@ -2,7 +2,7 @@ This file documents the community node network (`devplacepy/services/nodes/`, `d
|
||||
|
||||
## What exists today
|
||||
|
||||
Only Phase 1 (the specification) is built. Everything else is planned and listed under "Gaps" next to the phase that closes it. Nothing here is enabled for any account.
|
||||
Phase 1 (the specification) and Phase 2 (the data layer and usage ledger) are built. Everything else is planned and listed under "Gaps" next to the phase that closes it. Nothing here is enabled for any account.
|
||||
|
||||
| Piece | File | State |
|
||||
|-------|------|-------|
|
||||
@@ -10,6 +10,9 @@ Only Phase 1 (the specification) is built. Everything else is planned and listed
|
||||
| `GET /nodes/spec.json` | `routers/nodes/spec.py` | built, tested, public |
|
||||
| Generated protocol docs page | `templates/docs/nodes-protocol.html` | built, reads the `nodes_spec` Jinja global |
|
||||
| API docs group | `docs_api/groups/nodes.py` | built, one endpoint |
|
||||
| Tables and helpers | `database/nodes.py`, `NODES_COLUMNS` in `database/schema.py` | built, tested, no route uses them yet |
|
||||
| Usage ledger | `ledger.py` (`UsageLedger`, `DonationSummary`) | built, tested |
|
||||
| `bandwidth_sharing` consent kind | `CONSENT_KINDS` in `database/moderation.py` | registered, never granted by default, enforced nowhere yet |
|
||||
|
||||
## The specification is the contract
|
||||
|
||||
@@ -35,13 +38,24 @@ The docs page `nodes-protocol.html` renders its tables from the `nodes_spec` Jin
|
||||
| D5 | donors must be adults (`age_band`) |
|
||||
| D6 | a datacenter address is not a donor: residential and mobile only, admin override |
|
||||
|
||||
## Data layer (Phase 2)
|
||||
|
||||
Seven tables, all ensured with their full column set in `init_db` (`_ensure_nodes_tables`): `node_devices`, `node_settings`, `node_usage_daily`, `node_donors`, `node_flags` (soft-deletable, in `SOFT_DELETE_TABLES` and `UNREPORTABLE_TABLES` with a reason), and `node_requests`, `node_events` (ephemeral, hard garbage collection only). Account deletion cascades through `OWNED_TABLES` for devices, settings, donors and daily usage.
|
||||
|
||||
- **Atomic by one statement.** The device cap is an `INSERT ... SELECT ... WHERE (SELECT COUNT(*)...) < max`, the daily bandwidth cap is an `UPDATE ... WHERE COALESCE(bytes_up,0)+COALESCE(bytes_down,0)+:n <= :cap`, and the daily XP cap is a conditional `UPDATE` that rolls the day over inside the same statement. Verified with real separate processes against fresh, unseeded rows (40 racers on a 10-job cap gave exactly 10, 24 device creations gave exactly 5, 30 XP claims gave exactly the cap).
|
||||
- **`record_job` is one transaction**: the daily bucket (`INSERT OR IGNORE` then the conditional `UPDATE`) and the lifetime `node_donors` upsert commit together or not at all. A job refused by the cap counts nothing. A failed job counts `requests_failed` and never moves bytes or lifetime totals.
|
||||
- **Lifetime totals are maintained at write time**, so pruning the daily buckets (`usage_daily_retention_days`) can never lower them. Prune functions take `dry_run` and use only the constants in `LimitSpec`; there is no argument that lowers a retention floor.
|
||||
- **Secrets**: `dpn_` plus 32 random bytes, shown once at create or rotate, stored as sha256 plus a 12 character lookup prefix, compared with `hmac.compare_digest`. IP addresses are stored only as a keyed hash (`hash_ip`).
|
||||
- **Settings are clamped on write and on read** (`normalise_settings`): bandwidth to `1..max_donor_mb_per_day`, concurrency to `1..max_concurrent_per_node`, hours must be `HH:MM`, platforms are intersected with the spec's adapters. A member with no row is opted out.
|
||||
- Site settings seeded in `init_db`: `nodes_enabled` (`0`), `nodes_max_total`, `nodes_daily_quota_user`, `nodes_daily_quota_guest`, `nodes_cache_ttl_seconds`, `nodes_social_via_nodes` (`0`). Like every seed, consumers pass the default to `get_setting` because the seed does not run on an empty database.
|
||||
|
||||
## Two transports, two trust levels
|
||||
|
||||
`tunnel` (native client) relays raw TCP so TLS runs end to end between DevPlace and the destination: the donor cannot read or forge content. `fetch` (browser extension, which has no raw sockets) lets the donor read and forge the response, so every `fetch` result is verified and labelled. A plain web page or a service worker cannot do either (CORS, no sockets) and is out of scope for good.
|
||||
|
||||
## Gaps (what the next phases close)
|
||||
|
||||
- Phase 2: tables, indexes, the `bandwidth_sharing` consent kind, `OWNED_TABLES` and `UNREPORTABLE_TABLES` entries, the usage ledger.
|
||||
- Phase 2 leftovers: the `bandwidth_sharing` consent can currently be granted through `POST /profile/{username}/consent` and by Devii's `set_consent` (Devii calls that route as the user). Nothing reads it yet, so it is inert, but Phase 3 must make Devii unable to grant it (Devii may read and withdraw) at the same time as `ConsentGate` lands. No XP or badge is awarded yet (`UsageLedger.claim_xp` only reserves the daily budget); the privacy page and app-store declarations are updated with the first phase that lets a device connect, because until then no personal data is collected.
|
||||
- Phase 3: registry, session, the consent choke point, the kill switch, device and ticket routes, the lock-owner WebSocket, the nginx upgrade location.
|
||||
- Phase 4: broker, scheduler, dispatcher, destination policy, circuit breakers, result cache.
|
||||
- Phase 5 onward: the reference client, adapters, the Go client, the extension, the profile UI, Devii tools, the CLI, the privacy and donor-terms pages, the legal gate.
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from devplacepy.database import nodes as store
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DonationSummary:
|
||||
lifetime_requests: int
|
||||
lifetime_bytes: int
|
||||
first_donated_at: str
|
||||
last_donated_at: str
|
||||
month_requests: int
|
||||
month_bytes_up: int
|
||||
month_bytes_down: int
|
||||
today_requests: int
|
||||
today_bytes: int
|
||||
|
||||
def describe(self) -> dict:
|
||||
return dict(self.__dict__)
|
||||
|
||||
|
||||
class UsageLedger:
|
||||
def record_success(
|
||||
self,
|
||||
device_uid: str,
|
||||
user_uid: str,
|
||||
*,
|
||||
bytes_up: int,
|
||||
bytes_down: int,
|
||||
cache_hit: bool = False,
|
||||
) -> bool:
|
||||
return store.record_job(
|
||||
device_uid,
|
||||
user_uid,
|
||||
ok=True,
|
||||
bytes_up=bytes_up,
|
||||
bytes_down=bytes_down,
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
|
||||
def record_failure(self, device_uid: str, user_uid: str) -> None:
|
||||
store.record_job(device_uid, user_uid, ok=False)
|
||||
|
||||
def record_integrity_failure(self, device_uid: str, user_uid: str) -> None:
|
||||
store.record_integrity_failure(device_uid, user_uid)
|
||||
|
||||
def claim_xp(self, user_uid: str, amount: int) -> bool:
|
||||
return store.claim_xp(user_uid, amount)
|
||||
|
||||
def summary(self, user_uid: str) -> DonationSummary:
|
||||
now = datetime.now(timezone.utc)
|
||||
donor = store.get_donor(user_uid)
|
||||
month = store.usage_since(user_uid, now.replace(day=1).date().isoformat())
|
||||
today = store.usage_since(user_uid, now.date().isoformat())
|
||||
return DonationSummary(
|
||||
lifetime_requests=donor["requests_ok"],
|
||||
lifetime_bytes=donor["bytes_total"],
|
||||
first_donated_at=donor["first_donated_at"],
|
||||
last_donated_at=donor["last_donated_at"],
|
||||
month_requests=month["requests_ok"],
|
||||
month_bytes_up=month["bytes_up"],
|
||||
month_bytes_down=month["bytes_down"],
|
||||
today_requests=today["requests_ok"],
|
||||
today_bytes=today["bytes_up"] + today["bytes_down"],
|
||||
)
|
||||
|
||||
def prune(self, *, dry_run: bool = False) -> dict:
|
||||
return {
|
||||
"usage_daily": store.prune_usage_daily(dry_run=dry_run),
|
||||
"events": store.prune_events(dry_run=dry_run),
|
||||
"requests": store.prune_requests(dry_run=dry_run),
|
||||
}
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "devplacepy"
|
||||
version = "1.0.18"
|
||||
version = "1.0.19"
|
||||
description = "DevPlace - The Developer Social Network"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from devplacepy.database import (
|
||||
CONSENT_KINDS,
|
||||
SOFT_DELETE_TABLES,
|
||||
UNREPORTABLE_TABLES,
|
||||
consent_granted,
|
||||
get_setting,
|
||||
get_table,
|
||||
)
|
||||
from devplacepy.database import nodes as store
|
||||
from devplacepy.services.moderation.deletion import OWNED_TABLES
|
||||
from devplacepy.services.nodes.ledger import UsageLedger
|
||||
from devplacepy.services.nodes.spec import SPEC
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
LIMITS = SPEC.limits
|
||||
SOFT_TABLES = ("node_devices", "node_settings", "node_usage_daily", "node_donors", "node_flags")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def owner(local_db):
|
||||
return generate_uid()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def device(owner):
|
||||
row, _ = store.create_device(owner, "laptop", "native")
|
||||
return row
|
||||
|
||||
|
||||
def test_node_tables_are_registered_everywhere_they_must_be():
|
||||
for table in SOFT_TABLES:
|
||||
assert table in SOFT_DELETE_TABLES
|
||||
assert table in UNREPORTABLE_TABLES
|
||||
for table in ("node_devices", "node_settings", "node_donors", "node_usage_daily"):
|
||||
assert (table, "user_uid") in OWNED_TABLES
|
||||
|
||||
|
||||
def test_bandwidth_sharing_consent_exists_and_is_never_seeded(owner):
|
||||
assert "bandwidth_sharing" in CONSENT_KINDS
|
||||
assert not consent_granted("user", owner, "bandwidth_sharing")
|
||||
assert get_setting("nodes_enabled", "0") == "0"
|
||||
|
||||
|
||||
def test_secret_is_shown_once_and_stored_hashed(owner):
|
||||
row, secret = store.create_device(owner, " desk ", "extension")
|
||||
assert secret.startswith(store.SECRET_PREFIX)
|
||||
assert row["label"] == "desk"
|
||||
assert secret not in str(row)
|
||||
assert store.find_device_by_secret(secret)["uid"] == row["uid"]
|
||||
assert store.find_device_by_secret(secret + "x") is None
|
||||
assert store.find_device_by_secret("") is None
|
||||
|
||||
|
||||
def test_rotate_invalidates_the_old_secret_and_checks_the_owner(owner):
|
||||
row, old = store.create_device(owner, "a", "native")
|
||||
assert store.rotate_device_secret(row["uid"], generate_uid()) is None
|
||||
new = store.rotate_device_secret(row["uid"], owner)
|
||||
assert new and new != old
|
||||
assert store.find_device_by_secret(old) is None
|
||||
assert store.find_device_by_secret(new)["uid"] == row["uid"]
|
||||
|
||||
|
||||
def test_device_cap_per_user_is_enforced(owner):
|
||||
for _ in range(LIMITS.max_devices_per_user):
|
||||
store.create_device(owner, "x", "native")
|
||||
with pytest.raises(store.NodeLimitError):
|
||||
store.create_device(owner, "x", "native")
|
||||
assert store.count_devices(owner) == LIMITS.max_devices_per_user
|
||||
|
||||
|
||||
def test_unknown_device_kind_is_rejected(owner):
|
||||
with pytest.raises(store.NodeLimitError):
|
||||
store.create_device(owner, "x", "browser")
|
||||
|
||||
|
||||
def test_revoke_is_a_soft_delete_and_frees_the_slot(owner):
|
||||
row, secret = store.create_device(owner, "a", "native")
|
||||
assert store.revoke_device(row["uid"], owner) == 1
|
||||
assert store.get_device(row["uid"]) is None
|
||||
assert store.find_device_by_secret(secret) is None
|
||||
assert store.count_devices(owner) == 0
|
||||
assert get_table("node_devices").find_one(uid=row["uid"])["deleted_by"] == owner
|
||||
|
||||
|
||||
def test_suspension_expires_on_its_own_and_revoked_never_suspends(owner):
|
||||
row, _ = store.create_device(owner, "a", "native")
|
||||
assert store.suspend_device(row["uid"], "integrity", 30)
|
||||
suspended = store.get_device(row["uid"])
|
||||
assert store.effective_status(suspended) == "suspended"
|
||||
later = datetime.now(timezone.utc) + timedelta(minutes=31)
|
||||
assert store.effective_status(suspended, later) == "active"
|
||||
assert store.unsuspend_device(row["uid"])
|
||||
assert store.get_device(row["uid"])["status"] == "active"
|
||||
store.revoke_device(row["uid"], owner)
|
||||
assert not store.suspend_device(row["uid"], "x", 5)
|
||||
|
||||
|
||||
def test_settings_default_to_opted_out(owner):
|
||||
settings = store.get_node_settings(owner)
|
||||
assert settings["enabled"] is False
|
||||
assert settings["bandwidth_mb_day"] == LIMITS.default_donor_mb_per_day
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [-5, 0, 1, 500, 10**9, "x", None, 3.7])
|
||||
def test_settings_are_clamped_to_the_declared_ceilings(owner, value):
|
||||
saved = store.save_node_settings(
|
||||
owner, {"bandwidth_mb_day": value, "max_concurrent": value}
|
||||
)
|
||||
assert 1 <= saved["bandwidth_mb_day"] <= LIMITS.max_donor_mb_per_day
|
||||
assert 1 <= saved["max_concurrent"] <= LIMITS.max_concurrent_per_node
|
||||
assert store.get_node_settings(owner) == saved
|
||||
|
||||
|
||||
def test_settings_reject_bad_hours_and_unknown_platforms(owner):
|
||||
saved = store.save_node_settings(
|
||||
owner,
|
||||
{"enabled": True, "active_from": "25:00", "active_to": "07:30",
|
||||
"allowed_platforms": ["reddit", "evil"]},
|
||||
)
|
||||
assert saved["active_from"] == ""
|
||||
assert saved["active_to"] == "07:30"
|
||||
assert saved["allowed_platforms"] == ["reddit"]
|
||||
again = store.save_node_settings(owner, {"enabled": False})
|
||||
assert again["active_to"] == "07:30"
|
||||
assert again["enabled"] is False
|
||||
|
||||
|
||||
def test_daily_cap_blocks_the_job_that_would_cross_it(owner, device):
|
||||
cap = 10_000
|
||||
results = [
|
||||
store.record_job(device["uid"], owner, ok=True, bytes_up=100, bytes_down=900, cap_bytes=cap)
|
||||
for _ in range(14)
|
||||
]
|
||||
assert results.count(True) == 10
|
||||
today = store.usage_today(device["uid"])
|
||||
assert today["requests_ok"] == 10
|
||||
assert today["bytes_up"] + today["bytes_down"] == cap
|
||||
donor = store.get_donor(owner)
|
||||
assert donor["requests_ok"] == 10 and donor["bytes_total"] == cap
|
||||
|
||||
|
||||
def test_failed_jobs_count_but_move_no_bytes_or_lifetime_totals(owner, device):
|
||||
store.record_job(device["uid"], owner, ok=False, bytes_down=999)
|
||||
today = store.usage_today(device["uid"])
|
||||
assert today["requests_failed"] == 1
|
||||
assert today["bytes_down"] == 0
|
||||
assert store.get_donor(owner)["requests_ok"] == 0
|
||||
|
||||
|
||||
def test_lifetime_totals_survive_pruning_the_daily_buckets(owner, device):
|
||||
store.record_job(device["uid"], owner, ok=True, bytes_down=10, cap_bytes=1000)
|
||||
far = datetime.now(timezone.utc) + timedelta(days=LIMITS.usage_daily_retention_days + 2)
|
||||
assert store.prune_usage_daily(dry_run=True, moment=far) >= 1
|
||||
assert store.usage_today(device["uid"])["requests_ok"] == 1
|
||||
assert store.prune_usage_daily(moment=far) >= 1
|
||||
assert store.usage_today(device["uid"])["requests_ok"] == 0
|
||||
assert store.get_donor(owner)["requests_ok"] == 1
|
||||
|
||||
|
||||
def test_recent_buckets_are_not_pruned(owner, device):
|
||||
store.record_job(device["uid"], owner, ok=True, cap_bytes=1000)
|
||||
assert store.prune_usage_daily() == 0
|
||||
|
||||
|
||||
def test_xp_is_capped_per_day_and_resets_the_next_day(owner, device):
|
||||
store.record_job(device["uid"], owner, ok=True, cap_bytes=1000)
|
||||
granted = sum(store.claim_xp(owner, 7) for _ in range(30))
|
||||
assert granted == LIMITS.xp_daily_cap // 7
|
||||
tomorrow = datetime.now(timezone.utc) + timedelta(days=1)
|
||||
assert store.claim_xp(owner, 7, moment=tomorrow)
|
||||
assert not store.claim_xp(owner, 0)
|
||||
assert not store.claim_xp(generate_uid(), 7)
|
||||
|
||||
|
||||
def test_integrity_failures_are_counted(owner, device):
|
||||
store.record_integrity_failure(device["uid"], owner)
|
||||
store.record_integrity_failure(device["uid"], owner)
|
||||
assert store.usage_today(device["uid"])["integrity_failures"] == 2
|
||||
|
||||
|
||||
def test_ledger_summary_reports_lifetime_month_and_today(owner, device):
|
||||
ledger = UsageLedger()
|
||||
assert ledger.record_success(device["uid"], owner, bytes_up=10, bytes_down=90)
|
||||
ledger.record_failure(device["uid"], owner)
|
||||
summary = ledger.summary(owner)
|
||||
assert summary.lifetime_requests == 1
|
||||
assert summary.lifetime_bytes == 100
|
||||
assert summary.month_requests == 1
|
||||
assert summary.today_bytes == 100
|
||||
assert summary.first_donated_at
|
||||
|
||||
|
||||
def test_community_stats_are_aggregate_only(owner, device):
|
||||
store.record_job(device["uid"], owner, ok=True, bytes_down=50, cap_bytes=1000)
|
||||
stats = store.community_stats()
|
||||
assert set(stats) == {"requests", "bytes", "donors"}
|
||||
assert stats["requests"] >= 1 and stats["donors"] >= 1
|
||||
|
||||
|
||||
def test_flags_deduplicate_while_open_and_resolve_once(owner):
|
||||
flag = store.raise_flag("integrity", device_uid="d1", user_uid=owner, detail="x")
|
||||
assert store.raise_flag("integrity", device_uid="d1")["uid"] == flag["uid"]
|
||||
assert store.resolve_flag(flag["uid"], "admin")
|
||||
assert not store.resolve_flag(flag["uid"], "admin")
|
||||
assert flag["uid"] not in [item["uid"] for item in store.list_flags()]
|
||||
|
||||
|
||||
def test_event_log_is_bounded_by_age(owner):
|
||||
store.add_event("connect", user_uid=owner, detail="x" * 1000)
|
||||
row = get_table("node_events").find_one(user_uid=owner)
|
||||
assert len(row["detail"]) == store.EVENT_DETAIL_MAX
|
||||
far = datetime.now(timezone.utc) + timedelta(days=LIMITS.event_retention_days + 1)
|
||||
assert store.prune_events(dry_run=True, moment=far) >= 1
|
||||
assert store.prune_events(moment=far) >= 1
|
||||
|
||||
|
||||
def test_ip_is_only_ever_stored_hashed():
|
||||
digest = store.hash_ip("203.0.113.9")
|
||||
assert "203" not in digest and len(digest) == 32
|
||||
assert store.hash_ip("203.0.113.9") == digest
|
||||
assert store.hash_ip("") == ""
|
||||
@@ -3,6 +3,7 @@
|
||||
from devplacepy.database import CONSENT_KINDS
|
||||
from devplacepy.services.acceptance.agreements import (
|
||||
AGREEMENTS,
|
||||
EXCLUDED_CONSENTS,
|
||||
agreement_for,
|
||||
label_for,
|
||||
setting_key,
|
||||
@@ -11,7 +12,10 @@ from devplacepy.services.acceptance.service import AcceptanceService
|
||||
|
||||
|
||||
def test_every_consent_kind_is_classified_as_an_agreement():
|
||||
assert {agreement.kind for agreement in AGREEMENTS} == set(CONSENT_KINDS)
|
||||
classified = {agreement.kind for agreement in AGREEMENTS}
|
||||
assert classified.isdisjoint(EXCLUDED_CONSENTS)
|
||||
assert classified | set(EXCLUDED_CONSENTS) == set(CONSENT_KINDS)
|
||||
assert all(EXCLUDED_CONSENTS.values())
|
||||
|
||||
|
||||
def test_setting_keys_are_unique_and_namespaced():
|
||||
|
||||
Reference in New Issue
Block a user