Zo doen wijdat in oost

2026-10-16 07:20:44 +02:00
parent 25d9cb1c7d
commit 6f80a820da
229 changed files with 23894 additions and 74 deletions
@@ -621,9 +621,20 @@ def prune_usage_daily(*, dry_run: bool = False, moment: datetime | None = None)
            )
        )
    )["c"]
-    if not dry_run and count:
-        _execute("DELETE FROM node_usage_daily WHERE day < :cutoff", {"cutoff": cutoff})
-    return count
+    callers = next(
+        iter(
+            db.query(
+                "SELECT COUNT(*) AS c FROM node_caller_daily WHERE day < :cutoff",
+                cutoff=cutoff,
+            )
+        )
+    )["c"]
+    if not dry_run:
+        if count:
+            _execute("DELETE FROM node_usage_daily WHERE day < :cutoff", {"cutoff": cutoff})
+        if callers:
+            _execute("DELETE FROM node_caller_daily WHERE day < :cutoff", {"cutoff": cutoff})
+    return count + callers


 def prune_events(*, dry_run: bool = False, moment: datetime | None = None) -> int:
@@ -669,3 +680,312 @@ def prune_requests(*, dry_run: bool = False, moment: datetime | None = None) ->
    if not dry_run and count:
        _execute("DELETE FROM node_requests WHERE created_at < :cutoff", {"cutoff": cutoff})
    return count
+
+
+def clear_ephemeral(*, dry_run: bool = True) -> dict:
+    counts = {}
+    for table in ("node_requests", "node_events"):
+        counts[table] = next(iter(db.query(f"SELECT COUNT(*) AS c FROM {table}")))["c"]
+        if not dry_run and counts[table]:
+            _execute(f"DELETE FROM {table}", {})
+    return counts
+
+
+class QueueFull(NodeLimitError):
+    pass
+
+
+REQUEST_OPEN = ("queued", "assigned")
+REQUEST_COLUMNS_PUBLIC = (
+    "uid",
+    "status",
+    "result_status",
+    "result_bytes",
+    "result_headers",
+    "truncated",
+    "error",
+)
+
+
+def enqueue_request(
+    caller_kind: str,
+    caller_id: str,
+    url: str,
+    method: str,
+    adapter: str,
+    mode: str,
+) -> str:
+    uid = _uid()
+    created = _execute(
+        "INSERT INTO node_requests (uid, caller_kind, caller_id, url, method, adapter, "
+        "mode, status, device_uid, result_status, result_bytes, result_body, "
+        "result_headers, truncated, error, created_at, assigned_at, finished_at) "
+        "SELECT :uid, :kind, :caller, :url, :method, :adapter, :mode, 'queued', '', 0, 0, "
+        "'', '', 0, '', :now, '', '' "
+        "WHERE (SELECT COUNT(*) FROM node_requests WHERE status IN ('queued','assigned')) "
+        "< :max_depth",
+        {
+            "uid": uid,
+            "kind": caller_kind,
+            "caller": caller_id,
+            "url": url,
+            "method": method,
+            "adapter": adapter,
+            "mode": mode,
+            "now": _now_iso(),
+            "max_depth": LIMITS.max_queue_depth,
+        },
+    )
+    if created != 1:
+        raise QueueFull("The node queue is full")
+    return uid
+
+
+def get_request(request_uid: str) -> dict | None:
+    return get_table("node_requests").find_one(uid=request_uid)
+
+
+def list_queued(limit: int) -> list[dict]:
+    return [
+        dict(row)
+        for row in db.query(
+            "SELECT * FROM node_requests WHERE status = 'queued' "
+            "ORDER BY created_at, id LIMIT :n",
+            n=int(limit),
+        )
+    ]
+
+
+def count_open_requests() -> int:
+    return next(
+        iter(
+            db.query(
+                "SELECT COUNT(*) AS c FROM node_requests WHERE status IN ('queued','assigned')"
+            )
+        )
+    )["c"]
+
+
+def claim_request(request_uid: str, device_uid: str) -> bool:
+    return (
+        _execute(
+            "UPDATE node_requests SET status = 'assigned', device_uid = :device, "
+            "assigned_at = :now WHERE uid = :uid AND status = 'queued'",
+            {"device": device_uid, "now": _now_iso(), "uid": request_uid},
+        )
+        == 1
+    )
+
+
+def finish_request(
+    request_uid: str,
+    *,
+    result_status: int,
+    result_headers: str,
+    result_body: str,
+    result_bytes: int,
+    truncated: bool,
+) -> bool:
+    return (
+        _execute(
+            "UPDATE node_requests SET status = 'done', result_status = :status, "
+            "result_headers = :headers, result_body = :body, result_bytes = :bytes, "
+            "truncated = :truncated, finished_at = :now "
+            "WHERE uid = :uid AND status = 'assigned'",
+            {
+                "status": int(result_status),
+                "headers": result_headers,
+                "body": result_body,
+                "bytes": int(result_bytes),
+                "truncated": 1 if truncated else 0,
+                "now": _now_iso(),
+                "uid": request_uid,
+            },
+        )
+        == 1
+    )
+
+
+def set_request_transport(request_uid: str, transport: str) -> bool:
+    return (
+        _execute(
+            "UPDATE node_requests SET mode = :mode WHERE uid = :uid AND status = 'assigned'",
+            {"mode": transport, "uid": request_uid},
+        )
+        == 1
+    )
+
+
+def fail_request(request_uid: str, error: str, *, expired: bool = False) -> bool:
+    return (
+        _execute(
+            "UPDATE node_requests SET status = :status, error = :error, finished_at = :now "
+            "WHERE uid = :uid AND status IN ('queued','assigned')",
+            {
+                "status": "expired" if expired else "failed",
+                "error": (error or "")[:EVENT_DETAIL_MAX],
+                "now": _now_iso(),
+                "uid": request_uid,
+            },
+        )
+        == 1
+    )
+
+
+def expire_requests(moment: datetime | None = None) -> dict[str, int]:
+    now = moment or datetime.now(timezone.utc)
+    queued_cutoff = (now - timedelta(seconds=LIMITS.queue_max_wait_seconds)).isoformat()
+    assigned_cutoff = (
+        now - timedelta(seconds=LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds)
+    ).isoformat()
+    finished = now.isoformat()
+    queued = _execute(
+        "UPDATE node_requests SET status = 'expired', error = 'no_node', finished_at = :now "
+        "WHERE status = 'queued' AND created_at < :cutoff",
+        {"now": finished, "cutoff": queued_cutoff},
+    )
+    assigned = _execute(
+        "UPDATE node_requests SET status = 'failed', error = 'timeout', finished_at = :now "
+        "WHERE status = 'assigned' AND assigned_at < :cutoff",
+        {"now": finished, "cutoff": assigned_cutoff},
+    )
+    return {"queued": queued, "assigned": assigned}
+
+
+def reserve_caller_request(
+    caller_kind: str, caller_id: str, limit: int, *, moment: datetime | None = None
+) -> bool:
+    if limit <= 0 or not caller_id:
+        return False
+    now = (moment or datetime.now(timezone.utc)).isoformat()
+    day = _day(moment)
+    with db:
+        db.executable.execute(
+            text(
+                "INSERT OR IGNORE INTO node_caller_daily (uid, caller_kind, caller_id, day, "
+                "requests, created_at, updated_at) VALUES (:uid, :kind, :caller, :day, 0, :now, :now)"
+            ),
+            {"uid": _uid(), "kind": caller_kind, "caller": caller_id, "day": day, "now": now},
+        )
+        counted = db.executable.execute(
+            text(
+                "UPDATE node_caller_daily SET requests = COALESCE(requests, 0) + 1, "
+                "updated_at = :now WHERE caller_kind = :kind AND caller_id = :caller "
+                "AND day = :day AND deleted_at IS NULL AND COALESCE(requests, 0) < :limit"
+            ),
+            {"now": now, "kind": caller_kind, "caller": caller_id, "day": day, "limit": int(limit)},
+        ).rowcount
+    return counted == 1
+
+
+def caller_requests_today(caller_kind: str, caller_id: str) -> int:
+    row = get_table("node_caller_daily").find_one(
+        caller_kind=caller_kind, caller_id=caller_id, day=_day(), deleted_at=None
+    )
+    return int((row or {}).get("requests") or 0)
+
+
+def get_settings_by_users(user_uids) -> dict[str, dict]:
+    found: dict[str, dict] = {}
+    for batch in _batches(user_uids):
+        placeholders, params = _in_clause(batch)
+        for row in db.query(
+            f"SELECT * FROM node_settings WHERE user_uid IN ({placeholders}) "
+            "AND deleted_at IS NULL",
+            **params,
+        ):
+            found[row["user_uid"]] = normalise_settings(dict(row))
+    return found
+
+
+def donation_days(user_uid: str) -> int:
+    rows = db.query(
+        "SELECT COUNT(DISTINCT day) AS days FROM node_usage_daily "
+        "WHERE user_uid = :user AND COALESCE(requests_ok, 0) > 0 AND deleted_at IS NULL",
+        user=user_uid,
+    )
+    return int(next(iter(rows))["days"] or 0)
+
+
+def public_stats() -> dict:
+    row = next(
+        iter(
+            db.query(
+                "SELECT COALESCE(SUM(requests_ok), 0) AS requests, "
+                "COALESCE(SUM(bytes_total), 0) AS bytes, COUNT(*) AS donors "
+                "FROM node_donors WHERE deleted_at IS NULL AND COALESCE(requests_ok, 0) > 0"
+            )
+        )
+    )
+    return {
+        "requests": int(row["requests"] or 0),
+        "bytes": int(row["bytes"] or 0),
+        "donors": int(row["donors"] or 0),
+    }
+
+
+def top_donors(limit: int = 10) -> list[dict]:
+    limit = max(1, min(int(limit), 50))
+    rows = db.query(
+        "SELECT d.user_uid AS user_uid, d.requests_ok AS requests_ok, d.bytes_total AS bytes_total "
+        "FROM node_donors d JOIN node_settings s ON s.user_uid = d.user_uid "
+        "WHERE d.deleted_at IS NULL AND s.deleted_at IS NULL AND COALESCE(s.public_display, 0) = 1 "
+        "AND COALESCE(d.requests_ok, 0) > 0 ORDER BY d.bytes_total DESC LIMIT :limit",
+        limit=limit,
+    )
+    return [dict(row) for row in rows]
+
+
+ADMIN_LIST_LIMIT = 100
+ADMIN_BREAKDOWN_LIMIT = 50
+DEVICE_BREAKDOWN_COLUMNS = ("status", "kind", "asn_class", "country")
+REQUEST_STATUSES = ("queued", "assigned", "done", "failed", "expired")
+
+
+def device_breakdown(column: str) -> dict[str, int]:
+    if column not in DEVICE_BREAKDOWN_COLUMNS:
+        raise ValueError("Unknown device breakdown column")
+    rows = db.query(
+        f"SELECT COALESCE(NULLIF({column}, ''), 'unknown') AS bucket, COUNT(*) AS total "
+        "FROM node_devices WHERE deleted_at IS NULL "
+        "GROUP BY bucket ORDER BY total DESC, bucket LIMIT :n",
+        n=ADMIN_BREAKDOWN_LIMIT,
+    )
+    return {row["bucket"]: int(row["total"]) for row in rows}
+
+
+def request_status_counts() -> dict[str, int]:
+    counts = {status: 0 for status in REQUEST_STATUSES}
+    for row in db.query("SELECT status, COUNT(*) AS total FROM node_requests GROUP BY status"):
+        counts[row["status"]] = int(row["total"])
+    return counts
+
+
+def usage_day_totals(moment: datetime | None = None) -> dict[str, int]:
+    row = next(
+        iter(
+            db.query(
+                "SELECT COALESCE(SUM(requests_ok), 0) AS requests_ok, "
+                "COALESCE(SUM(requests_failed), 0) AS requests_failed, "
+                "COALESCE(SUM(bytes_up), 0) AS bytes_up, "
+                "COALESCE(SUM(bytes_down), 0) AS bytes_down, "
+                "COALESCE(SUM(cache_hits_served), 0) AS cache_hits_served, "
+                "COALESCE(SUM(integrity_failures), 0) AS integrity_failures "
+                "FROM node_usage_daily WHERE day = :day AND deleted_at IS NULL",
+                day=_day(moment),
+            )
+        )
+    )
+    return {key: int(value or 0) for key, value in dict(row).items()}
+
+
+def recent_devices(limit: int = ADMIN_LIST_LIMIT) -> list[dict]:
+    limit = max(1, min(int(limit), ADMIN_LIST_LIMIT))
+    return [
+        dict(row)
+        for row in db.query(
+            "SELECT * FROM node_devices WHERE deleted_at IS NULL "
+            "ORDER BY COALESCE(NULLIF(last_seen_at, ''), created_at) DESC, id DESC LIMIT :n",
+            n=limit,
+        )
+    ]