Zo doen wijdat in oost
DevPlace CI / test (push) Failing after 3m13s

This commit is contained in:
2026-10-16 07:20:44 +02:00
parent 25d9cb1c7d
commit 6f80a820da
229 changed files with 23894 additions and 74 deletions
+1
View File
@@ -223,6 +223,7 @@ Site settings are seeded on startup (`site_settings` table):
| `terms_version` / `privacy_version` / `guidelines_version` | `"1"` | Bumping `terms_version` forces re-acceptance before the next write. **Every reader uses `get_setting(key, "1") or "1"`** - an empty stored value must read as the default or the gate 403s every write |
| `ai_third_party_provider` | `""` | Named in the consent copy and the privacy policy |
| `extra_head` | `""` | Raw HTML emitted verbatim into every page `<head>` by `templating.extra_head_tag()`; site-wide trusted-admin input, not sanitized |
| `sync_enabled` / `sync_origins` | `"0"` / `""` | Community sync kill switch (`routers/sync/_shared.py`) and an optional comma separated allowlist of sending origins; empty allows any origin |
Besides the site/news/upload keys above, the **Operational** group is admin-editable at `/admin/settings`: `site_url` (public origin for absolute links incl. container ingress; resolved by `seo.public_base_url()` = setting -> `DEVPLACE_SITE_URL` env -> request origin), `rate_limit_per_minute`, `rate_limit_window_seconds`, `news_service_interval`, `session_max_age_days`, `session_remember_days`, `registration_open`, `maintenance_mode`, `maintenance_message`, `docs_search_mode` (`agent`|`bm25`, default `agent` - picks the `/docs/search.html` surface: the in-page Devii chat or the classic BM25 list; a viewer over their daily AI limit, or a guest when Devii is disabled, auto-falls-back to BM25 via `routers/docs/views.py` `_agent_search_state`), `outbound_proxy_url` (empty by default - when set, every `stealth.stealth_async_client`/`stealth_sync_client` call across the whole app routes through it via `stealth.configured_proxy_url()`; validated as `http(s)://`/`socks5(h)://` with a host in `AdminSettingsForm`; falls back to `DEVPLACE_OUTBOUND_PROXY_URL` when unset - see the "Outbound HTTP" note in the root `CLAUDE.md`). The **Custom Code** key `extra_head` and the `ios_app_url` badge link are the keys in the settings handler's `CLEARABLE_SETTINGS` set, so saving an empty textarea/field removes them (the default loop skips empty values).
+2
View File
@@ -49,10 +49,12 @@ UNREPORTABLE_TABLES: dict[str, str] = {
"bookmarks": "private to the owner",
"notes": "private to the owner",
"skills": "private to the owner, never shown to others",
"communities": "administrator-only configuration, never shown to members",
"node_devices": "private to the owner, the label is never shown to others",
"node_settings": "private to the owner",
"node_usage_daily": "derived donation counters, carry no authored content",
"node_donors": "derived donation totals, carry no authored content",
"node_caller_daily": "derived daily request counters, carry no authored content",
"node_flags": "operational flags for administrators, carry no authored content",
"follows": "relationship rows, carry no authored content",
"poll_votes": "private ballots",
+323 -3
View File
@@ -621,9 +621,20 @@ def prune_usage_daily(*, dry_run: bool = False, moment: datetime | None = None)
)
)
)["c"]
if not dry_run and count:
_execute("DELETE FROM node_usage_daily WHERE day < :cutoff", {"cutoff": cutoff})
return count
callers = next(
iter(
db.query(
"SELECT COUNT(*) AS c FROM node_caller_daily WHERE day < :cutoff",
cutoff=cutoff,
)
)
)["c"]
if not dry_run:
if count:
_execute("DELETE FROM node_usage_daily WHERE day < :cutoff", {"cutoff": cutoff})
if callers:
_execute("DELETE FROM node_caller_daily WHERE day < :cutoff", {"cutoff": cutoff})
return count + callers
def prune_events(*, dry_run: bool = False, moment: datetime | None = None) -> int:
@@ -669,3 +680,312 @@ def prune_requests(*, dry_run: bool = False, moment: datetime | None = None) ->
if not dry_run and count:
_execute("DELETE FROM node_requests WHERE created_at < :cutoff", {"cutoff": cutoff})
return count
def clear_ephemeral(*, dry_run: bool = True) -> dict:
counts = {}
for table in ("node_requests", "node_events"):
counts[table] = next(iter(db.query(f"SELECT COUNT(*) AS c FROM {table}")))["c"]
if not dry_run and counts[table]:
_execute(f"DELETE FROM {table}", {})
return counts
class QueueFull(NodeLimitError):
pass
REQUEST_OPEN = ("queued", "assigned")
REQUEST_COLUMNS_PUBLIC = (
"uid",
"status",
"result_status",
"result_bytes",
"result_headers",
"truncated",
"error",
)
def enqueue_request(
caller_kind: str,
caller_id: str,
url: str,
method: str,
adapter: str,
mode: str,
) -> str:
uid = _uid()
created = _execute(
"INSERT INTO node_requests (uid, caller_kind, caller_id, url, method, adapter, "
"mode, status, device_uid, result_status, result_bytes, result_body, "
"result_headers, truncated, error, created_at, assigned_at, finished_at) "
"SELECT :uid, :kind, :caller, :url, :method, :adapter, :mode, 'queued', '', 0, 0, "
"'', '', 0, '', :now, '', '' "
"WHERE (SELECT COUNT(*) FROM node_requests WHERE status IN ('queued','assigned')) "
"< :max_depth",
{
"uid": uid,
"kind": caller_kind,
"caller": caller_id,
"url": url,
"method": method,
"adapter": adapter,
"mode": mode,
"now": _now_iso(),
"max_depth": LIMITS.max_queue_depth,
},
)
if created != 1:
raise QueueFull("The node queue is full")
return uid
def get_request(request_uid: str) -> dict | None:
return get_table("node_requests").find_one(uid=request_uid)
def list_queued(limit: int) -> list[dict]:
return [
dict(row)
for row in db.query(
"SELECT * FROM node_requests WHERE status = 'queued' "
"ORDER BY created_at, id LIMIT :n",
n=int(limit),
)
]
def count_open_requests() -> int:
return next(
iter(
db.query(
"SELECT COUNT(*) AS c FROM node_requests WHERE status IN ('queued','assigned')"
)
)
)["c"]
def claim_request(request_uid: str, device_uid: str) -> bool:
return (
_execute(
"UPDATE node_requests SET status = 'assigned', device_uid = :device, "
"assigned_at = :now WHERE uid = :uid AND status = 'queued'",
{"device": device_uid, "now": _now_iso(), "uid": request_uid},
)
== 1
)
def finish_request(
request_uid: str,
*,
result_status: int,
result_headers: str,
result_body: str,
result_bytes: int,
truncated: bool,
) -> bool:
return (
_execute(
"UPDATE node_requests SET status = 'done', result_status = :status, "
"result_headers = :headers, result_body = :body, result_bytes = :bytes, "
"truncated = :truncated, finished_at = :now "
"WHERE uid = :uid AND status = 'assigned'",
{
"status": int(result_status),
"headers": result_headers,
"body": result_body,
"bytes": int(result_bytes),
"truncated": 1 if truncated else 0,
"now": _now_iso(),
"uid": request_uid,
},
)
== 1
)
def set_request_transport(request_uid: str, transport: str) -> bool:
return (
_execute(
"UPDATE node_requests SET mode = :mode WHERE uid = :uid AND status = 'assigned'",
{"mode": transport, "uid": request_uid},
)
== 1
)
def fail_request(request_uid: str, error: str, *, expired: bool = False) -> bool:
return (
_execute(
"UPDATE node_requests SET status = :status, error = :error, finished_at = :now "
"WHERE uid = :uid AND status IN ('queued','assigned')",
{
"status": "expired" if expired else "failed",
"error": (error or "")[:EVENT_DETAIL_MAX],
"now": _now_iso(),
"uid": request_uid,
},
)
== 1
)
def expire_requests(moment: datetime | None = None) -> dict[str, int]:
now = moment or datetime.now(timezone.utc)
queued_cutoff = (now - timedelta(seconds=LIMITS.queue_max_wait_seconds)).isoformat()
assigned_cutoff = (
now - timedelta(seconds=LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds)
).isoformat()
finished = now.isoformat()
queued = _execute(
"UPDATE node_requests SET status = 'expired', error = 'no_node', finished_at = :now "
"WHERE status = 'queued' AND created_at < :cutoff",
{"now": finished, "cutoff": queued_cutoff},
)
assigned = _execute(
"UPDATE node_requests SET status = 'failed', error = 'timeout', finished_at = :now "
"WHERE status = 'assigned' AND assigned_at < :cutoff",
{"now": finished, "cutoff": assigned_cutoff},
)
return {"queued": queued, "assigned": assigned}
def reserve_caller_request(
caller_kind: str, caller_id: str, limit: int, *, moment: datetime | None = None
) -> bool:
if limit <= 0 or not caller_id:
return False
now = (moment or datetime.now(timezone.utc)).isoformat()
day = _day(moment)
with db:
db.executable.execute(
text(
"INSERT OR IGNORE INTO node_caller_daily (uid, caller_kind, caller_id, day, "
"requests, created_at, updated_at) VALUES (:uid, :kind, :caller, :day, 0, :now, :now)"
),
{"uid": _uid(), "kind": caller_kind, "caller": caller_id, "day": day, "now": now},
)
counted = db.executable.execute(
text(
"UPDATE node_caller_daily SET requests = COALESCE(requests, 0) + 1, "
"updated_at = :now WHERE caller_kind = :kind AND caller_id = :caller "
"AND day = :day AND deleted_at IS NULL AND COALESCE(requests, 0) < :limit"
),
{"now": now, "kind": caller_kind, "caller": caller_id, "day": day, "limit": int(limit)},
).rowcount
return counted == 1
def caller_requests_today(caller_kind: str, caller_id: str) -> int:
row = get_table("node_caller_daily").find_one(
caller_kind=caller_kind, caller_id=caller_id, day=_day(), deleted_at=None
)
return int((row or {}).get("requests") or 0)
def get_settings_by_users(user_uids) -> dict[str, dict]:
found: dict[str, dict] = {}
for batch in _batches(user_uids):
placeholders, params = _in_clause(batch)
for row in db.query(
f"SELECT * FROM node_settings WHERE user_uid IN ({placeholders}) "
"AND deleted_at IS NULL",
**params,
):
found[row["user_uid"]] = normalise_settings(dict(row))
return found
def donation_days(user_uid: str) -> int:
rows = db.query(
"SELECT COUNT(DISTINCT day) AS days FROM node_usage_daily "
"WHERE user_uid = :user AND COALESCE(requests_ok, 0) > 0 AND deleted_at IS NULL",
user=user_uid,
)
return int(next(iter(rows))["days"] or 0)
def public_stats() -> dict:
row = next(
iter(
db.query(
"SELECT COALESCE(SUM(requests_ok), 0) AS requests, "
"COALESCE(SUM(bytes_total), 0) AS bytes, COUNT(*) AS donors "
"FROM node_donors WHERE deleted_at IS NULL AND COALESCE(requests_ok, 0) > 0"
)
)
)
return {
"requests": int(row["requests"] or 0),
"bytes": int(row["bytes"] or 0),
"donors": int(row["donors"] or 0),
}
def top_donors(limit: int = 10) -> list[dict]:
limit = max(1, min(int(limit), 50))
rows = db.query(
"SELECT d.user_uid AS user_uid, d.requests_ok AS requests_ok, d.bytes_total AS bytes_total "
"FROM node_donors d JOIN node_settings s ON s.user_uid = d.user_uid "
"WHERE d.deleted_at IS NULL AND s.deleted_at IS NULL AND COALESCE(s.public_display, 0) = 1 "
"AND COALESCE(d.requests_ok, 0) > 0 ORDER BY d.bytes_total DESC LIMIT :limit",
limit=limit,
)
return [dict(row) for row in rows]
ADMIN_LIST_LIMIT = 100
ADMIN_BREAKDOWN_LIMIT = 50
DEVICE_BREAKDOWN_COLUMNS = ("status", "kind", "asn_class", "country")
REQUEST_STATUSES = ("queued", "assigned", "done", "failed", "expired")
def device_breakdown(column: str) -> dict[str, int]:
if column not in DEVICE_BREAKDOWN_COLUMNS:
raise ValueError("Unknown device breakdown column")
rows = db.query(
f"SELECT COALESCE(NULLIF({column}, ''), 'unknown') AS bucket, COUNT(*) AS total "
"FROM node_devices WHERE deleted_at IS NULL "
"GROUP BY bucket ORDER BY total DESC, bucket LIMIT :n",
n=ADMIN_BREAKDOWN_LIMIT,
)
return {row["bucket"]: int(row["total"]) for row in rows}
def request_status_counts() -> dict[str, int]:
counts = {status: 0 for status in REQUEST_STATUSES}
for row in db.query("SELECT status, COUNT(*) AS total FROM node_requests GROUP BY status"):
counts[row["status"]] = int(row["total"])
return counts
def usage_day_totals(moment: datetime | None = None) -> dict[str, int]:
row = next(
iter(
db.query(
"SELECT COALESCE(SUM(requests_ok), 0) AS requests_ok, "
"COALESCE(SUM(requests_failed), 0) AS requests_failed, "
"COALESCE(SUM(bytes_up), 0) AS bytes_up, "
"COALESCE(SUM(bytes_down), 0) AS bytes_down, "
"COALESCE(SUM(cache_hits_served), 0) AS cache_hits_served, "
"COALESCE(SUM(integrity_failures), 0) AS integrity_failures "
"FROM node_usage_daily WHERE day = :day AND deleted_at IS NULL",
day=_day(moment),
)
)
)
return {key: int(value or 0) for key, value in dict(row).items()}
def recent_devices(limit: int = ADMIN_LIST_LIMIT) -> list[dict]:
limit = max(1, min(int(limit), ADMIN_LIST_LIMIT))
return [
dict(row)
for row in db.query(
"SELECT * FROM node_devices WHERE deleted_at IS NULL "
"ORDER BY COALESCE(NULLIF(last_seen_at, ''), created_at) DESC, id DESC LIMIT :n",
n=limit,
)
]
+139
View File
@@ -1854,6 +1854,8 @@ def init_db():
_ensure_moderation_tables()
_ensure_skills_table()
_ensure_nodes_tables()
_ensure_sync_tables()
_ensure_community_tables()
for table in db.tables:
_uid_index(db, table)
@@ -2014,6 +2016,8 @@ def init_db():
"nodes_daily_quota_guest": str(NODES_SPEC.limits.default_daily_requests_guest),
"nodes_cache_ttl_seconds": str(NODES_SPEC.limits.cache_ttl_seconds),
"nodes_social_via_nodes": "0",
"sync_enabled": "0",
"sync_origins": "",
}
for key, value in operational_defaults.items():
existing = db["site_settings"].find_one(key=key)
@@ -2285,11 +2289,24 @@ NODES_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
("result_status", 0),
("result_bytes", 0),
("result_body", ""),
("result_headers", ""),
("truncated", 0),
("error", ""),
("created_at", ""),
("assigned_at", ""),
("finished_at", ""),
),
"node_caller_daily": (
("uid", ""),
("caller_kind", ""),
("caller_id", ""),
("day", ""),
("requests", 0),
("created_at", ""),
("updated_at", ""),
("deleted_at", ""),
("deleted_by", ""),
),
"node_events": (
("uid", ""),
("device_uid", ""),
@@ -2343,6 +2360,14 @@ def _ensure_nodes_tables() -> None:
where="status IN ('queued','assigned')",
)
_index(db, "node_requests", "idx_node_requests_created", ["created_at"])
_index(
db,
"node_caller_daily",
"idx_node_caller_day",
["caller_kind", "caller_id", "day"],
unique=True,
)
_index(db, "node_caller_daily", "idx_node_caller_day_only", ["day"])
_index(db, "node_events", "idx_node_events_created", ["created_at"])
_index(db, "node_events", "idx_node_events_device", ["device_uid", "created_at"])
_index(
@@ -2354,6 +2379,120 @@ def _ensure_nodes_tables() -> None:
)
SYNC_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
"sync_links": (
("uid", ""),
("origin", ""),
("kind", ""),
("origin_uid", ""),
("local_uid", ""),
("fingerprint", ""),
("origin_updated_at", ""),
("first_synced_at", ""),
("synced_at", ""),
),
"sync_identities": (
("uid", ""),
("origin", ""),
("source_name", ""),
("source_key", ""),
("our_name", ""),
("our_key", ""),
("style_fingerprint", ""),
("style_samples", ""),
("ai_generated", 0),
("created_at", ""),
),
}
def _ensure_sync_tables() -> None:
for table_name, columns in SYNC_COLUMNS.items():
table = get_table(table_name)
for column, example in columns:
if not table.has_column(column):
table.create_column_by_example(column, example)
_index(
db,
"sync_links",
"idx_sync_links_origin",
["origin", "kind", "origin_uid"],
unique=True,
)
_index(db, "sync_links", "idx_sync_links_local", ["kind", "local_uid"])
_index(
db,
"sync_identities",
"idx_sync_identities_source",
["origin", "source_key"],
unique=True,
)
_index(db, "sync_identities", "idx_sync_identities_our", ["our_key"], unique=True)
COMMUNITY_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
"communities": (
("uid", ""),
("slug", ""),
("name", ""),
("origin", ""),
("url", ""),
("query", ""),
("status", ""),
("description", ""),
("rules", ""),
("language", ""),
("favicon", ""),
("user_count", 0),
("schema", ""),
("watermark", ""),
("enabled", 1),
("interval_seconds", 0),
("next_run_at", ""),
("last_run_at", ""),
("last_status", ""),
("last_error", ""),
("run_count", 0),
("failure_count", 0),
("totals", ""),
("learned_at", ""),
("created_at", ""),
("created_by", ""),
("login_username", ""),
("login_email", ""),
("login_password", ""),
("sync_options", ""),
("claimed_at", ""),
("deleted_at", ""),
("deleted_by", ""),
),
"community_runs": (
("uid", ""),
("community_uid", ""),
("started_at", ""),
("finished_at", ""),
("status", ""),
("dry_run", 0),
("trigger", ""),
("collected", ""),
("pushed", ""),
("errors", ""),
("log", ""),
),
}
def _ensure_community_tables() -> None:
for table_name, columns in COMMUNITY_COLUMNS.items():
table = get_table(table_name)
for column, example in columns:
if not table.has_column(column):
table.create_column_by_example(column, example)
_index(db, "communities", "idx_communities_slug", ["slug"], unique=True, where="deleted_at IS NULL")
_index(db, "communities", "idx_communities_due", ["status", "enabled", "next_run_at"])
_index(db, "community_runs", "idx_community_runs_community", ["community_uid", "started_at"])
def _ensure_moderation_tables() -> None:
for table_name, columns in MODERATION_COLUMNS.items():
table = get_table(table_name)
+2
View File
@@ -18,11 +18,13 @@ SOFT_DELETE_TABLES = [
"bookmarks",
"notes",
"skills",
"communities",
"node_devices",
"node_settings",
"node_usage_daily",
"node_donors",
"node_flags",
"node_caller_daily",
"follows",
"poll_votes",
"polls",
+131
View File
@@ -0,0 +1,131 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from sqlalchemy import text
from devplacepy.utils.text import generate_uid
from .core import _now_iso, db, get_table
def get_link(origin: str, kind: str, origin_uid: str) -> dict | None:
return get_table("sync_links").find_one(
origin=origin, kind=kind, origin_uid=origin_uid
)
def get_links_by_origin_uids(
origin: str, kind: str, origin_uids: list[str]
) -> dict[str, dict]:
uids = [uid for uid in dict.fromkeys(origin_uids) if uid]
if not uids:
return {}
table = get_table("sync_links")
column = table.table.columns.origin_uid
rows = table.find(column.in_(uids), origin=origin, kind=kind)
return {row["origin_uid"]: row for row in rows}
def claim_link(
origin: str,
kind: str,
origin_uid: str,
local_uid: str,
fingerprint: str,
origin_updated_at: str,
) -> bool:
now = _now_iso()
with db:
result = db.executable.execute(
text(
"INSERT OR IGNORE INTO sync_links (uid, origin, kind, origin_uid, "
"local_uid, fingerprint, origin_updated_at, first_synced_at, synced_at) "
"VALUES (:uid, :origin, :kind, :origin_uid, :local_uid, :fingerprint, "
":origin_updated_at, :now, :now)"
),
{
"uid": generate_uid(),
"origin": origin,
"kind": kind,
"origin_uid": origin_uid,
"local_uid": local_uid,
"fingerprint": fingerprint,
"origin_updated_at": origin_updated_at,
"now": now,
},
)
return result.rowcount == 1
def release_link(origin: str, kind: str, origin_uid: str, local_uid: str) -> None:
get_table("sync_links").delete(
origin=origin, kind=kind, origin_uid=origin_uid, local_uid=local_uid
)
def advance_link(link_uid: str, fingerprint: str, origin_updated_at: str) -> None:
get_table("sync_links").update(
{
"uid": link_uid,
"fingerprint": fingerprint,
"origin_updated_at": origin_updated_at,
"synced_at": _now_iso(),
},
["uid"],
)
def username_taken(username: str) -> bool:
return get_table("users").find_one(username=username) is not None
def count_links(origin: str) -> int:
return get_table("sync_links").count(origin=origin)
def identity_map(origin: str) -> dict[str, dict]:
return {row["source_key"]: row for row in get_table("sync_identities").find(origin=origin)}
def identity_by_source(origin: str, source_key: str) -> dict | None:
return get_table("sync_identities").find_one(origin=origin, source_key=source_key)
def identity_name_taken(our_key: str) -> bool:
return get_table("sync_identities").find_one(our_key=our_key) is not None
def claim_identity(
origin: str,
source_name: str,
source_key: str,
our_name: str,
style_fingerprint: str,
ai_generated: bool,
) -> bool:
with db:
result = db.executable.execute(
text(
"INSERT OR IGNORE INTO sync_identities (uid, origin, source_name, source_key, "
"our_name, our_key, style_fingerprint, style_samples, ai_generated, created_at) "
"VALUES (:uid, :origin, :source_name, :source_key, :our_name, :our_key, "
":style_fingerprint, '[]', :ai_generated, :now)"
),
{
"uid": generate_uid(),
"origin": origin,
"source_name": source_name,
"source_key": source_key,
"our_name": our_name,
"our_key": our_name.lower(),
"style_fingerprint": style_fingerprint,
"ai_generated": int(ai_generated),
"now": _now_iso(),
},
)
return result.rowcount == 1
def save_style_samples(identity_uid: str, samples_json: str) -> None:
get_table("sync_identities").update({"uid": identity_uid, "style_samples": samples_json}, ["uid"])