Code Farm: per-plot auto-replant pause and crop-lock check; community sync and node work
DevPlace CI / test (push) Failing after 44m45s

Continuous Delivery could never be turned off once a crop was planted. Add a
one-shot per-plot "stop after harvest" flag (POST /game/plot/stop, Devii tool
game_plot_stop, game_plots.stop_after_harvest), cleared by the next plant,
autoreplant claim or Refactor. Autoreplant now also respects crop locks through
the shared _crop_lock predicate. Plan and measurements in happyplanting.md.

Also includes the pending community sync, node network and related work in the
tree; the full suite passed (4999) with all of it.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-17 02:23:43 +02:00
co-authored by Claude Sonnet 5.5
parent ce899fc5e1
commit 02ab747cd0
203 changed files with 11624 additions and 301 deletions
+3
View File
@@ -2,6 +2,7 @@
import dataset
import logging
import warnings
from pathlib import Path
from sqlalchemy import or_
from sqlalchemy.pool import NullPool
@@ -20,6 +21,8 @@ from devplacepy.config import (
logger = logging.getLogger(__name__)
warnings.filterwarnings("ignore", message="Skipped unsupported reflection of expression-based index")
ensure_data_dirs()
if DATABASE_URL.startswith("sqlite:///"):
_db_file = DATABASE_URL[len("sqlite:///") :]
+180 -5
View File
@@ -24,6 +24,7 @@ LABEL_MAX = 60
EVENT_DETAIL_MAX = 500
MEGABYTE = 1024 * 1024
BATCH_SIZE = 500
COUNTER_ROW_LIMIT = 500
HOURS_PATTERN = re.compile(r"^([01]\d|2[0-3]):[0-5]\d$")
PLATFORMS = tuple(sorted({item.adapter for item in SPEC.destinations}))
FLAG_OPEN = "open"
@@ -714,14 +715,18 @@ def enqueue_request(
method: str,
adapter: str,
mode: str,
*,
shadow_of: str = "",
exclude_device: str = "",
) -> str:
uid = _uid()
created = _execute(
"INSERT INTO node_requests (uid, caller_kind, caller_id, url, method, adapter, "
"mode, status, device_uid, result_status, result_bytes, result_body, "
"result_headers, truncated, error, created_at, assigned_at, finished_at) "
"result_headers, truncated, error, device_asn_class, shadow_of, exclude_device, "
"created_at, assigned_at, finished_at) "
"SELECT :uid, :kind, :caller, :url, :method, :adapter, :mode, 'queued', '', 0, 0, "
"'', '', 0, '', :now, '', '' "
"'', '', 0, '', '', :shadow_of, :exclude_device, :now, '', '' "
"WHERE (SELECT COUNT(*) FROM node_requests WHERE status IN ('queued','assigned')) "
"< :max_depth",
{
@@ -732,6 +737,8 @@ def enqueue_request(
"method": method,
"adapter": adapter,
"mode": mode,
"shadow_of": shadow_of,
"exclude_device": exclude_device,
"now": _now_iso(),
"max_depth": LIMITS.max_queue_depth,
},
@@ -766,12 +773,18 @@ def count_open_requests() -> int:
)["c"]
def claim_request(request_uid: str, device_uid: str) -> bool:
def claim_request(request_uid: str, device_uid: str, asn_class: str = "") -> bool:
return (
_execute(
"UPDATE node_requests SET status = 'assigned', device_uid = :device, "
"assigned_at = :now WHERE uid = :uid AND status = 'queued'",
{"device": device_uid, "now": _now_iso(), "uid": request_uid},
"device_asn_class = :asn_class, assigned_at = :now "
"WHERE uid = :uid AND status = 'queued'",
{
"device": device_uid,
"asn_class": asn_class,
"now": _now_iso(),
"uid": request_uid,
},
)
== 1
)
@@ -989,3 +1002,165 @@ def recent_devices(limit: int = ADMIN_LIST_LIMIT) -> list[dict]:
n=limit,
)
]
def set_class_override(device_uid: str, value: str) -> bool:
return (
_execute(
"UPDATE node_devices SET class_override = :value, updated_at = :now "
"WHERE uid = :uid AND deleted_at IS NULL",
{"value": value, "now": _now_iso(), "uid": device_uid},
)
== 1
)
def touch_devices(device_uids, moment: datetime | None = None) -> int:
stamp = (moment or datetime.now(timezone.utc)).isoformat()
touched = 0
for chunk in _batches(device_uids):
placeholders, params = _in_clause(chunk)
touched += _execute(
f"UPDATE node_devices SET last_seen_at = :now WHERE uid IN ({placeholders})",
{"now": stamp, **params},
)
return touched
def flush_counters(deltas: dict[str, int]) -> int:
rows = {name: int(value) for name, value in deltas.items() if int(value) > 0}
if not rows:
return 0
now = _now_iso()
with db:
for name, value in rows.items():
db.executable.execute(
text(
"INSERT INTO node_counters (name, value, updated_at) VALUES (:name, :value, :now) "
"ON CONFLICT(name) DO UPDATE SET value = COALESCE(value, 0) + excluded.value, "
"updated_at = excluded.updated_at"
),
{"name": name, "value": value, "now": now},
)
return len(rows)
def lifetime_counters(prefix: str = "") -> dict[str, int]:
rows = db.query(
"SELECT name, COALESCE(value, 0) AS value FROM node_counters "
"WHERE name LIKE :prefix ESCAPE '\\' ORDER BY name LIMIT :n",
prefix=prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%",
n=COUNTER_ROW_LIMIT,
)
return {row["name"]: int(row["value"]) for row in rows}
def save_circuits(rows: list[dict]) -> int:
now = _now_iso()
with db:
db.executable.execute(text("DELETE FROM node_circuits"))
for row in rows[: LIMITS.circuit_snapshot_max]:
db.executable.execute(
text(
"INSERT INTO node_circuits (key, failures, state, opened_at, updated_at) "
"VALUES (:key, :failures, :state, :opened_at, :now)"
),
{
"key": row["key"],
"failures": int(row["failures"]),
"state": row["state"],
"opened_at": float(row["opened_at"]),
"now": now,
},
)
return min(len(rows), LIMITS.circuit_snapshot_max)
def load_circuits() -> list[dict]:
return [
dict(row)
for row in db.query(
"SELECT key, failures, state, opened_at FROM node_circuits LIMIT :n",
n=LIMITS.circuit_snapshot_max,
)
]
def cache_get(key: str, moment: datetime | None = None) -> dict | None:
now = (moment or datetime.now(timezone.utc)).isoformat()
rows = list(
db.query(
"SELECT status, headers, body, stored_at FROM node_cache "
"WHERE key = :key AND expires_at > :now LIMIT 1",
key=key,
now=now,
)
)
return dict(rows[0]) if rows else None
def cache_put(
key: str,
method: str,
url: str,
status: int,
headers: str,
body: str,
size: int,
ttl_seconds: int,
moment: datetime | None = None,
) -> bool:
if ttl_seconds <= 0:
return False
now = moment or datetime.now(timezone.utc)
return (
_execute(
"INSERT INTO node_cache (key, method, url, status, headers, body, size, stored_at, expires_at) "
"VALUES (:key, :method, :url, :status, :headers, :body, :size, :stored, :expires) "
"ON CONFLICT(key) DO UPDATE SET status = excluded.status, headers = excluded.headers, "
"body = excluded.body, size = excluded.size, stored_at = excluded.stored_at, "
"expires_at = excluded.expires_at",
{
"key": key,
"method": method,
"url": url[: LIMITS.url_max_chars],
"status": int(status),
"headers": headers,
"body": body,
"size": int(size),
"stored": now.isoformat(),
"expires": (now + timedelta(seconds=ttl_seconds)).isoformat(),
},
)
== 1
)
def cache_prune(max_bytes: int, moment: datetime | None = None) -> dict[str, int]:
now = (moment or datetime.now(timezone.utc)).isoformat()
expired = _execute("DELETE FROM node_cache WHERE expires_at <= :now", {"now": now})
total = next(iter(db.query("SELECT COALESCE(SUM(size), 0) AS total FROM node_cache")))["total"]
evicted = 0
while total > max_bytes:
rows = list(
db.query("SELECT key, size FROM node_cache ORDER BY stored_at, id LIMIT :n", n=BATCH_SIZE)
)
if not rows:
break
for row in rows:
if total <= max_bytes:
break
evicted += _execute("DELETE FROM node_cache WHERE key = :key", {"key": row["key"]})
total -= int(row["size"] or 0)
return {"expired": expired, "evicted": evicted, "bytes": max(0, int(total))}
def cache_clear() -> int:
return _execute("DELETE FROM node_cache", {})
def cache_stats() -> dict[str, int]:
row = next(
iter(db.query("SELECT COUNT(*) AS entries, COALESCE(SUM(size), 0) AS bytes FROM node_cache"))
)
return {"entries": int(row["entries"]), "bytes": int(row["bytes"])}
+97
View File
@@ -1506,6 +1506,7 @@ def init_db():
("ready_at", ""),
("watered_by", "[]"),
("raided_fraction", 0.0),
("stop_after_harvest", 0),
("created_at", ""),
("updated_at", ""),
):
@@ -1863,6 +1864,10 @@ def init_db():
for table in SOFT_DELETE_TABLES:
ensure_soft_delete_columns(table)
from .sync import ensure_export_indexes
ensure_export_indexes()
_index(
db,
"comments",
@@ -2222,6 +2227,7 @@ NODES_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
("country", ""),
("asn", ""),
("asn_class", ""),
("class_override", ""),
("ip_hash", ""),
("last_seen_at", ""),
("created_at", ""),
@@ -2292,10 +2298,36 @@ NODES_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
("result_headers", ""),
("truncated", 0),
("error", ""),
("device_asn_class", ""),
("shadow_of", ""),
("exclude_device", ""),
("created_at", ""),
("assigned_at", ""),
("finished_at", ""),
),
"node_counters": (
("name", ""),
("value", 0),
("updated_at", ""),
),
"node_circuits": (
("key", ""),
("failures", 0),
("state", ""),
("opened_at", 0.0),
("updated_at", ""),
),
"node_cache": (
("key", ""),
("method", ""),
("url", ""),
("status", 0),
("headers", ""),
("body", ""),
("size", 0),
("stored_at", ""),
("expires_at", ""),
),
"node_caller_daily": (
("uid", ""),
("caller_kind", ""),
@@ -2360,6 +2392,11 @@ def _ensure_nodes_tables() -> None:
where="status IN ('queued','assigned')",
)
_index(db, "node_requests", "idx_node_requests_created", ["created_at"])
_index(db, "node_requests", "idx_node_requests_shadow", ["shadow_of"], where="shadow_of != ''")
_index(db, "node_counters", "idx_node_counters_name", ["name"], unique=True)
_index(db, "node_circuits", "idx_node_circuits_key", ["key"], unique=True)
_index(db, "node_cache", "idx_node_cache_key", ["key"], unique=True)
_index(db, "node_cache", "idx_node_cache_expires", ["expires_at"])
_index(
db,
"node_caller_daily",
@@ -2390,6 +2427,28 @@ SYNC_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
("origin_updated_at", ""),
("first_synced_at", ""),
("synced_at", ""),
("ack_at", ""),
),
"sync_pending": (
("uid", ""),
("origin", ""),
("kind", ""),
("origin_uid", ""),
("missing_kind", ""),
("missing_uid", ""),
("record", ""),
("attempts", 0),
("created_at", ""),
("expires_at", ""),
),
"sync_candidates": (
("uid", ""),
("origin", ""),
("post_uid", ""),
("record", ""),
("comment_count", 0),
("created_at", ""),
("expires_at", ""),
),
"sync_identities": (
("uid", ""),
@@ -2420,6 +2479,11 @@ def _ensure_sync_tables() -> None:
unique=True,
)
_index(db, "sync_links", "idx_sync_links_local", ["kind", "local_uid"])
_index(db, "sync_pending", "idx_sync_pending_record", ["origin", "kind", "origin_uid"], unique=True)
_index(db, "sync_pending", "idx_sync_pending_missing", ["origin", "missing_kind", "missing_uid"])
_index(db, "sync_pending", "idx_sync_pending_expires", ["expires_at"])
_index(db, "sync_candidates", "idx_sync_candidates_post", ["origin", "post_uid"], unique=True)
_index(db, "sync_candidates", "idx_sync_candidates_expires", ["expires_at"])
_index(
db,
"sync_identities",
@@ -2461,6 +2525,8 @@ COMMUNITY_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
("login_username", ""),
("login_email", ""),
("login_password", ""),
("login_totp_secret", ""),
("schema_previous", ""),
("sync_options", ""),
("claimed_at", ""),
("deleted_at", ""),
@@ -2516,6 +2582,37 @@ def _ensure_community_tables() -> None:
_index(db, "community_run_items", "idx_community_items_run_outcome", ["run_uid", "outcome", "kind", "seq"])
_index(db, "community_run_items", "idx_community_items_community_rank", ["community_uid", "rank", "created_at DESC", "seq"])
_index(db, "community_run_items", "idx_community_items_community_outcome", ["community_uid", "outcome", "created_at DESC", "seq"])
_index(db, "community_run_items", "idx_community_items_run_seq", ["run_uid", "seq"], unique=True)
_ensure_ledger_search()
LEDGER_SEARCH_TABLE = "community_run_items_fts"
LEDGER_SEARCH_COLUMNS = ("label", "origin_uid", "detail")
def _ensure_ledger_search() -> None:
try:
exists = bool(list(db.query("SELECT name FROM sqlite_master WHERE name = :name", name=LEDGER_SEARCH_TABLE)))
columns = ", ".join(LEDGER_SEARCH_COLUMNS)
new_columns = ", ".join(f"new.{column}" for column in LEDGER_SEARCH_COLUMNS)
old_columns = ", ".join(f"old.{column}" for column in LEDGER_SEARCH_COLUMNS)
with db:
db.query(
f"CREATE VIRTUAL TABLE IF NOT EXISTS {LEDGER_SEARCH_TABLE} USING fts5({columns}, "
"content='community_run_items', content_rowid='id', tokenize='trigram')"
)
db.query(
"CREATE TRIGGER IF NOT EXISTS community_run_items_fts_insert AFTER INSERT ON community_run_items BEGIN "
f"INSERT INTO {LEDGER_SEARCH_TABLE}(rowid, {columns}) VALUES (new.id, {new_columns}); END"
)
db.query(
"CREATE TRIGGER IF NOT EXISTS community_run_items_fts_delete AFTER DELETE ON community_run_items BEGIN "
f"INSERT INTO {LEDGER_SEARCH_TABLE}({LEDGER_SEARCH_TABLE}, rowid, {columns}) VALUES ('delete', old.id, {old_columns}); END"
)
if not exists:
db.query(f"INSERT INTO {LEDGER_SEARCH_TABLE}({LEDGER_SEARCH_TABLE}) VALUES ('rebuild')")
except Exception as exc:
logger.warning("Ledger full-text search is unavailable, the text filter will scan: %s", exc)
def _ensure_moderation_tables() -> None:
+229
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from sqlalchemy import text
from devplacepy.utils.text import generate_uid
@@ -129,3 +131,230 @@ def claim_identity(
def save_style_samples(identity_uid: str, samples_json: str) -> None:
get_table("sync_identities").update({"uid": identity_uid, "style_samples": samples_json}, ["uid"])
PENDING_TABLE = "sync_pending"
def park_pending(
origin: str,
kind: str,
origin_uid: str,
missing_kind: str,
missing_uid: str,
record_json: str,
*,
attempts: int,
ttl_days: int,
cap: int,
) -> bool:
now = datetime.now(timezone.utc)
with db:
result = db.executable.execute(
text(
"INSERT INTO sync_pending (uid, origin, kind, origin_uid, missing_kind, missing_uid, "
"record, attempts, created_at, expires_at) "
"SELECT :uid, :origin, :kind, :origin_uid, :missing_kind, :missing_uid, :record, "
":attempts, :now, :expires "
"WHERE (SELECT COUNT(*) FROM sync_pending WHERE origin = :origin) < :cap "
"ON CONFLICT(origin, kind, origin_uid) DO UPDATE SET "
"missing_kind = excluded.missing_kind, missing_uid = excluded.missing_uid, "
"record = excluded.record, attempts = excluded.attempts, expires_at = excluded.expires_at"
),
{
"uid": generate_uid(),
"origin": origin,
"kind": kind,
"origin_uid": origin_uid,
"missing_kind": missing_kind,
"missing_uid": missing_uid,
"record": record_json,
"attempts": int(attempts),
"now": now.isoformat(),
"expires": (now + timedelta(days=ttl_days)).isoformat(),
"cap": int(cap),
},
)
return result.rowcount >= 1
def pending_exists(origin: str, kind: str, origin_uid: str) -> bool:
return get_table(PENDING_TABLE).find_one(origin=origin, kind=kind, origin_uid=origin_uid) is not None
def ready_pending(limit: int, origin: str = "") -> list[dict]:
clause = "AND p.origin = :origin " if origin else ""
rows = db.query(
"SELECT p.* FROM sync_pending p JOIN sync_links l ON l.origin = p.origin "
"AND l.kind = p.missing_kind AND l.origin_uid = p.missing_uid "
f"WHERE 1 = 1 {clause}ORDER BY p.created_at, p.id LIMIT :n",
n=int(limit),
**({"origin": origin} if origin else {}),
)
return [dict(row) for row in rows]
def delete_pending(uid: str) -> int:
with db:
return db.executable.execute(text("DELETE FROM sync_pending WHERE uid = :uid"), {"uid": uid}).rowcount
def expire_pending(max_attempts: int) -> int:
with db:
return db.executable.execute(
text("DELETE FROM sync_pending WHERE expires_at <= :now OR attempts >= :attempts"),
{"now": _now_iso(), "attempts": int(max_attempts)},
).rowcount
def pending_counts() -> dict[str, int]:
rows = db.query("SELECT origin, COUNT(*) AS n FROM sync_pending GROUP BY origin ORDER BY n DESC LIMIT 100")
return {row["origin"]: int(row["n"]) for row in rows}
STAR_TABLES = {"post": "posts", "project": "projects"}
def recompute_stars(target_type: str, target_uid: str) -> int | None:
table = STAR_TABLES.get(target_type)
if table is None:
return None
with db:
db.executable.execute(
text(
f"UPDATE {table} SET stars = (SELECT COALESCE(SUM(value), 0) FROM votes "
"WHERE target_uid = :uid AND target_type = :type AND deleted_at IS NULL) "
"WHERE uid = :uid"
),
{"uid": target_uid, "type": target_type},
)
row = get_table(table).find_one(uid=target_uid)
return int(row["stars"] or 0) if row else None
EXPORT_CHANGED: dict[str, str] = {
"users": "created_at",
"projects": "COALESCE(deleted_at, updated_at, created_at)",
"posts": "COALESCE(deleted_at, updated_at, created_at)",
"comments": "COALESCE(deleted_at, updated_at, created_at)",
"polls": "COALESCE(deleted_at, created_at)",
"follows": "COALESCE(deleted_at, created_at)",
"attachments": "COALESCE(deleted_at, created_at)",
}
def ensure_export_indexes() -> None:
from .core import _index
for table, expression in EXPORT_CHANGED.items():
if table in db.tables:
_index(db, table, f"idx_{table}_changed", [expression, "uid"])
def deletion_rows(origin: str, kind: str, table: str, after_id: int, limit: int) -> list[dict]:
if table not in db.tables:
return []
soft = db[table].has_column("deleted_at")
columns = "t.deleted_at AS deleted_at, t.deleted_by AS deleted_by" if soft else "NULL AS deleted_at, NULL AS deleted_by"
deleted = "OR (t.deleted_at IS NOT NULL AND COALESCE(t.deleted_by, '') != :actor)" if soft else ""
rows = db.query(
f"SELECT l.id AS link_id, l.kind AS kind, l.origin_uid AS origin_uid, t.uid AS local_uid, {columns} "
f"FROM sync_links l LEFT JOIN {table} t ON t.uid = l.local_uid "
"WHERE l.origin = :origin AND l.kind = :kind AND l.id > :after "
"AND COALESCE(l.ack_at, '') = '' "
f"AND (t.uid IS NULL {deleted}) "
"ORDER BY l.id LIMIT :n",
origin=origin,
kind=kind,
after=int(after_id),
n=int(limit),
**({"actor": "sync"} if soft else {}),
)
return [dict(row) for row in rows]
def acknowledge_links(origin: str, items: list[tuple[str, str]]) -> int:
stamp = _now_iso()
done = 0
with db:
for kind, origin_uid in items:
done += db.executable.execute(
text(
"UPDATE sync_links SET ack_at = :now WHERE origin = :origin AND kind = :kind "
"AND origin_uid = :origin_uid AND COALESCE(ack_at, '') = ''"
),
{"now": stamp, "origin": origin, "kind": kind, "origin_uid": origin_uid},
).rowcount
return done
def rename_identity(identity_uid: str, our_name: str) -> bool:
try:
with db:
result = db.executable.execute(
text("UPDATE sync_identities SET our_name = :name, our_key = :key WHERE uid = :uid"),
{"name": our_name, "key": our_name.lower(), "uid": identity_uid},
)
return result.rowcount == 1
except Exception:
return False
def linked_rows_containing(origin: str, kind: str, table: str, columns: tuple[str, ...], needle: str, limit: int) -> list[dict]:
pattern = "%" + needle.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
matches = " OR ".join(f"t.{column} LIKE :pattern ESCAPE '\\'" for column in columns)
selected = ", ".join(f"t.{column} AS {column}" for column in columns)
rows = db.query(
f"SELECT t.uid AS uid, {selected} FROM sync_links l JOIN {table} t ON t.uid = l.local_uid "
f"WHERE l.origin = :origin AND l.kind = :kind AND ({matches}) ORDER BY l.id LIMIT :limit",
origin=origin,
kind=kind,
pattern=pattern,
limit=int(limit),
)
return [dict(row) for row in rows]
def load_candidates(origin: str) -> list[dict]:
expire_candidates()
return [dict(row) for row in get_table("sync_candidates").find(origin=origin, order_by=["id"])]
def save_candidate(origin: str, post_uid: str, record_json: str, comment_count: int, *, ttl_days: int, cap: int) -> bool:
now = datetime.now(timezone.utc)
with db:
result = db.executable.execute(
text(
"INSERT INTO sync_candidates (uid, origin, post_uid, record, comment_count, created_at, expires_at) "
"SELECT :uid, :origin, :post_uid, :record, :count, :now, :expires "
"WHERE (SELECT COUNT(*) FROM sync_candidates WHERE origin = :origin) < :cap "
"ON CONFLICT(origin, post_uid) DO UPDATE SET record = excluded.record, "
"comment_count = excluded.comment_count"
),
{
"uid": generate_uid(),
"origin": origin,
"post_uid": post_uid,
"record": record_json,
"count": int(comment_count),
"now": now.isoformat(),
"expires": (now + timedelta(days=ttl_days)).isoformat(),
"cap": int(cap),
},
)
return result.rowcount >= 1
def delete_candidate(origin: str, post_uid: str) -> int:
with db:
return db.executable.execute(
text("DELETE FROM sync_candidates WHERE origin = :origin AND post_uid = :post_uid"),
{"origin": origin, "post_uid": post_uid},
).rowcount
def expire_candidates() -> int:
with db:
return db.executable.execute(
text("DELETE FROM sync_candidates WHERE expires_at <= :now"), {"now": _now_iso()}
).rowcount