Files
devplacepy/devplacepy/services/community/runner.py
T
retoorandClaude Sonnet 5.5 02ab747cd0 Code Farm: per-plot auto-replant pause and crop-lock check; community sync and node work
Continuous Delivery could never be turned off once a crop was planted. Add a
one-shot per-plot "stop after harvest" flag (POST /game/plot/stop, Devii tool
game_plot_stop, game_plots.stop_after_harvest), cleared by the next plant,
autoreplant claim or Refactor. Autoreplant now also respects crop locks through
the shared _crop_lock predicate. Plan and measurements in happyplanting.md.

Also includes the pending community sync, node network and related work in the
tree; the full suite passed (4999) with all of it.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-17 02:23:43 +02:00

366 lines
18 KiB
Python

# retoor <retoor@molodetz.nl>
from __future__ import annotations
import asyncio
import logging
from dataclasses import dataclass, field
from devplacepy.services.sync.aliaser import AliasRefused, Aliaser
from devplacepy.services.sync.healing import AliasHealer
from devplacepy.services.sync.options import SyncOptions, alias_allowed
from devplacepy.services.sync.selection import Selector
from devplacepy.services.community.checkpoint import RunCheckpoint
from devplacepy.services.community.discoverer import CommunityDiscoverer, Discovery
from devplacepy.services.community.fetcher import Fetcher
from devplacepy.services.community.files import CommunityFiles, files as default_files
from devplacepy.services.community.ledger import Ledger, tally_of, verdict
from devplacepy.services.community.ledger_store import LedgerStore, ledger_store as default_ledger_store
from devplacepy.services.community.logbook import LogBook
from devplacepy.services.community.pusher import PushReport, SyncPusher
from devplacepy.services.community.runtime import Collector, LoginRequired
from devplacepy.services.community.schemacheck import sample_limits
from devplacepy.services.community.spec import (
LIMITS,
RUN_DRY,
RUN_FAILED,
RUN_OK,
RUN_PARTIAL,
RUN_SKIPPED,
STATUS_LOGIN,
STATUS_READY,
)
from devplacepy.services.community.store import CommunityStore, iso, now, store as default_store
from devplacepy.services.sync.gate import refusal
from devplacepy.services.sync.spec import SKIP_ALIAS_ENGAGEMENT, SKIP_PARENT
logger = logging.getLogger("community.run")
TRIGGER_CLI = "cli"
TRIGGER_ADMIN = "admin"
TRIGGER_SERVICE = "service"
KINDS = ("users", "posts", "comments")
COLLECTED_KINDS = (*KINDS, "votes", "reactions", "attachments")
ENGAGEMENT_KINDS = ("votes", "reactions", "attachments")
def options_alias_active(row: dict) -> bool:
return SyncOptions.from_row(row).alias
@dataclass
class SchemaVerdict:
ok: bool
posts: int
comments: int
problems: list
@dataclass
class RunReport:
status: str
dry_run: bool
collected: dict = field(default_factory=dict)
pushed: dict = field(default_factory=dict)
errors: list = field(default_factory=list)
log: str = ""
run_uid: str = ""
watermark: dict = field(default_factory=dict)
ledger: Ledger = field(default_factory=Ledger)
live: PushReport = field(default_factory=lambda: PushReport(dry_run=False))
class CommunityRunner:
def __init__(self, fetcher_factory=Fetcher, store: CommunityStore | None = None, files: CommunityFiles | None = None, sleep=None, ledgers: LedgerStore | None = None):
self.fetcher_factory = fetcher_factory
self.sleep = sleep
self.store = store or default_store
self.files = files or default_files
self.ledgers = ledgers or default_ledger_store
async def discover(
self,
query: str,
created_by: str,
*,
dry_run: bool = False,
relearn_slug: str | None = None,
credentials: dict | None = None,
) -> tuple[Discovery, dict | None, str]:
with LogBook() as book:
logger.info("discover%s: %r", " (dry run, nothing is saved)" if dry_run else "", query)
existing = self.store.get(relearn_slug) if relearn_slug else None
if existing and not dry_run:
self.store.claim(existing["uid"])
secrets = credentials or (self.store.credentials(existing) if existing else {})
try:
async with self.fetcher_factory() as fetcher:
result = await asyncio.wait_for(
CommunityDiscoverer(fetcher, credentials=secrets).discover(query), timeout=LIMITS.run_timeout_seconds
)
except asyncio.TimeoutError:
result = Discovery(query=query, error=f"discovery exceeded {LIMITS.run_timeout_seconds}s")
logger.error("discover: %s", result.error)
except Exception as exc:
logger.exception("discover: unexpected failure")
result = Discovery(query=query, error=f"{type(exc).__name__}: {exc}")
row = None
if not dry_run and (result.slug or relearn_slug):
row = self.persist(result, query, created_by, relearn_slug, credentials)
logger.info("discover: saved community %s with status %s", result.slug, result.status)
elif dry_run:
logger.info("discover: dry run, would save %s with status %s (%s feed(s))", result.slug, result.status, len(result.schema.get("feeds", [])))
return result, row, book.text()
def persist(self, result: Discovery, query: str, created_by: str, relearn_slug: str | None, credentials: dict | None = None) -> dict:
slug = relearn_slug or result.slug
existing = self.store.get(slug)
learned = result.status in (STATUS_READY, STATUS_LOGIN)
if existing and existing["status"] == STATUS_READY and not learned:
logger.warning("discover: relearn failed, keeping the working schema of %s: %s", slug, result.error)
return self.store.update(existing["uid"], last_error=result.error[:1000])
fields = {
"name": result.name or existing and existing.get("name") or slug,
"origin": result.origin,
"url": result.url,
"status": result.status,
"description": result.metadata.get("description", ""),
"rules": result.metadata.get("rules", ""),
"language": result.metadata.get("language", ""),
"favicon": result.metadata.get("favicon", ""),
"user_count": int(result.metadata.get("user_count") or 0),
"schema": result.schema,
"learned_at": iso(now()) if learned else "",
"last_error": result.error,
"failure_count": 0,
}
if credentials and any(credentials.values()):
fields.update(
login_username=credentials.get("username", ""),
login_email=credentials.get("email", ""),
login_password=credentials.get("password", ""),
login_totp_secret=credentials.get("totp", ""),
)
if existing:
row = self.store.update(existing["uid"], **fields, enabled=1 if result.status == STATUS_READY else 0)
else:
row = self.store.create(slug, query, created_by, **fields)
if result.schema.get("feeds"):
self.files.write(slug, result.schema)
return row
async def run(self, row: dict, *, dry_run: bool, trigger: str) -> RunReport:
self.store.reap_interrupted()
run_uid = self.store.start_run(row["uid"], trigger, dry_run)
report = RunReport(status=RUN_FAILED, dry_run=dry_run, run_uid=run_uid)
report.live.dry_run = dry_run
with LogBook() as book:
checkpoint = RunCheckpoint(self.store, self.ledgers, run_uid, row["uid"], report, report.live, book)
ticker = asyncio.create_task(checkpoint.loop())
mode = "DRY RUN" if dry_run else "LIVE"
logger.info("run[%s]: %s (%s) trigger=%s", mode, row["name"], row["origin"], trigger)
try:
await asyncio.wait_for(self.execute(row, report), timeout=LIMITS.run_timeout_seconds)
except asyncio.TimeoutError:
report.errors.append(f"run exceeded {LIMITS.run_timeout_seconds}s")
report.ledger.add_error(report.errors[-1])
except LoginRequired as exc:
report.errors.append(str(exc))
report.ledger.add_error(str(exc))
if not dry_run:
self.store.update(row["uid"], status=STATUS_LOGIN, enabled=0, last_error=str(exc))
except Exception as exc:
logger.exception("run[%s]: unexpected failure", mode)
report.errors.append(f"{type(exc).__name__}: {exc}")
report.ledger.add_error(report.errors[-1])
for error in report.errors:
logger.error("run[%s]: %s", mode, error)
logger.info("run[%s]: finished with status %s", mode, report.status)
ticker.cancel()
await asyncio.gather(ticker, return_exceptions=True)
report.log = book.text()
if not dry_run:
self.settle(row, report)
tally = tally_of(report.pushed, report.ledger)
outcome = verdict(report.status, report.dry_run, tally)
stored = checkpoint.saved + self.ledgers.append(run_uid, row["uid"], report.ledger, checkpoint.saved)
self.store.finish_run(
run_uid,
report.status,
collected=report.collected,
pushed=report.pushed,
errors=report.errors,
log=report.log,
tally=tally,
verdict=outcome["level"],
headline=outcome["headline"],
items_stored=stored,
items_dropped=report.ledger.dropped,
)
self.store.prune_runs(row["uid"])
return report
async def verify_schema(self, row: dict, schema: dict) -> "SchemaVerdict":
sample = {**schema, "limits": sample_limits(schema)}
try:
async with self.fetcher_factory() as fetcher:
result = await asyncio.wait_for(
Collector(fetcher, sample, None, sleep=self.sleep, credentials=self.store.credentials(row)).run(),
timeout=LIMITS.run_timeout_seconds,
)
except asyncio.TimeoutError:
return SchemaVerdict(False, 0, 0, [f"the verification run exceeded {LIMITS.run_timeout_seconds}s"])
except LoginRequired as exc:
return SchemaVerdict(False, 0, 0, [str(exc)])
except Exception as exc:
logger.exception("schema verification crashed")
return SchemaVerdict(False, 0, 0, [f"{type(exc).__name__}: {exc}"])
posts, comments = len(result["records"]["posts"]), len(result["records"]["comments"])
problems = list(result.get("errors", []))
if not posts:
problems.append("the schema collected no posts from the live source")
return SchemaVerdict(bool(result["ok"]) and posts > 0, posts, comments, problems)
async def execute(self, row: dict, report: RunReport) -> None:
refused = refusal(row["origin"])
if refused and not report.dry_run:
report.errors.append(refused)
report.status = RUN_SKIPPED
return
if refused:
logger.warning("run[DRY RUN]: %s; continuing because nothing is written", refused)
module = self.files.load(row["slug"], row["schema"])
options = SyncOptions.from_row(row)
async with self.fetcher_factory() as fetcher:
result = await module.collect_all(
fetcher,
row.get("watermark") or None,
sleep=self.sleep,
credentials=self.store.credentials(row),
options={"attachments": options.collect_attachments},
)
records = result["records"]
report.collected = {kind: len(records.get(kind, [])) for kind in COLLECTED_KINDS if records.get(kind) or kind in KINDS}
logger.info("run: collected %s", report.collected)
report.errors.extend(result.get("errors", []))
for warning in result.get("warnings", []):
logger.warning("run: %s", warning)
report.ledger.sources.update(result.get("sources", {}))
for notice in result.get("notices", []):
report.ledger.add_notice(notice)
for error in result.get("errors", []):
report.ledger.add_error(error)
if not result["ok"]:
report.errors.append("no feed could be collected")
return
prepared = await self.prepare(row, options, records, report)
if prepared is None:
report.status = RUN_SKIPPED
return
records, skipped, held = prepared
push = await asyncio.to_thread(SyncPusher(row["origin"], dry_run=report.dry_run, ledger=report.ledger, report=report.live).push, records)
if options_alias_active(row) and not report.dry_run:
self.heal(row["origin"])
report.pushed = push.counts | ({"skipped": dict(skipped)} if skipped else {})
report.errors.extend(push.problems[:20])
report.watermark = {} if held else result["state"]
if held:
report.errors.append(f"{held} item(s) held back for a transient reason and will be retried")
logger.warning("run: %d item(s) were skipped for a transient reason, the watermark is held so the next run retries them", held)
failed = push.total("failed")
report.status = RUN_DRY if report.dry_run else (RUN_PARTIAL if report.errors or failed else RUN_OK)
@staticmethod
def heal(origin: str) -> None:
healed = AliasHealer(origin).heal()
if healed.renamed:
logger.info("alias: %d alias(es) followed their clone, %d stored text(s) rewritten", len(healed.renamed), healed.rewritten)
if healed.conflicts:
logger.warning("alias: %d alias(es) could not follow their clone: %s", len(healed.conflicts), ", ".join(healed.conflicts))
async def prepare(self, row: dict, options: SyncOptions, records: dict, report: RunReport):
selector = Selector(row["origin"], options, persist=not report.dry_run)
if options.alias and not report.dry_run:
self.heal(row["origin"])
extras = {kind: records.get(kind, []) for kind in ENGAGEMENT_KINDS}
collected_users = list(records.get("users", []))
records, skipped = selector.select(records)
for kind, record, reason in selector.dropped:
report.ledger.add_dropped_record(kind, record, reason)
held = 0
if options.alias or options.translate_to:
if options.alias and not alias_allowed():
report.errors.append("identity aliasing is switched off: enable community_alias_enabled on the Community sync service")
return None
try:
outcome = await Aliaser(row["origin"], options, language=row.get("language") or "", dry_run=report.dry_run).apply(records)
except AliasRefused as exc:
report.errors.append(str(exc))
return None
records, held = outcome.records, outcome.transient()
for kind, record, reason in outcome.dropped:
report.ledger.add_dropped_record(kind, record, reason)
skipped.update(outcome.skipped)
logger.info("alias: %d substitution(s), %d translated, %d new identities", outcome.substitutions, outcome.translated, outcome.identities_created)
records = self.attach_engagement(records, extras, collected_users, options, skipped, report)
if skipped:
logger.info("select: skipped %s", dict(skipped))
return records, skipped, held
@staticmethod
def attach_engagement(records: dict, extras: dict, collected_users: list, options: SyncOptions, skipped, report: RunReport) -> dict:
authors = {("post", str(post.get("origin_uid"))): post.get("author_uid") for post in records.get("posts", [])}
authors |= {("comment", str(comment.get("origin_uid"))): comment.get("author_uid") for comment in records.get("comments", [])}
merged = dict(records)
for kind in ENGAGEMENT_KINDS:
kept = []
for record in extras.get(kind, []):
target = (record.get("target_kind"), str(record.get("target_uid")))
if target not in authors:
skipped[SKIP_PARENT] += 1
report.ledger.add_dropped_record(kind, record, SKIP_PARENT)
elif options.alias and kind != "attachments":
skipped[SKIP_ALIAS_ENGAGEMENT] += 1
report.ledger.add_dropped_record(kind, record, SKIP_ALIAS_ENGAGEMENT)
else:
kept.append({**record, "author_uid": authors[target]} if kind == "attachments" and options.alias else record)
merged[kind] = kept
needed = {str(item.get(field)) for kind in ("votes", "reactions") for item in merged.get(kind, []) for field in ("voter_uid", "author_uid")}
if needed:
known = {str(user.get("origin_uid")) for user in merged.get("users", [])}
merged["users"] = [*merged.get("users", []), *[user for user in collected_users if str(user.get("origin_uid")) in needed - known]]
return merged
def settle(self, row: dict, report: RunReport) -> None:
current = self.store.get(row["uid"])
if current is None:
return
if report.status in (RUN_OK, RUN_PARTIAL):
totals = dict(current.get("totals") or {})
for kind, counts in report.pushed.items():
for status, number in counts.items():
key = f"{kind}_{status}"
totals[key] = int(totals.get(key, 0)) + number
self.store.update(
row["uid"],
watermark=report.watermark or current.get("watermark") or {},
totals=totals,
run_count=int(current.get("run_count") or 0) + 1,
failure_count=0,
last_status=report.status,
last_error="; ".join(report.errors)[:1000],
last_run_at=iso(now()),
)
self.store.schedule_next(row["uid"], int(current.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(current).jitter_percent)
elif report.status == RUN_SKIPPED:
self.store.update(row["uid"], last_status=report.status, last_error="; ".join(report.errors)[:1000], last_run_at=iso(now()))
self.store.schedule_next(row["uid"], int(current.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(current).jitter_percent)
logger.warning("run: skipped for %s by configuration, not counted as a failure: %s", row["slug"], "; ".join(report.errors))
elif current["status"] != STATUS_LOGIN:
failed = self.store.fail(current, "; ".join(report.errors) or "run failed")
self.store.schedule_next(row["uid"], int(failed.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(failed).jitter_percent)
logger.error(
"run: failure %d of %d for %s. Structure drift is likely: re-run discovery with relearn.",
failed["failure_count"], LIMITS.max_failures, row["slug"],
)