forked from retoor/devplacepy
Continuous Delivery could never be turned off once a crop was planted. Add a one-shot per-plot "stop after harvest" flag (POST /game/plot/stop, Devii tool game_plot_stop, game_plots.stop_after_harvest), cleared by the next plant, autoreplant claim or Refactor. Autoreplant now also respects crop locks through the shared _crop_lock predicate. Plan and measurements in happyplanting.md. Also includes the pending community sync, node network and related work in the tree; the full suite passed (4999) with all of it. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
366 lines
18 KiB
Python
366 lines
18 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
|
|
from devplacepy.services.sync.aliaser import AliasRefused, Aliaser
|
|
from devplacepy.services.sync.healing import AliasHealer
|
|
from devplacepy.services.sync.options import SyncOptions, alias_allowed
|
|
from devplacepy.services.sync.selection import Selector
|
|
from devplacepy.services.community.checkpoint import RunCheckpoint
|
|
from devplacepy.services.community.discoverer import CommunityDiscoverer, Discovery
|
|
from devplacepy.services.community.fetcher import Fetcher
|
|
from devplacepy.services.community.files import CommunityFiles, files as default_files
|
|
from devplacepy.services.community.ledger import Ledger, tally_of, verdict
|
|
from devplacepy.services.community.ledger_store import LedgerStore, ledger_store as default_ledger_store
|
|
from devplacepy.services.community.logbook import LogBook
|
|
from devplacepy.services.community.pusher import PushReport, SyncPusher
|
|
from devplacepy.services.community.runtime import Collector, LoginRequired
|
|
from devplacepy.services.community.schemacheck import sample_limits
|
|
from devplacepy.services.community.spec import (
|
|
LIMITS,
|
|
RUN_DRY,
|
|
RUN_FAILED,
|
|
RUN_OK,
|
|
RUN_PARTIAL,
|
|
RUN_SKIPPED,
|
|
STATUS_LOGIN,
|
|
STATUS_READY,
|
|
)
|
|
from devplacepy.services.community.store import CommunityStore, iso, now, store as default_store
|
|
from devplacepy.services.sync.gate import refusal
|
|
from devplacepy.services.sync.spec import SKIP_ALIAS_ENGAGEMENT, SKIP_PARENT
|
|
|
|
logger = logging.getLogger("community.run")
|
|
|
|
TRIGGER_CLI = "cli"
|
|
TRIGGER_ADMIN = "admin"
|
|
TRIGGER_SERVICE = "service"
|
|
KINDS = ("users", "posts", "comments")
|
|
COLLECTED_KINDS = (*KINDS, "votes", "reactions", "attachments")
|
|
ENGAGEMENT_KINDS = ("votes", "reactions", "attachments")
|
|
|
|
|
|
def options_alias_active(row: dict) -> bool:
|
|
return SyncOptions.from_row(row).alias
|
|
|
|
|
|
@dataclass
|
|
class SchemaVerdict:
|
|
ok: bool
|
|
posts: int
|
|
comments: int
|
|
problems: list
|
|
|
|
|
|
@dataclass
|
|
class RunReport:
|
|
status: str
|
|
dry_run: bool
|
|
collected: dict = field(default_factory=dict)
|
|
pushed: dict = field(default_factory=dict)
|
|
errors: list = field(default_factory=list)
|
|
log: str = ""
|
|
run_uid: str = ""
|
|
watermark: dict = field(default_factory=dict)
|
|
ledger: Ledger = field(default_factory=Ledger)
|
|
live: PushReport = field(default_factory=lambda: PushReport(dry_run=False))
|
|
|
|
|
|
class CommunityRunner:
|
|
def __init__(self, fetcher_factory=Fetcher, store: CommunityStore | None = None, files: CommunityFiles | None = None, sleep=None, ledgers: LedgerStore | None = None):
|
|
self.fetcher_factory = fetcher_factory
|
|
self.sleep = sleep
|
|
self.store = store or default_store
|
|
self.files = files or default_files
|
|
self.ledgers = ledgers or default_ledger_store
|
|
|
|
async def discover(
|
|
self,
|
|
query: str,
|
|
created_by: str,
|
|
*,
|
|
dry_run: bool = False,
|
|
relearn_slug: str | None = None,
|
|
credentials: dict | None = None,
|
|
) -> tuple[Discovery, dict | None, str]:
|
|
with LogBook() as book:
|
|
logger.info("discover%s: %r", " (dry run, nothing is saved)" if dry_run else "", query)
|
|
existing = self.store.get(relearn_slug) if relearn_slug else None
|
|
if existing and not dry_run:
|
|
self.store.claim(existing["uid"])
|
|
secrets = credentials or (self.store.credentials(existing) if existing else {})
|
|
try:
|
|
async with self.fetcher_factory() as fetcher:
|
|
result = await asyncio.wait_for(
|
|
CommunityDiscoverer(fetcher, credentials=secrets).discover(query), timeout=LIMITS.run_timeout_seconds
|
|
)
|
|
except asyncio.TimeoutError:
|
|
result = Discovery(query=query, error=f"discovery exceeded {LIMITS.run_timeout_seconds}s")
|
|
logger.error("discover: %s", result.error)
|
|
except Exception as exc:
|
|
logger.exception("discover: unexpected failure")
|
|
result = Discovery(query=query, error=f"{type(exc).__name__}: {exc}")
|
|
row = None
|
|
if not dry_run and (result.slug or relearn_slug):
|
|
row = self.persist(result, query, created_by, relearn_slug, credentials)
|
|
logger.info("discover: saved community %s with status %s", result.slug, result.status)
|
|
elif dry_run:
|
|
logger.info("discover: dry run, would save %s with status %s (%s feed(s))", result.slug, result.status, len(result.schema.get("feeds", [])))
|
|
return result, row, book.text()
|
|
|
|
def persist(self, result: Discovery, query: str, created_by: str, relearn_slug: str | None, credentials: dict | None = None) -> dict:
|
|
slug = relearn_slug or result.slug
|
|
existing = self.store.get(slug)
|
|
learned = result.status in (STATUS_READY, STATUS_LOGIN)
|
|
if existing and existing["status"] == STATUS_READY and not learned:
|
|
logger.warning("discover: relearn failed, keeping the working schema of %s: %s", slug, result.error)
|
|
return self.store.update(existing["uid"], last_error=result.error[:1000])
|
|
fields = {
|
|
"name": result.name or existing and existing.get("name") or slug,
|
|
"origin": result.origin,
|
|
"url": result.url,
|
|
"status": result.status,
|
|
"description": result.metadata.get("description", ""),
|
|
"rules": result.metadata.get("rules", ""),
|
|
"language": result.metadata.get("language", ""),
|
|
"favicon": result.metadata.get("favicon", ""),
|
|
"user_count": int(result.metadata.get("user_count") or 0),
|
|
"schema": result.schema,
|
|
"learned_at": iso(now()) if learned else "",
|
|
"last_error": result.error,
|
|
"failure_count": 0,
|
|
}
|
|
if credentials and any(credentials.values()):
|
|
fields.update(
|
|
login_username=credentials.get("username", ""),
|
|
login_email=credentials.get("email", ""),
|
|
login_password=credentials.get("password", ""),
|
|
login_totp_secret=credentials.get("totp", ""),
|
|
)
|
|
if existing:
|
|
row = self.store.update(existing["uid"], **fields, enabled=1 if result.status == STATUS_READY else 0)
|
|
else:
|
|
row = self.store.create(slug, query, created_by, **fields)
|
|
if result.schema.get("feeds"):
|
|
self.files.write(slug, result.schema)
|
|
return row
|
|
|
|
async def run(self, row: dict, *, dry_run: bool, trigger: str) -> RunReport:
|
|
self.store.reap_interrupted()
|
|
run_uid = self.store.start_run(row["uid"], trigger, dry_run)
|
|
report = RunReport(status=RUN_FAILED, dry_run=dry_run, run_uid=run_uid)
|
|
report.live.dry_run = dry_run
|
|
with LogBook() as book:
|
|
checkpoint = RunCheckpoint(self.store, self.ledgers, run_uid, row["uid"], report, report.live, book)
|
|
ticker = asyncio.create_task(checkpoint.loop())
|
|
mode = "DRY RUN" if dry_run else "LIVE"
|
|
logger.info("run[%s]: %s (%s) trigger=%s", mode, row["name"], row["origin"], trigger)
|
|
try:
|
|
await asyncio.wait_for(self.execute(row, report), timeout=LIMITS.run_timeout_seconds)
|
|
except asyncio.TimeoutError:
|
|
report.errors.append(f"run exceeded {LIMITS.run_timeout_seconds}s")
|
|
report.ledger.add_error(report.errors[-1])
|
|
except LoginRequired as exc:
|
|
report.errors.append(str(exc))
|
|
report.ledger.add_error(str(exc))
|
|
if not dry_run:
|
|
self.store.update(row["uid"], status=STATUS_LOGIN, enabled=0, last_error=str(exc))
|
|
except Exception as exc:
|
|
logger.exception("run[%s]: unexpected failure", mode)
|
|
report.errors.append(f"{type(exc).__name__}: {exc}")
|
|
report.ledger.add_error(report.errors[-1])
|
|
for error in report.errors:
|
|
logger.error("run[%s]: %s", mode, error)
|
|
logger.info("run[%s]: finished with status %s", mode, report.status)
|
|
ticker.cancel()
|
|
await asyncio.gather(ticker, return_exceptions=True)
|
|
report.log = book.text()
|
|
if not dry_run:
|
|
self.settle(row, report)
|
|
tally = tally_of(report.pushed, report.ledger)
|
|
outcome = verdict(report.status, report.dry_run, tally)
|
|
stored = checkpoint.saved + self.ledgers.append(run_uid, row["uid"], report.ledger, checkpoint.saved)
|
|
self.store.finish_run(
|
|
run_uid,
|
|
report.status,
|
|
collected=report.collected,
|
|
pushed=report.pushed,
|
|
errors=report.errors,
|
|
log=report.log,
|
|
tally=tally,
|
|
verdict=outcome["level"],
|
|
headline=outcome["headline"],
|
|
items_stored=stored,
|
|
items_dropped=report.ledger.dropped,
|
|
)
|
|
self.store.prune_runs(row["uid"])
|
|
return report
|
|
|
|
async def verify_schema(self, row: dict, schema: dict) -> "SchemaVerdict":
|
|
sample = {**schema, "limits": sample_limits(schema)}
|
|
try:
|
|
async with self.fetcher_factory() as fetcher:
|
|
result = await asyncio.wait_for(
|
|
Collector(fetcher, sample, None, sleep=self.sleep, credentials=self.store.credentials(row)).run(),
|
|
timeout=LIMITS.run_timeout_seconds,
|
|
)
|
|
except asyncio.TimeoutError:
|
|
return SchemaVerdict(False, 0, 0, [f"the verification run exceeded {LIMITS.run_timeout_seconds}s"])
|
|
except LoginRequired as exc:
|
|
return SchemaVerdict(False, 0, 0, [str(exc)])
|
|
except Exception as exc:
|
|
logger.exception("schema verification crashed")
|
|
return SchemaVerdict(False, 0, 0, [f"{type(exc).__name__}: {exc}"])
|
|
posts, comments = len(result["records"]["posts"]), len(result["records"]["comments"])
|
|
problems = list(result.get("errors", []))
|
|
if not posts:
|
|
problems.append("the schema collected no posts from the live source")
|
|
return SchemaVerdict(bool(result["ok"]) and posts > 0, posts, comments, problems)
|
|
|
|
async def execute(self, row: dict, report: RunReport) -> None:
|
|
refused = refusal(row["origin"])
|
|
if refused and not report.dry_run:
|
|
report.errors.append(refused)
|
|
report.status = RUN_SKIPPED
|
|
return
|
|
if refused:
|
|
logger.warning("run[DRY RUN]: %s; continuing because nothing is written", refused)
|
|
module = self.files.load(row["slug"], row["schema"])
|
|
options = SyncOptions.from_row(row)
|
|
async with self.fetcher_factory() as fetcher:
|
|
result = await module.collect_all(
|
|
fetcher,
|
|
row.get("watermark") or None,
|
|
sleep=self.sleep,
|
|
credentials=self.store.credentials(row),
|
|
options={"attachments": options.collect_attachments},
|
|
)
|
|
records = result["records"]
|
|
report.collected = {kind: len(records.get(kind, [])) for kind in COLLECTED_KINDS if records.get(kind) or kind in KINDS}
|
|
logger.info("run: collected %s", report.collected)
|
|
report.errors.extend(result.get("errors", []))
|
|
for warning in result.get("warnings", []):
|
|
logger.warning("run: %s", warning)
|
|
report.ledger.sources.update(result.get("sources", {}))
|
|
for notice in result.get("notices", []):
|
|
report.ledger.add_notice(notice)
|
|
for error in result.get("errors", []):
|
|
report.ledger.add_error(error)
|
|
if not result["ok"]:
|
|
report.errors.append("no feed could be collected")
|
|
return
|
|
prepared = await self.prepare(row, options, records, report)
|
|
if prepared is None:
|
|
report.status = RUN_SKIPPED
|
|
return
|
|
records, skipped, held = prepared
|
|
push = await asyncio.to_thread(SyncPusher(row["origin"], dry_run=report.dry_run, ledger=report.ledger, report=report.live).push, records)
|
|
if options_alias_active(row) and not report.dry_run:
|
|
self.heal(row["origin"])
|
|
report.pushed = push.counts | ({"skipped": dict(skipped)} if skipped else {})
|
|
report.errors.extend(push.problems[:20])
|
|
report.watermark = {} if held else result["state"]
|
|
if held:
|
|
report.errors.append(f"{held} item(s) held back for a transient reason and will be retried")
|
|
logger.warning("run: %d item(s) were skipped for a transient reason, the watermark is held so the next run retries them", held)
|
|
failed = push.total("failed")
|
|
report.status = RUN_DRY if report.dry_run else (RUN_PARTIAL if report.errors or failed else RUN_OK)
|
|
|
|
@staticmethod
|
|
def heal(origin: str) -> None:
|
|
healed = AliasHealer(origin).heal()
|
|
if healed.renamed:
|
|
logger.info("alias: %d alias(es) followed their clone, %d stored text(s) rewritten", len(healed.renamed), healed.rewritten)
|
|
if healed.conflicts:
|
|
logger.warning("alias: %d alias(es) could not follow their clone: %s", len(healed.conflicts), ", ".join(healed.conflicts))
|
|
|
|
async def prepare(self, row: dict, options: SyncOptions, records: dict, report: RunReport):
|
|
selector = Selector(row["origin"], options, persist=not report.dry_run)
|
|
if options.alias and not report.dry_run:
|
|
self.heal(row["origin"])
|
|
extras = {kind: records.get(kind, []) for kind in ENGAGEMENT_KINDS}
|
|
collected_users = list(records.get("users", []))
|
|
records, skipped = selector.select(records)
|
|
for kind, record, reason in selector.dropped:
|
|
report.ledger.add_dropped_record(kind, record, reason)
|
|
held = 0
|
|
if options.alias or options.translate_to:
|
|
if options.alias and not alias_allowed():
|
|
report.errors.append("identity aliasing is switched off: enable community_alias_enabled on the Community sync service")
|
|
return None
|
|
try:
|
|
outcome = await Aliaser(row["origin"], options, language=row.get("language") or "", dry_run=report.dry_run).apply(records)
|
|
except AliasRefused as exc:
|
|
report.errors.append(str(exc))
|
|
return None
|
|
records, held = outcome.records, outcome.transient()
|
|
for kind, record, reason in outcome.dropped:
|
|
report.ledger.add_dropped_record(kind, record, reason)
|
|
skipped.update(outcome.skipped)
|
|
logger.info("alias: %d substitution(s), %d translated, %d new identities", outcome.substitutions, outcome.translated, outcome.identities_created)
|
|
records = self.attach_engagement(records, extras, collected_users, options, skipped, report)
|
|
if skipped:
|
|
logger.info("select: skipped %s", dict(skipped))
|
|
return records, skipped, held
|
|
|
|
@staticmethod
|
|
def attach_engagement(records: dict, extras: dict, collected_users: list, options: SyncOptions, skipped, report: RunReport) -> dict:
|
|
authors = {("post", str(post.get("origin_uid"))): post.get("author_uid") for post in records.get("posts", [])}
|
|
authors |= {("comment", str(comment.get("origin_uid"))): comment.get("author_uid") for comment in records.get("comments", [])}
|
|
merged = dict(records)
|
|
for kind in ENGAGEMENT_KINDS:
|
|
kept = []
|
|
for record in extras.get(kind, []):
|
|
target = (record.get("target_kind"), str(record.get("target_uid")))
|
|
if target not in authors:
|
|
skipped[SKIP_PARENT] += 1
|
|
report.ledger.add_dropped_record(kind, record, SKIP_PARENT)
|
|
elif options.alias and kind != "attachments":
|
|
skipped[SKIP_ALIAS_ENGAGEMENT] += 1
|
|
report.ledger.add_dropped_record(kind, record, SKIP_ALIAS_ENGAGEMENT)
|
|
else:
|
|
kept.append({**record, "author_uid": authors[target]} if kind == "attachments" and options.alias else record)
|
|
merged[kind] = kept
|
|
needed = {str(item.get(field)) for kind in ("votes", "reactions") for item in merged.get(kind, []) for field in ("voter_uid", "author_uid")}
|
|
if needed:
|
|
known = {str(user.get("origin_uid")) for user in merged.get("users", [])}
|
|
merged["users"] = [*merged.get("users", []), *[user for user in collected_users if str(user.get("origin_uid")) in needed - known]]
|
|
return merged
|
|
|
|
def settle(self, row: dict, report: RunReport) -> None:
|
|
current = self.store.get(row["uid"])
|
|
if current is None:
|
|
return
|
|
if report.status in (RUN_OK, RUN_PARTIAL):
|
|
totals = dict(current.get("totals") or {})
|
|
for kind, counts in report.pushed.items():
|
|
for status, number in counts.items():
|
|
key = f"{kind}_{status}"
|
|
totals[key] = int(totals.get(key, 0)) + number
|
|
self.store.update(
|
|
row["uid"],
|
|
watermark=report.watermark or current.get("watermark") or {},
|
|
totals=totals,
|
|
run_count=int(current.get("run_count") or 0) + 1,
|
|
failure_count=0,
|
|
last_status=report.status,
|
|
last_error="; ".join(report.errors)[:1000],
|
|
last_run_at=iso(now()),
|
|
)
|
|
self.store.schedule_next(row["uid"], int(current.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(current).jitter_percent)
|
|
elif report.status == RUN_SKIPPED:
|
|
self.store.update(row["uid"], last_status=report.status, last_error="; ".join(report.errors)[:1000], last_run_at=iso(now()))
|
|
self.store.schedule_next(row["uid"], int(current.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(current).jitter_percent)
|
|
logger.warning("run: skipped for %s by configuration, not counted as a failure: %s", row["slug"], "; ".join(report.errors))
|
|
elif current["status"] != STATUS_LOGIN:
|
|
failed = self.store.fail(current, "; ".join(report.errors) or "run failed")
|
|
self.store.schedule_next(row["uid"], int(failed.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(failed).jitter_percent)
|
|
logger.error(
|
|
"run: failure %d of %d for %s. Structure drift is likely: re-run discovery with relearn.",
|
|
failed["failure_count"], LIMITS.max_failures, row["slug"],
|
|
)
|