# retoor from __future__ import annotations import asyncio import logging from dataclasses import dataclass, field from devplacepy.services.sync.aliaser import AliasRefused, Aliaser from devplacepy.services.sync.healing import AliasHealer from devplacepy.services.sync.options import SyncOptions, alias_allowed from devplacepy.services.sync.selection import Selector from devplacepy.services.community.checkpoint import RunCheckpoint from devplacepy.services.community.discoverer import CommunityDiscoverer, Discovery from devplacepy.services.community.fetcher import Fetcher from devplacepy.services.community.files import CommunityFiles, files as default_files from devplacepy.services.community.ledger import Ledger, tally_of, verdict from devplacepy.services.community.ledger_store import LedgerStore, ledger_store as default_ledger_store from devplacepy.services.community.logbook import LogBook from devplacepy.services.community.pusher import PushReport, SyncPusher from devplacepy.services.community.runtime import Collector, LoginRequired from devplacepy.services.community.schemacheck import sample_limits from devplacepy.services.community.spec import ( LIMITS, RUN_DRY, RUN_FAILED, RUN_OK, RUN_PARTIAL, RUN_SKIPPED, STATUS_LOGIN, STATUS_READY, ) from devplacepy.services.community.store import CommunityStore, iso, now, store as default_store from devplacepy.services.sync.gate import refusal from devplacepy.services.sync.spec import SKIP_ALIAS_ENGAGEMENT, SKIP_PARENT logger = logging.getLogger("community.run") TRIGGER_CLI = "cli" TRIGGER_ADMIN = "admin" TRIGGER_SERVICE = "service" KINDS = ("users", "posts", "comments") COLLECTED_KINDS = (*KINDS, "votes", "reactions", "attachments") ENGAGEMENT_KINDS = ("votes", "reactions", "attachments") def options_alias_active(row: dict) -> bool: return SyncOptions.from_row(row).alias @dataclass class SchemaVerdict: ok: bool posts: int comments: int problems: list @dataclass class RunReport: status: str dry_run: bool collected: dict = field(default_factory=dict) pushed: dict = field(default_factory=dict) errors: list = field(default_factory=list) log: str = "" run_uid: str = "" watermark: dict = field(default_factory=dict) ledger: Ledger = field(default_factory=Ledger) live: PushReport = field(default_factory=lambda: PushReport(dry_run=False)) class CommunityRunner: def __init__(self, fetcher_factory=Fetcher, store: CommunityStore | None = None, files: CommunityFiles | None = None, sleep=None, ledgers: LedgerStore | None = None): self.fetcher_factory = fetcher_factory self.sleep = sleep self.store = store or default_store self.files = files or default_files self.ledgers = ledgers or default_ledger_store async def discover( self, query: str, created_by: str, *, dry_run: bool = False, relearn_slug: str | None = None, credentials: dict | None = None, ) -> tuple[Discovery, dict | None, str]: with LogBook() as book: logger.info("discover%s: %r", " (dry run, nothing is saved)" if dry_run else "", query) existing = self.store.get(relearn_slug) if relearn_slug else None if existing and not dry_run: self.store.claim(existing["uid"]) secrets = credentials or (self.store.credentials(existing) if existing else {}) try: async with self.fetcher_factory() as fetcher: result = await asyncio.wait_for( CommunityDiscoverer(fetcher, credentials=secrets).discover(query), timeout=LIMITS.run_timeout_seconds ) except asyncio.TimeoutError: result = Discovery(query=query, error=f"discovery exceeded {LIMITS.run_timeout_seconds}s") logger.error("discover: %s", result.error) except Exception as exc: logger.exception("discover: unexpected failure") result = Discovery(query=query, error=f"{type(exc).__name__}: {exc}") row = None if not dry_run and (result.slug or relearn_slug): row = self.persist(result, query, created_by, relearn_slug, credentials) logger.info("discover: saved community %s with status %s", result.slug, result.status) elif dry_run: logger.info("discover: dry run, would save %s with status %s (%s feed(s))", result.slug, result.status, len(result.schema.get("feeds", []))) return result, row, book.text() def persist(self, result: Discovery, query: str, created_by: str, relearn_slug: str | None, credentials: dict | None = None) -> dict: slug = relearn_slug or result.slug existing = self.store.get(slug) learned = result.status in (STATUS_READY, STATUS_LOGIN) if existing and existing["status"] == STATUS_READY and not learned: logger.warning("discover: relearn failed, keeping the working schema of %s: %s", slug, result.error) return self.store.update(existing["uid"], last_error=result.error[:1000]) fields = { "name": result.name or existing and existing.get("name") or slug, "origin": result.origin, "url": result.url, "status": result.status, "description": result.metadata.get("description", ""), "rules": result.metadata.get("rules", ""), "language": result.metadata.get("language", ""), "favicon": result.metadata.get("favicon", ""), "user_count": int(result.metadata.get("user_count") or 0), "schema": result.schema, "learned_at": iso(now()) if learned else "", "last_error": result.error, "failure_count": 0, } if credentials and any(credentials.values()): fields.update( login_username=credentials.get("username", ""), login_email=credentials.get("email", ""), login_password=credentials.get("password", ""), login_totp_secret=credentials.get("totp", ""), ) if existing: row = self.store.update(existing["uid"], **fields, enabled=1 if result.status == STATUS_READY else 0) else: row = self.store.create(slug, query, created_by, **fields) if result.schema.get("feeds"): self.files.write(slug, result.schema) return row async def run(self, row: dict, *, dry_run: bool, trigger: str) -> RunReport: self.store.reap_interrupted() run_uid = self.store.start_run(row["uid"], trigger, dry_run) report = RunReport(status=RUN_FAILED, dry_run=dry_run, run_uid=run_uid) report.live.dry_run = dry_run with LogBook() as book: checkpoint = RunCheckpoint(self.store, self.ledgers, run_uid, row["uid"], report, report.live, book) ticker = asyncio.create_task(checkpoint.loop()) mode = "DRY RUN" if dry_run else "LIVE" logger.info("run[%s]: %s (%s) trigger=%s", mode, row["name"], row["origin"], trigger) try: await asyncio.wait_for(self.execute(row, report), timeout=LIMITS.run_timeout_seconds) except asyncio.TimeoutError: report.errors.append(f"run exceeded {LIMITS.run_timeout_seconds}s") report.ledger.add_error(report.errors[-1]) except LoginRequired as exc: report.errors.append(str(exc)) report.ledger.add_error(str(exc)) if not dry_run: self.store.update(row["uid"], status=STATUS_LOGIN, enabled=0, last_error=str(exc)) except Exception as exc: logger.exception("run[%s]: unexpected failure", mode) report.errors.append(f"{type(exc).__name__}: {exc}") report.ledger.add_error(report.errors[-1]) for error in report.errors: logger.error("run[%s]: %s", mode, error) logger.info("run[%s]: finished with status %s", mode, report.status) ticker.cancel() await asyncio.gather(ticker, return_exceptions=True) report.log = book.text() if not dry_run: self.settle(row, report) tally = tally_of(report.pushed, report.ledger) outcome = verdict(report.status, report.dry_run, tally) stored = checkpoint.saved + self.ledgers.append(run_uid, row["uid"], report.ledger, checkpoint.saved) self.store.finish_run( run_uid, report.status, collected=report.collected, pushed=report.pushed, errors=report.errors, log=report.log, tally=tally, verdict=outcome["level"], headline=outcome["headline"], items_stored=stored, items_dropped=report.ledger.dropped, ) self.store.prune_runs(row["uid"]) return report async def verify_schema(self, row: dict, schema: dict) -> "SchemaVerdict": sample = {**schema, "limits": sample_limits(schema)} try: async with self.fetcher_factory() as fetcher: result = await asyncio.wait_for( Collector(fetcher, sample, None, sleep=self.sleep, credentials=self.store.credentials(row)).run(), timeout=LIMITS.run_timeout_seconds, ) except asyncio.TimeoutError: return SchemaVerdict(False, 0, 0, [f"the verification run exceeded {LIMITS.run_timeout_seconds}s"]) except LoginRequired as exc: return SchemaVerdict(False, 0, 0, [str(exc)]) except Exception as exc: logger.exception("schema verification crashed") return SchemaVerdict(False, 0, 0, [f"{type(exc).__name__}: {exc}"]) posts, comments = len(result["records"]["posts"]), len(result["records"]["comments"]) problems = list(result.get("errors", [])) if not posts: problems.append("the schema collected no posts from the live source") return SchemaVerdict(bool(result["ok"]) and posts > 0, posts, comments, problems) async def execute(self, row: dict, report: RunReport) -> None: refused = refusal(row["origin"]) if refused and not report.dry_run: report.errors.append(refused) report.status = RUN_SKIPPED return if refused: logger.warning("run[DRY RUN]: %s; continuing because nothing is written", refused) module = self.files.load(row["slug"], row["schema"]) options = SyncOptions.from_row(row) async with self.fetcher_factory() as fetcher: result = await module.collect_all( fetcher, row.get("watermark") or None, sleep=self.sleep, credentials=self.store.credentials(row), options={"attachments": options.collect_attachments}, ) records = result["records"] report.collected = {kind: len(records.get(kind, [])) for kind in COLLECTED_KINDS if records.get(kind) or kind in KINDS} logger.info("run: collected %s", report.collected) report.errors.extend(result.get("errors", [])) for warning in result.get("warnings", []): logger.warning("run: %s", warning) report.ledger.sources.update(result.get("sources", {})) for notice in result.get("notices", []): report.ledger.add_notice(notice) for error in result.get("errors", []): report.ledger.add_error(error) if not result["ok"]: report.errors.append("no feed could be collected") return prepared = await self.prepare(row, options, records, report) if prepared is None: report.status = RUN_SKIPPED return records, skipped, held = prepared push = await asyncio.to_thread(SyncPusher(row["origin"], dry_run=report.dry_run, ledger=report.ledger, report=report.live).push, records) if options_alias_active(row) and not report.dry_run: self.heal(row["origin"]) report.pushed = push.counts | ({"skipped": dict(skipped)} if skipped else {}) report.errors.extend(push.problems[:20]) report.watermark = {} if held else result["state"] if held: report.errors.append(f"{held} item(s) held back for a transient reason and will be retried") logger.warning("run: %d item(s) were skipped for a transient reason, the watermark is held so the next run retries them", held) failed = push.total("failed") report.status = RUN_DRY if report.dry_run else (RUN_PARTIAL if report.errors or failed else RUN_OK) @staticmethod def heal(origin: str) -> None: healed = AliasHealer(origin).heal() if healed.renamed: logger.info("alias: %d alias(es) followed their clone, %d stored text(s) rewritten", len(healed.renamed), healed.rewritten) if healed.conflicts: logger.warning("alias: %d alias(es) could not follow their clone: %s", len(healed.conflicts), ", ".join(healed.conflicts)) async def prepare(self, row: dict, options: SyncOptions, records: dict, report: RunReport): selector = Selector(row["origin"], options, persist=not report.dry_run) if options.alias and not report.dry_run: self.heal(row["origin"]) extras = {kind: records.get(kind, []) for kind in ENGAGEMENT_KINDS} collected_users = list(records.get("users", [])) records, skipped = selector.select(records) for kind, record, reason in selector.dropped: report.ledger.add_dropped_record(kind, record, reason) held = 0 if options.alias or options.translate_to: if options.alias and not alias_allowed(): report.errors.append("identity aliasing is switched off: enable community_alias_enabled on the Community sync service") return None try: outcome = await Aliaser(row["origin"], options, language=row.get("language") or "", dry_run=report.dry_run).apply(records) except AliasRefused as exc: report.errors.append(str(exc)) return None records, held = outcome.records, outcome.transient() for kind, record, reason in outcome.dropped: report.ledger.add_dropped_record(kind, record, reason) skipped.update(outcome.skipped) logger.info("alias: %d substitution(s), %d translated, %d new identities", outcome.substitutions, outcome.translated, outcome.identities_created) records = self.attach_engagement(records, extras, collected_users, options, skipped, report) if skipped: logger.info("select: skipped %s", dict(skipped)) return records, skipped, held @staticmethod def attach_engagement(records: dict, extras: dict, collected_users: list, options: SyncOptions, skipped, report: RunReport) -> dict: authors = {("post", str(post.get("origin_uid"))): post.get("author_uid") for post in records.get("posts", [])} authors |= {("comment", str(comment.get("origin_uid"))): comment.get("author_uid") for comment in records.get("comments", [])} merged = dict(records) for kind in ENGAGEMENT_KINDS: kept = [] for record in extras.get(kind, []): target = (record.get("target_kind"), str(record.get("target_uid"))) if target not in authors: skipped[SKIP_PARENT] += 1 report.ledger.add_dropped_record(kind, record, SKIP_PARENT) elif options.alias and kind != "attachments": skipped[SKIP_ALIAS_ENGAGEMENT] += 1 report.ledger.add_dropped_record(kind, record, SKIP_ALIAS_ENGAGEMENT) else: kept.append({**record, "author_uid": authors[target]} if kind == "attachments" and options.alias else record) merged[kind] = kept needed = {str(item.get(field)) for kind in ("votes", "reactions") for item in merged.get(kind, []) for field in ("voter_uid", "author_uid")} if needed: known = {str(user.get("origin_uid")) for user in merged.get("users", [])} merged["users"] = [*merged.get("users", []), *[user for user in collected_users if str(user.get("origin_uid")) in needed - known]] return merged def settle(self, row: dict, report: RunReport) -> None: current = self.store.get(row["uid"]) if current is None: return if report.status in (RUN_OK, RUN_PARTIAL): totals = dict(current.get("totals") or {}) for kind, counts in report.pushed.items(): for status, number in counts.items(): key = f"{kind}_{status}" totals[key] = int(totals.get(key, 0)) + number self.store.update( row["uid"], watermark=report.watermark or current.get("watermark") or {}, totals=totals, run_count=int(current.get("run_count") or 0) + 1, failure_count=0, last_status=report.status, last_error="; ".join(report.errors)[:1000], last_run_at=iso(now()), ) self.store.schedule_next(row["uid"], int(current.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(current).jitter_percent) elif report.status == RUN_SKIPPED: self.store.update(row["uid"], last_status=report.status, last_error="; ".join(report.errors)[:1000], last_run_at=iso(now())) self.store.schedule_next(row["uid"], int(current.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(current).jitter_percent) logger.warning("run: skipped for %s by configuration, not counted as a failure: %s", row["slug"], "; ".join(report.errors)) elif current["status"] != STATUS_LOGIN: failed = self.store.fail(current, "; ".join(report.errors) or "run failed") self.store.schedule_next(row["uid"], int(failed.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(failed).jitter_percent) logger.error( "run: failure %d of %d for %s. Structure drift is likely: re-run discovery with relearn.", failed["failure_count"], LIMITS.max_failures, row["slug"], )