233 lines
11 KiB
Python
233 lines
11 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
|
|
from devplacepy.services.sync.aliaser import AliasRefused, Aliaser
|
|
from devplacepy.services.sync.options import SyncOptions, alias_allowed
|
|
from devplacepy.services.sync.selection import Selector
|
|
from devplacepy.services.community.discoverer import CommunityDiscoverer, Discovery
|
|
from devplacepy.services.community.fetcher import Fetcher
|
|
from devplacepy.services.community.files import CommunityFiles, files as default_files
|
|
from devplacepy.services.community.logbook import LogBook
|
|
from devplacepy.services.community.pusher import SyncPusher
|
|
from devplacepy.services.community.runtime import LoginRequired
|
|
from devplacepy.services.community.spec import (
|
|
LIMITS,
|
|
RUN_DRY,
|
|
RUN_FAILED,
|
|
RUN_OK,
|
|
RUN_PARTIAL,
|
|
STATUS_LOGIN,
|
|
STATUS_READY,
|
|
)
|
|
from devplacepy.services.community.store import CommunityStore, iso, now, store as default_store
|
|
from devplacepy.services.sync.gate import refusal
|
|
|
|
logger = logging.getLogger("community.run")
|
|
|
|
TRIGGER_CLI = "cli"
|
|
TRIGGER_ADMIN = "admin"
|
|
TRIGGER_SERVICE = "service"
|
|
KINDS = ("users", "posts", "comments")
|
|
|
|
|
|
@dataclass
|
|
class RunReport:
|
|
status: str
|
|
dry_run: bool
|
|
collected: dict = field(default_factory=dict)
|
|
pushed: dict = field(default_factory=dict)
|
|
errors: list = field(default_factory=list)
|
|
log: str = ""
|
|
run_uid: str = ""
|
|
watermark: dict = field(default_factory=dict)
|
|
|
|
|
|
class CommunityRunner:
|
|
def __init__(self, fetcher_factory=Fetcher, store: CommunityStore | None = None, files: CommunityFiles | None = None, sleep=None):
|
|
self.fetcher_factory = fetcher_factory
|
|
self.sleep = sleep
|
|
self.store = store or default_store
|
|
self.files = files or default_files
|
|
|
|
async def discover(
|
|
self,
|
|
query: str,
|
|
created_by: str,
|
|
*,
|
|
dry_run: bool = False,
|
|
relearn_slug: str | None = None,
|
|
credentials: dict | None = None,
|
|
) -> tuple[Discovery, dict | None, str]:
|
|
with LogBook() as book:
|
|
logger.info("discover%s: %r", " (dry run, nothing is saved)" if dry_run else "", query)
|
|
existing = self.store.get(relearn_slug) if relearn_slug else None
|
|
if existing and not dry_run:
|
|
self.store.claim(existing["uid"])
|
|
secrets = credentials or (self.store.credentials(existing) if existing else {})
|
|
try:
|
|
async with self.fetcher_factory() as fetcher:
|
|
result = await asyncio.wait_for(
|
|
CommunityDiscoverer(fetcher, credentials=secrets).discover(query), timeout=LIMITS.run_timeout_seconds
|
|
)
|
|
except asyncio.TimeoutError:
|
|
result = Discovery(query=query, error=f"discovery exceeded {LIMITS.run_timeout_seconds}s")
|
|
logger.error("discover: %s", result.error)
|
|
except Exception as exc:
|
|
logger.exception("discover: unexpected failure")
|
|
result = Discovery(query=query, error=f"{type(exc).__name__}: {exc}")
|
|
row = None
|
|
if not dry_run and (result.slug or relearn_slug):
|
|
row = self.persist(result, query, created_by, relearn_slug, credentials)
|
|
logger.info("discover: saved community %s with status %s", result.slug, result.status)
|
|
elif dry_run:
|
|
logger.info("discover: dry run, would save %s with status %s (%s feed(s))", result.slug, result.status, len(result.schema.get("feeds", [])))
|
|
return result, row, book.text()
|
|
|
|
def persist(self, result: Discovery, query: str, created_by: str, relearn_slug: str | None, credentials: dict | None = None) -> dict:
|
|
slug = relearn_slug or result.slug
|
|
existing = self.store.get(slug)
|
|
learned = result.status in (STATUS_READY, STATUS_LOGIN)
|
|
if existing and existing["status"] == STATUS_READY and not learned:
|
|
logger.warning("discover: relearn failed, keeping the working schema of %s: %s", slug, result.error)
|
|
return self.store.update(existing["uid"], last_error=result.error[:1000])
|
|
fields = {
|
|
"name": result.name or existing and existing.get("name") or slug,
|
|
"origin": result.origin,
|
|
"url": result.url,
|
|
"status": result.status,
|
|
"description": result.metadata.get("description", ""),
|
|
"rules": result.metadata.get("rules", ""),
|
|
"language": result.metadata.get("language", ""),
|
|
"favicon": result.metadata.get("favicon", ""),
|
|
"user_count": int(result.metadata.get("user_count") or 0),
|
|
"schema": result.schema,
|
|
"learned_at": iso(now()) if learned else "",
|
|
"last_error": result.error,
|
|
"failure_count": 0,
|
|
}
|
|
if credentials and any(credentials.values()):
|
|
fields.update(
|
|
login_username=credentials.get("username", ""),
|
|
login_email=credentials.get("email", ""),
|
|
login_password=credentials.get("password", ""),
|
|
)
|
|
if existing:
|
|
row = self.store.update(existing["uid"], **fields, enabled=1 if result.status == STATUS_READY else 0)
|
|
else:
|
|
row = self.store.create(slug, query, created_by, **fields)
|
|
if result.schema.get("feeds"):
|
|
self.files.write(slug, result.schema)
|
|
return row
|
|
|
|
async def run(self, row: dict, *, dry_run: bool, trigger: str) -> RunReport:
|
|
run_uid = self.store.start_run(row["uid"], trigger, dry_run)
|
|
report = RunReport(status=RUN_FAILED, dry_run=dry_run, run_uid=run_uid)
|
|
with LogBook() as book:
|
|
mode = "DRY RUN" if dry_run else "LIVE"
|
|
logger.info("run[%s]: %s (%s) trigger=%s", mode, row["name"], row["origin"], trigger)
|
|
try:
|
|
await asyncio.wait_for(self.execute(row, report), timeout=LIMITS.run_timeout_seconds)
|
|
except asyncio.TimeoutError:
|
|
report.errors.append(f"run exceeded {LIMITS.run_timeout_seconds}s")
|
|
except LoginRequired as exc:
|
|
report.errors.append(str(exc))
|
|
if not dry_run:
|
|
self.store.update(row["uid"], status=STATUS_LOGIN, enabled=0, last_error=str(exc))
|
|
except Exception as exc:
|
|
logger.exception("run[%s]: unexpected failure", mode)
|
|
report.errors.append(f"{type(exc).__name__}: {exc}")
|
|
for error in report.errors:
|
|
logger.error("run[%s]: %s", mode, error)
|
|
logger.info("run[%s]: finished with status %s", mode, report.status)
|
|
report.log = book.text()
|
|
if not dry_run:
|
|
self.settle(row, report)
|
|
self.store.finish_run(run_uid, report.status, collected=report.collected, pushed=report.pushed, errors=report.errors, log=report.log)
|
|
self.store.prune_runs(row["uid"])
|
|
return report
|
|
|
|
async def execute(self, row: dict, report: RunReport) -> None:
|
|
refused = refusal(row["origin"])
|
|
if refused and not report.dry_run:
|
|
report.errors.append(refused)
|
|
return
|
|
if refused:
|
|
logger.warning("run[DRY RUN]: %s; continuing because nothing is written", refused)
|
|
module = self.files.load(row["slug"], row["schema"])
|
|
async with self.fetcher_factory() as fetcher:
|
|
result = await module.collect_all(fetcher, row.get("watermark") or None, sleep=self.sleep, credentials=self.store.credentials(row))
|
|
records = result["records"]
|
|
report.collected = {kind: len(records[kind]) for kind in KINDS}
|
|
logger.info("run: collected %s", report.collected)
|
|
report.errors.extend(result.get("errors", []))
|
|
if not result["ok"]:
|
|
report.errors.append("no feed could be collected")
|
|
return
|
|
options = SyncOptions.from_row(row)
|
|
prepared = await self.prepare(row, options, records, report)
|
|
if prepared is None:
|
|
return
|
|
records, skipped, held = prepared
|
|
push = await asyncio.to_thread(SyncPusher(row["origin"], dry_run=report.dry_run).push, records)
|
|
report.pushed = push.counts | ({"skipped": dict(skipped)} if skipped else {})
|
|
report.errors.extend(push.problems[:20])
|
|
report.watermark = {} if held else result["state"]
|
|
if held:
|
|
report.errors.append(f"{held} item(s) held back for a transient reason and will be retried")
|
|
logger.warning("run: %d item(s) were skipped for a transient reason, the watermark is held so the next run retries them", held)
|
|
failed = push.total("failed")
|
|
report.status = RUN_DRY if report.dry_run else (RUN_PARTIAL if report.errors or failed else RUN_OK)
|
|
|
|
async def prepare(self, row: dict, options: SyncOptions, records: dict, report: RunReport):
|
|
records, skipped = Selector(row["origin"], options).select(records)
|
|
held = 0
|
|
if options.alias or options.translate_to:
|
|
if options.alias and not alias_allowed():
|
|
report.errors.append("identity aliasing is switched off: enable community_alias_enabled on the Community sync service")
|
|
return None
|
|
try:
|
|
outcome = await Aliaser(row["origin"], options, language=row.get("language") or "", dry_run=report.dry_run).apply(records)
|
|
except AliasRefused as exc:
|
|
report.errors.append(str(exc))
|
|
return None
|
|
records, held = outcome.records, outcome.transient()
|
|
skipped.update(outcome.skipped)
|
|
logger.info("alias: %d substitution(s), %d translated, %d new identities", outcome.substitutions, outcome.translated, outcome.identities_created)
|
|
if skipped:
|
|
logger.info("select: skipped %s", dict(skipped))
|
|
return records, skipped, held
|
|
|
|
def settle(self, row: dict, report: RunReport) -> None:
|
|
current = self.store.get(row["uid"])
|
|
if current is None:
|
|
return
|
|
if report.status in (RUN_OK, RUN_PARTIAL):
|
|
totals = dict(current.get("totals") or {})
|
|
for kind, counts in report.pushed.items():
|
|
for status, number in counts.items():
|
|
key = f"{kind}_{status}"
|
|
totals[key] = int(totals.get(key, 0)) + number
|
|
self.store.update(
|
|
row["uid"],
|
|
watermark=report.watermark or current.get("watermark") or {},
|
|
totals=totals,
|
|
run_count=int(current.get("run_count") or 0) + 1,
|
|
failure_count=0,
|
|
last_status=report.status,
|
|
last_error="; ".join(report.errors)[:1000],
|
|
last_run_at=iso(now()),
|
|
)
|
|
self.store.schedule_next(row["uid"], int(current.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(current).jitter_percent)
|
|
elif current["status"] != STATUS_LOGIN:
|
|
failed = self.store.fail(current, "; ".join(report.errors) or "run failed")
|
|
self.store.schedule_next(row["uid"], int(failed.get("interval_seconds") or LIMITS.default_interval_seconds), SyncOptions.from_row(failed).jitter_percent)
|
|
logger.error(
|
|
"run: failure %d of %d for %s. Structure drift is likely: re-run discovery with relearn.",
|
|
failed["failure_count"], LIMITS.max_failures, row["slug"],
|
|
)
|