devplacepy/devplacepy/services/community/store.py

# retoor <retoor@molodetz.nl>

from __future__ import annotations

import json
from datetime import datetime, timedelta, timezone

from devplacepy.database import db, get_table
from devplacepy.database.soft_delete import soft_delete
from devplacepy.services.community.spec import (
    LIMITS,
    STATUS_FAILING,
    STATUS_LEARNING,
    STATUS_PENDING,
    STATUS_READY,
)
from devplacepy.services.sync.selection import jittered
from devplacepy.utils.text import generate_uid

JSON_COLUMNS = ("schema", "watermark", "totals", "sync_options")
RUN_JSON_COLUMNS = ("collected", "pushed", "errors", "tally")
RUN_SUMMARY_COLUMNS = "uid, community_uid, started_at, finished_at, status, dry_run, trigger, pushed, tally, verdict, headline, items_stored, items_dropped"
ITEMS_TABLE = "community_run_items"
EDITABLE = (
    "name",
    "description",
    "rules",
    "language",
    "favicon",
    "user_count",
    "status",
    "schema",
    "watermark",
    "enabled",
    "interval_seconds",
    "next_run_at",
    "last_run_at",
    "last_status",
    "last_error",
    "run_count",
    "failure_count",
    "totals",
    "learned_at",
    "url",
    "origin",
    "login_username",
    "login_email",
    "login_password",
    "sync_options",
    "claimed_at",
)


def now() -> datetime:
    return datetime.now(timezone.utc)


def iso(moment: datetime) -> str:
    return moment.isoformat()


def decode(row: dict | None, columns: tuple[str, ...]) -> dict | None:
    if row is None:
        return None
    view = dict(row)
    for column in columns:
        raw = view.get(column)
        try:
            view[column] = json.loads(raw) if raw else ({} if column != "errors" else [])
        except ValueError:
            view[column] = {} if column != "errors" else []
    return view


def encode(fields: dict, columns: tuple[str, ...]) -> dict:
    return {
        key: json.dumps(value, ensure_ascii=False) if key in columns and not isinstance(value, str) else value
        for key, value in fields.items()
    }


class CommunityStore:
    def get(self, ref: str) -> dict | None:
        table = get_table("communities")
        row = table.find_one(slug=ref, deleted_at=None) or table.find_one(uid=ref, deleted_at=None)
        return decode(row, JSON_COLUMNS)

    def list(self) -> list[dict]:
        rows = get_table("communities").find(deleted_at=None, order_by=["name"])
        return [decode(row, JSON_COLUMNS) for row in rows]

    def pending(self, limit: int) -> list[dict]:
        horizon = iso(now() - timedelta(seconds=LIMITS.run_timeout_seconds))
        table = get_table("communities")
        rows = list(table.find(status=STATUS_PENDING, deleted_at=None, order_by=["created_at"]))
        rows += [row for row in table.find(status=STATUS_LEARNING, deleted_at=None) if (row.get("claimed_at") or "") < horizon]
        return [decode(row, JSON_COLUMNS) for row in rows[:limit]]

    def claim(self, uid: str) -> dict:
        return self.update(uid, status=STATUS_LEARNING, claimed_at=iso(now()))

    @staticmethod
    def credentials(row: dict) -> dict:
        return {
            "username": row.get("login_username") or "",
            "email": row.get("login_email") or "",
            "password": row.get("login_password") or "",
        }

    def due(self, limit: int) -> list[dict]:
        moment = iso(now())
        rows = get_table("communities").find(status=STATUS_READY, enabled=1, deleted_at=None, order_by=["next_run_at"])
        due = [row for row in rows if not row.get("next_run_at") or row["next_run_at"] <= moment]
        return [decode(row, JSON_COLUMNS) for row in due[:limit]]

    def create(self, slug: str, query: str, created_by: str, **fields) -> dict:
        row = {
            "uid": generate_uid(),
            "slug": slug,
            "query": query,
            "status": STATUS_PENDING,
            "enabled": 1,
            "interval_seconds": LIMITS.default_interval_seconds,
            "run_count": 0,
            "failure_count": 0,
            "created_at": iso(now()),
            "created_by": created_by,
            "deleted_at": None,
            "deleted_by": None,
            **fields,
        }
        get_table("communities").insert(encode(row, JSON_COLUMNS))
        return self.get(slug)

    def update(self, uid: str, **fields) -> dict:
        unknown = set(fields) - set(EDITABLE)
        if unknown:
            raise KeyError(f"not editable: {sorted(unknown)}")
        get_table("communities").update({"uid": uid, **encode(fields, JSON_COLUMNS)}, ["uid"])
        return self.get(uid)

    def delete(self, uid: str, actor: str) -> int:
        return soft_delete("communities", actor, uid=uid)

    def schedule_next(self, uid: str, interval_seconds: int, jitter_percent: int = 0) -> str:
        seconds = max(jittered(interval_seconds, jitter_percent), LIMITS.min_interval_seconds)
        moment = iso(now() + timedelta(seconds=seconds))
        get_table("communities").update({"uid": uid, "next_run_at": moment}, ["uid"])
        return moment

    def start_run(self, community_uid: str, trigger: str, dry_run: bool) -> str:
        uid = generate_uid()
        get_table("community_runs").insert(
            {
                "uid": uid,
                "community_uid": community_uid,
                "started_at": iso(now()),
                "finished_at": "",
                "status": "running",
                "dry_run": int(dry_run),
                "trigger": trigger,
                "collected": "",
                "pushed": "",
                "errors": "",
                "log": "",
            }
        )
        return uid

    def finish_run(self, uid: str, status: str, **fields) -> None:
        get_table("community_runs").update(
            {"uid": uid, "status": status, "finished_at": iso(now()), **encode(fields, RUN_JSON_COLUMNS)}, ["uid"]
        )

    def runs(self, community_uid: str, limit: int = 20) -> list[dict]:
        rows = get_table("community_runs").find(community_uid=community_uid, order_by=["-started_at"], _limit=limit)
        return [decode(row, RUN_JSON_COLUMNS) for row in rows]

    def run_summaries(self, community_uid: str, limit: int = LIMITS.runs_kept) -> list[dict]:
        if "community_runs" not in db.tables:
            return []
        rows = db.query(
            f"SELECT {RUN_SUMMARY_COLUMNS} FROM community_runs WHERE community_uid = :uid ORDER BY started_at DESC LIMIT :limit",
            uid=community_uid,
            limit=limit,
        )
        return [decode(row, ("tally", "pushed")) for row in rows]

    def run(self, run_uid: str) -> dict | None:
        return decode(get_table("community_runs").find_one(uid=run_uid), RUN_JSON_COLUMNS)

    def busy(self, community_uid: str) -> bool:
        horizon = iso(now() - timedelta(seconds=LIMITS.run_timeout_seconds))
        rows = get_table("community_runs").find(community_uid=community_uid, status="running")
        if any(row["started_at"] >= horizon for row in rows):
            return True
        row = get_table("communities").find_one(uid=community_uid)
        return bool(row and row.get("status") == STATUS_LEARNING and (row.get("claimed_at") or "") >= horizon)

    def prune_runs(self, community_uid: str) -> int:
        table = get_table("community_runs")
        ranked = list(table.find(community_uid=community_uid, order_by=["-started_at"]))
        for row in ranked[LIMITS.ledger_runs_kept :]:
            get_table(ITEMS_TABLE).delete(run_uid=row["uid"])
        stale = ranked[LIMITS.runs_kept :]
        for row in stale:
            table.delete(uid=row["uid"])
        return len(stale)

    def fail(self, row: dict, error: str) -> dict:
        failures = int(row.get("failure_count") or 0) + 1
        fields = {"failure_count": failures, "last_error": error[:1000], "last_status": "failed", "last_run_at": iso(now())}
        if failures >= LIMITS.max_failures:
            fields.update(status=STATUS_FAILING, enabled=0)
        return self.update(row["uid"], **fields)


store = CommunityStore()