Quiiz system

devplacepy/services/devii/tasks/__init__.py

@@ -2,7 +2,16 @@
 
 from .actions import TASK_ACTIONS
 from .controller import TaskController
-from .scheduler import Scheduler
+from .guards import AutomationDenied, automation_allowed
+from .scheduler import GlobalScheduler, Scheduler
 from .store import TaskStore
 
-__all__ = ["TASK_ACTIONS", "TaskController", "Scheduler", "TaskStore"]
+__all__ = [
+    "TASK_ACTIONS",
+    "AutomationDenied",
+    "GlobalScheduler",
+    "Scheduler",
+    "TaskController",
+    "TaskStore",
+    "automation_allowed",
+]

devplacepy/services/devii/tasks/actions.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from ..actions.spec import Action, Param
+from .schedule import MAX_LIFETIME_DAYS, MAX_MAX_RUNS, MIN_INTERVAL_SECONDS
 
 
 def field(
@@ -35,7 +36,7 @@ SCHEDULE_FIELDS: tuple[Param, ...] = (
     ),
     field(
         "every_seconds",
-        "For kind=interval: number of seconds between runs.",
+        f"For kind=interval: number of seconds between runs. Minimum {MIN_INTERVAL_SECONDS}.",
         kind="integer",
     ),
     field(
@@ -46,13 +47,20 @@ SCHEDULE_FIELDS: tuple[Param, ...] = (
     field(
         "cron",
         "For kind=cron: a 5-field cron expression 'minute hour day-of-month month day-of-week'. "
-        "Supports *, ranges (1-5), lists (1,3,5) and steps (*/15).",
+        "Supports *, ranges (1-5), lists (1,3,5) and steps (*/15). Two consecutive fires must be "
+        f"at least {MIN_INTERVAL_SECONDS} seconds apart.",
     ),
     field(
         "max_runs",
-        "Optional maximum number of executions; the task disables itself afterwards.",
+        "Maximum number of executions; the task disables itself afterwards. A recurring task "
+        f"without one gets a default, and the ceiling is {MAX_MAX_RUNS}.",
         kind="integer",
     ),
+    field(
+        "expires_at",
+        "Absolute UTC time in ISO 8601 after which the task stops for good. Defaults to "
+        f"{MAX_LIFETIME_DAYS} days after the first run, which is also the maximum.",
+    ),
 )
 
 TASK_ACTIONS: tuple[Action, ...] = (
@@ -80,10 +88,12 @@ TASK_ACTIONS: tuple[Action, ...] = (
         summary="Queue a prompt to run autonomously on a schedule with full tool access",
         description=(
             "The prompt is executed later by a fresh agent that has every tool available, "
-            "running inside the current authenticated session."
+            "running inside the current authenticated session. Scheduling is restricted to "
+            "administrators, is capped per owner, and every recurring task stops by itself."
         ),
         handler="task",
-        requires_auth=False,
+        requires_auth=True,
+        requires_admin=True,
         params=(
             field(
                 "prompt",
@@ -132,9 +142,13 @@ TASK_ACTIONS: tuple[Action, ...] = (
         method="LOCAL",
         path="",
         summary="Update a task's prompt, label, enabled state, or schedule",
-        description="Provide schedule fields together with kind to reschedule the task.",
+        description=(
+            "Provide schedule fields together with kind to reschedule the task. Restricted to "
+            "administrators."
+        ),
         handler="task",
-        requires_auth=False,
+        requires_auth=True,
+        requires_admin=True,
         params=(
             field("uid", "Uid of the task.", required=True),
             field("prompt", "New prompt."),
@@ -152,6 +166,7 @@ TASK_ACTIONS: tuple[Action, ...] = (
             field("start_at", "New first-run time for kind=interval."),
             field("cron", "New cron expression for kind=cron."),
             field("max_runs", "New maximum number of executions.", kind="integer"),
+            field("expires_at", "New absolute UTC expiry in ISO 8601."),
         ),
     ),
     Action(
@@ -168,8 +183,10 @@ TASK_ACTIONS: tuple[Action, ...] = (
         method="LOCAL",
         path="",
         summary="Trigger a task to execute immediately on the next scheduler tick",
+        description="Restricted to administrators.",
         handler="task",
-        requires_auth=False,
+        requires_auth=True,
+        requires_admin=True,
         params=(field("uid", "Uid of the task.", required=True),),
     ),
 )

devplacepy/services/devii/tasks/controller.py

@@ -10,6 +10,7 @@ from typing import Any
 from pydantic import ValidationError
 
 from ..errors import ToolInputError
+from .guards import AutomationDenied, max_active_per_owner, task_columns
 from .schedule import Schedule, next_run, now_utc, to_iso
 from .store import TaskStore
 
@@ -23,6 +24,7 @@ SCHEDULE_KEYS = (
     "start_at",
     "cron",
     "max_runs",
+    "expires_at",
 )
 RESULT_PREVIEW_CHARS = 500
 TRUTHY = {"1", "true", "yes", "on"}
@@ -48,6 +50,8 @@ def _serialize(row: dict[str, Any], preview: bool) -> dict[str, Any]:
         "last_run_at": row.get("last_run_at"),
         "run_count": row.get("run_count"),
         "max_runs": row.get("max_runs"),
+        "expires_at": row.get("expires_at"),
+        "failure_count": int(row.get("failure_count") or 0),
         "every_seconds": row.get("every_seconds"),
         "cron": row.get("cron"),
         "run_at": row.get("run_at"),
@@ -89,6 +93,8 @@ class TaskController:
         prompt = str(arguments.get("prompt", "")).strip()
         if not prompt:
             raise ToolInputError("create_task requires a non-empty prompt.")
+        self._require_automation()
+        self._require_capacity()
         schedule = self._build_schedule(arguments)
         reference = now_utc()
         first = schedule.first_run(reference)
@@ -104,11 +110,15 @@ class TaskController:
             "run_count": 0,
             "last_result": None,
             "last_error": None,
+            "failure_count": 0,
             "notify": 1 if _as_bool(arguments.get("notify"), default=False) else 0,
             "tz": (arguments.get("tz") or "").strip() or None,
-            **schedule.columns(),
+            **task_columns(schedule, reference),
         }
-        self._store.create(record)
+        try:
+            self._store.create(record)
+        except AutomationDenied as exc:
+            raise ToolInputError(f"Task refused: {exc.reason}.") from exc
         return json.dumps(
             {"status": "created", "task": _serialize(record, preview=True)},
             ensure_ascii=False,
@@ -134,6 +144,8 @@ class TaskController:
 
     def update_task(self, arguments: dict[str, Any]) -> str:
         row = self._require_task(arguments)
+        self._require_automation()
+        reference = now_utc()
         changes: dict[str, Any] = {}
 
         if "prompt" in arguments and arguments["prompt"] is not None:
@@ -158,9 +170,10 @@ class TaskController:
                 if key in arguments and arguments[key] is not None:
                     merged[key] = arguments[key]
             schedule = self._build_schedule(merged)
-            changes.update(schedule.columns())
-            changes["next_run_at"] = to_iso(schedule.first_run(now_utc()))
+            changes.update(task_columns(schedule, reference))
+            changes["next_run_at"] = to_iso(schedule.first_run(reference))
             changes["status"] = "pending"
+            changes["failure_count"] = 0
 
         if changes.get("enabled") and row.get("status") in (
             "done",
@@ -172,7 +185,7 @@ class TaskController:
                 schedule = self._build_schedule(
                     {key: row.get(key) for key in SCHEDULE_KEYS}
                 )
-                changes["next_run_at"] = to_iso(schedule.first_run(now_utc()))
+                changes["next_run_at"] = to_iso(schedule.first_run(reference))
         if changes.get("enabled") is False:
             changes["status"] = "disabled"
 
@@ -197,6 +210,7 @@ class TaskController:
 
     def run_task_now(self, arguments: dict[str, Any]) -> str:
         row = self._require_task(arguments)
+        self._require_automation()
         self._store.update(
             row["uid"],
             {"enabled": True, "status": "pending", "next_run_at": to_iso(now_utc())},
@@ -210,6 +224,21 @@ class TaskController:
             ensure_ascii=False,
         )
 
+    def _require_automation(self) -> None:
+        try:
+            self._store.require_automation()
+        except AutomationDenied as exc:
+            raise ToolInputError(
+                f"Scheduled tasks are restricted to administrators: {exc.reason}."
+            ) from exc
+
+    def _require_capacity(self) -> None:
+        limit = max_active_per_owner()
+        if limit > 0 and self._store.count_active() >= limit:
+            raise ToolInputError(
+                f"You already have {limit} active tasks. Delete or disable one first."
+            )
+
     def _build_schedule(self, source: dict[str, Any]) -> Schedule:
         payload = {
             key: source.get(key) for key in SCHEDULE_KEYS if source.get(key) is not None

devplacepy/services/devii/tasks/guards.py

@@ -0,0 +1,96 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+from typing import Any, Callable, Optional
+
+from .schedule import (
+    DEFAULT_MAX_RUNS,
+    MAX_LIFETIME_DAYS,
+    RECURRING_KINDS,
+    Schedule,
+    from_iso,
+    to_iso,
+)
+
+REASON_NOT_ADMIN = "owner is not an administrator"
+REASON_EXPIRED = "task lifetime expired"
+REASON_MAX_RUNS = "run limit reached"
+REASON_BUDGET = "automation budget reached"
+REASON_FAILURES = "too many consecutive failures"
+REASON_OWNER_IDLE = "owner has been inactive"
+
+BudgetProbe = Callable[[str, str], bool]
+
+FIELD_MAX_PER_OWNER = "devii_task_max_per_owner"
+DEFAULT_MAX_PER_OWNER = 10
+
+
+class AutomationDenied(Exception):
+    def __init__(self, reason: str) -> None:
+        super().__init__(reason)
+        self.reason = reason
+
+
+def automation_allowed(owner_kind: str, owner_id: str) -> bool:
+    if owner_kind != "user" or not owner_id:
+        return False
+    from devplacepy.database import get_admin_uids
+
+    return owner_id in get_admin_uids()
+
+
+def max_active_per_owner() -> int:
+    from devplacepy.database import get_int_setting
+
+    return get_int_setting(FIELD_MAX_PER_OWNER, DEFAULT_MAX_PER_OWNER)
+
+
+def task_columns(schedule: Schedule, reference: datetime) -> dict[str, Any]:
+    columns = dict(schedule.columns())
+    if schedule.kind in RECURRING_KINDS and columns.get("max_runs") is None:
+        columns["max_runs"] = DEFAULT_MAX_RUNS
+    columns["expires_at"] = to_iso(schedule.expiry(reference))
+    return columns
+
+
+def expiry_of(row: dict[str, Any]) -> Optional[datetime]:
+    for column, offset in (("expires_at", None), ("created_at", MAX_LIFETIME_DAYS)):
+        stamp = row.get(column)
+        if not stamp:
+            continue
+        try:
+            moment = from_iso(str(stamp)[:19])
+        except ValueError:
+            continue
+        return moment if offset is None else moment + timedelta(days=offset)
+    return None
+
+
+def refusal(
+    row: dict[str, Any],
+    reference: datetime,
+    budget_exceeded: Optional[BudgetProbe] = None,
+    max_failures: int = 0,
+) -> Optional[str]:
+    owner_kind = str(row.get("owner_kind") or "")
+    owner_id = str(row.get("owner_id") or "")
+    if not automation_allowed(owner_kind, owner_id):
+        return REASON_NOT_ADMIN
+
+    expiry = expiry_of(row)
+    if expiry is not None and reference >= expiry:
+        return REASON_EXPIRED
+
+    max_runs = row.get("max_runs")
+    if max_runs is not None and int(row.get("run_count") or 0) >= int(max_runs):
+        return REASON_MAX_RUNS
+
+    if max_failures > 0 and int(row.get("failure_count") or 0) >= max_failures:
+        return REASON_FAILURES
+
+    if budget_exceeded is not None and budget_exceeded(owner_kind, owner_id):
+        return REASON_BUDGET
+
+    return None

devplacepy/services/devii/tasks/schedule.py

@@ -10,7 +10,15 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida
 ISO_FORMAT = "%Y-%m-%dT%H:%M:%S"
 CRON_MINUTES_LIMIT = 525600 * 4
 
+MIN_INTERVAL_SECONDS = 900
+CRON_SPACING_SAMPLES = 48
+SPACING_WINDOW_SECONDS = 86400
+DEFAULT_MAX_RUNS = 1000
+MAX_MAX_RUNS = 10000
+MAX_LIFETIME_DAYS = 30
+
 ScheduleKind = Literal["once", "interval", "cron"]
+RECURRING_KINDS = ("interval", "cron")
 
 
 def now_utc() -> datetime:
@@ -92,12 +100,13 @@ class Schedule(BaseModel):
     kind: ScheduleKind
     run_at: Optional[datetime] = None
     delay_seconds: Optional[int] = Field(default=None, ge=1)
-    every_seconds: Optional[int] = Field(default=None, ge=1)
+    every_seconds: Optional[int] = Field(default=None, ge=MIN_INTERVAL_SECONDS)
     start_at: Optional[datetime] = None
     cron: Optional[str] = None
-    max_runs: Optional[int] = Field(default=None, ge=1)
+    max_runs: Optional[int] = Field(default=None, ge=1, le=MAX_MAX_RUNS)
+    expires_at: Optional[datetime] = None
 
-    @field_validator("run_at", "start_at")
+    @field_validator("run_at", "start_at", "expires_at")
     @classmethod
     def _ensure_utc(cls, value: Optional[datetime]) -> Optional[datetime]:
         if value is None:
@@ -115,9 +124,29 @@ class Schedule(BaseModel):
         if self.kind == "cron":
             if not self.cron:
                 raise ValueError("kind=cron requires a cron expression")
-            cron_next(self.cron, now_utc())
+            self._validate_cron_spacing(self.cron)
         return self
 
+    @staticmethod
+    def _validate_cron_spacing(expr: str) -> None:
+        start = now_utc().replace(hour=0, minute=0, second=0, microsecond=0)
+        previous = cron_next(expr, start)
+        for _ in range(CRON_SPACING_SAMPLES):
+            upcoming = cron_next(expr, previous)
+            if (upcoming - previous).total_seconds() < MIN_INTERVAL_SECONDS:
+                raise ValueError(
+                    f"cron fires more often than once every {MIN_INTERVAL_SECONDS} seconds"
+                )
+            previous = upcoming
+            if (previous - start).total_seconds() >= SPACING_WINDOW_SECONDS:
+                break
+
+    def expiry(self, reference: datetime) -> datetime:
+        ceiling = self.first_run(reference) + timedelta(days=MAX_LIFETIME_DAYS)
+        if self.expires_at is None:
+            return ceiling
+        return min(self.expires_at, ceiling)
+
     def first_run(self, reference: datetime) -> datetime:
         if self.kind == "once":
             if self.run_at is not None:
@@ -138,6 +167,7 @@ class Schedule(BaseModel):
             "start_at": to_iso(self.start_at) if self.start_at else None,
             "cron": self.cron,
             "max_runs": self.max_runs,
+            "expires_at": to_iso(self.expires_at) if self.expires_at else None,
         }
 
 

devplacepy/services/devii/tasks/scheduler.py

@@ -4,11 +4,12 @@ from __future__ import annotations
 
 import asyncio
 import logging
-from typing import Any, Awaitable, Callable
+from typing import Any, Awaitable, Callable, Optional
 
 from .controller import compute_followup
-from .schedule import now_utc, to_iso
-from .store import TaskStore
+from .guards import BudgetProbe, refusal
+from .schedule import next_run, now_utc, to_iso
+from .store import TaskStore, claim, due_rows
 
 logger = logging.getLogger("devii.tasks.scheduler")
 
@@ -33,10 +34,103 @@ def _audit_task_execute(row: dict[str, Any], result: str, error: str | None) ->
         links=[audit.task(row.get("uid"))],
     )
 
+
+def _audit_task_blocked(row: dict[str, Any], reason: str) -> None:
+    from devplacepy.services.audit import record as audit
+
+    owner_kind = row.get("owner_kind") or "user"
+    owner_id = row.get("owner_id") or ""
+    audit.record_system(
+        "devii.task.blocked",
+        actor_kind="user" if owner_kind == "user" else owner_kind,
+        actor_uid=owner_id if owner_kind == "user" else None,
+        actor_role="user" if owner_kind == "user" else owner_kind,
+        origin="scheduler",
+        via_agent=1,
+        result="denied",
+        target_type="task",
+        target_uid=row.get("uid"),
+        summary=f"Devii task {row.get('uid')} disabled - {reason}",
+        metadata={"reason": reason, "run_count": row.get("run_count")},
+        links=[audit.task(row.get("uid"))],
+    )
+
+
 PromptExecutor = Callable[[str], Awaitable[str]]
 EventCallback = Callable[[str, dict[str, Any], str], None]
+OwnerResolver = Callable[
+    [dict[str, Any]], Optional[tuple[TaskStore, PromptExecutor, EventCallback]]
+]
 
 DEFAULT_TICK_SECONDS = 1.0
+DEFAULT_MAX_CONCURRENT = 4
+DEFAULT_MAX_FAILURES = 3
+DUE_BATCH_LIMIT = 200
+
+
+def retire(store: TaskStore, row: dict[str, Any], reason: str) -> None:
+    store.update(
+        row["uid"],
+        {
+            "enabled": False,
+            "status": "disabled",
+            "next_run_at": None,
+            "last_error": reason,
+        },
+    )
+    logger.info("Task uid=%s disabled: %s", row.get("uid"), reason)
+    _audit_task_blocked(row, reason)
+
+
+async def run_row(
+    row: dict[str, Any],
+    store: TaskStore,
+    executor: PromptExecutor,
+    on_event: EventCallback,
+    max_failures: int = DEFAULT_MAX_FAILURES,
+) -> None:
+    uid = row["uid"]
+    on_event("start", row, "")
+    logger.info("Executing task uid=%s", uid)
+
+    try:
+        result = await executor(row["prompt"])
+    except Exception as exc:  # noqa: BLE001 - surfaced into the task record
+        logger.exception("Task uid=%s crashed", uid)
+        store.update(uid, _failure_changes(row, str(exc), max_failures))
+        on_event("error", row, str(exc))
+        _audit_task_execute(row, "", str(exc))
+        return
+
+    changes, finished = compute_followup(row, now_utc())
+    changes["last_result"] = result
+    changes["last_error"] = None
+    changes["failure_count"] = 0
+    store.update(uid, changes)
+    on_event("done" if not finished else "finished", row, result)
+    _audit_task_execute(row, result, None)
+
+
+def _failure_changes(
+    row: dict[str, Any], error: str, max_failures: int
+) -> dict[str, Any]:
+    failures = int(row.get("failure_count") or 0) + 1
+    changes: dict[str, Any] = {
+        "failure_count": failures,
+        "last_error": error,
+        "last_run_at": to_iso(now_utc()),
+    }
+    upcoming = next_run(
+        row.get("kind"), row.get("every_seconds"), row.get("cron"), now_utc()
+    )
+    if upcoming is None or (max_failures > 0 and failures >= max_failures):
+        changes["status"] = "error"
+        changes["enabled"] = False
+        changes["next_run_at"] = None
+    else:
+        changes["status"] = "pending"
+        changes["next_run_at"] = to_iso(upcoming)
+    return changes
 
 
 class Scheduler:
@@ -81,33 +175,105 @@ class Scheduler:
     async def _tick(self) -> None:
         now = now_utc()
         for row in self._store.due(to_iso(now)):
-            await self._execute(row)
+            reason = refusal(row, now)
+            if reason is not None:
+                retire(self._store, row, reason)
+                continue
+            if not claim(self._store.db, row["uid"]):
+                continue
+            await run_row(row, self._store, self._executor, self._on_event)
 
-    async def _execute(self, row: dict[str, Any]) -> None:
-        uid = row["uid"]
-        self._store.update(uid, {"status": "running"})
-        self._on_event("start", row, "")
-        logger.info("Executing task uid=%s", uid)
 
-        try:
-            result = await self._executor(row["prompt"])
-            changes, finished = compute_followup(row, now_utc())
-            changes["last_result"] = result
-            changes["last_error"] = None
-        except Exception as exc:  # noqa: BLE001 - surfaced into the task record
-            logger.exception("Task uid=%s crashed", uid)
-            self._store.update(
-                uid,
-                {
-                    "status": "error",
-                    "last_error": str(exc),
-                    "last_run_at": to_iso(now_utc()),
-                },
-            )
-            self._on_event("error", row, str(exc))
-            _audit_task_execute(row, "", str(exc))
+class GlobalScheduler:
+    def __init__(
+        self,
+        db: Any,
+        resolve: OwnerResolver,
+        tick_seconds: float = DEFAULT_TICK_SECONDS,
+        max_concurrent: int = DEFAULT_MAX_CONCURRENT,
+        max_failures: int = DEFAULT_MAX_FAILURES,
+        budget_exceeded: Optional[BudgetProbe] = None,
+    ) -> None:
+        self._db = db
+        self._resolve = resolve
+        self._tick_seconds = tick_seconds
+        self._max_concurrent = max_concurrent
+        self._max_failures = max_failures
+        self._budget_exceeded = budget_exceeded
+        self._loop_task: asyncio.Task[None] | None = None
+        self._running: dict[str, asyncio.Task[None]] = {}
+
+    @property
+    def running(self) -> int:
+        return len(self._running)
+
+    def configure(self, max_concurrent: int, max_failures: int) -> None:
+        self._max_concurrent = max_concurrent
+        self._max_failures = max_failures
+
+    def start(self) -> None:
+        if self._loop_task is None:
+            self._loop_task = asyncio.create_task(self._loop())
+            logger.info("Global task scheduler started")
+
+    async def stop(self) -> None:
+        if self._loop_task is None:
             return
+        self._loop_task.cancel()
+        try:
+            await self._loop_task
+        except asyncio.CancelledError:
+            pass
+        self._loop_task = None
+        for task in list(self._running.values()):
+            task.cancel()
+        self._running.clear()
+        logger.info("Global task scheduler stopped")
 
-        self._store.update(uid, changes)
-        self._on_event("done" if not finished else "finished", row, result)
-        _audit_task_execute(row, result, None)
+    async def _loop(self) -> None:
+        while True:
+            try:
+                await self._tick()
+            except Exception:  # noqa: BLE001 - the scheduler must never die
+                logger.exception("Global scheduler tick failed")
+            await asyncio.sleep(self._tick_seconds)
+
+    async def _tick(self) -> None:
+        now = now_utc()
+        free = self._max_concurrent - len(self._running)
+        for row in due_rows(self._db, to_iso(now), DUE_BATCH_LIMIT):
+            reason = refusal(row, now, self._budget_exceeded, self._max_failures)
+            if reason is not None:
+                retire(self._store_for(row), row, reason)
+                continue
+            if free <= 0:
+                continue
+            if str(row.get("owner_id") or "") in self._running:
+                continue
+            if self._dispatch(row):
+                free -= 1
+
+    def _dispatch(self, row: dict[str, Any]) -> bool:
+        resolved = self._resolve(row)
+        if resolved is None:
+            retire(self._store_for(row), row, "owner could not be resolved")
+            return False
+        store, executor, on_event = resolved
+
+        if not claim(self._db, row["uid"]):
+            return False
+
+        owner_id = str(row.get("owner_id") or "")
+        task = asyncio.create_task(
+            run_row(row, store, executor, on_event, self._max_failures)
+        )
+        self._running[owner_id] = task
+        task.add_done_callback(lambda _: self._running.pop(owner_id, None))
+        return True
+
+    def _store_for(self, row: dict[str, Any]) -> TaskStore:
+        return TaskStore(
+            self._db,
+            str(row.get("owner_kind") or "user"),
+            str(row.get("owner_id") or ""),
+        )

devplacepy/services/devii/tasks/store.py

@@ -7,6 +7,9 @@ from datetime import datetime, timezone
 from typing import Any
 
 import dataset
+import sqlalchemy
+
+from .guards import AutomationDenied, REASON_NOT_ADMIN, automation_allowed
 
 logger = logging.getLogger("devii.tasks.store")
 
@@ -18,6 +21,23 @@ INDEXED_COLUMNS = (
     ["next_run_at"],
     ["status"],
 )
+ADDED_COLUMNS = (
+    ("deleted_at", ""),
+    ("deleted_by", ""),
+    ("notify", 0),
+    ("tz", ""),
+    ("expires_at", ""),
+    ("failure_count", 0),
+)
+DUE_SQL = (
+    "SELECT * FROM devii_tasks WHERE enabled = 1 AND status = 'pending' "
+    "AND deleted_at IS NULL AND next_run_at IS NOT NULL AND next_run_at <= :now "
+    "ORDER BY next_run_at LIMIT :limit"
+)
+CLAIM_SQL = (
+    "UPDATE devii_tasks SET status = 'running' WHERE uid = :uid "
+    "AND status = 'pending' AND enabled = 1 AND deleted_at IS NULL"
+)
 
 
 def memory_db() -> Any:
@@ -27,43 +47,56 @@ def memory_db() -> Any:
 ACTIVE_STATUSES = ("pending", "running")
 
 
-def pending_owner_ids(db: Any) -> list[str]:
+def due_rows(db: Any, now_iso: str, limit: int) -> list[dict[str, Any]]:
     if TABLE not in db.tables:
         return []
-    table = db[TABLE]
-    if "owner_id" not in table.columns:
-        return []
-    rows = table.find(owner_kind="user", enabled=True, deleted_at=None)
-    owners = {
-        row["owner_id"]
-        for row in rows
-        if row.get("owner_id") and row.get("status") in ACTIVE_STATUSES
-    }
-    return sorted(owners)
+    return list(db.query(DUE_SQL, now=now_iso, limit=limit))
+
+
+def claim(db: Any, uid: str) -> bool:
+    statement = sqlalchemy.text(CLAIM_SQL)
+    with db:
+        result = db.executable.execute(statement, {"uid": uid})
+    return result.rowcount == 1
 
 
 class TaskStore:
-    def __init__(self, db: Any, owner_kind: str, owner_id: str) -> None:
+    def __init__(
+        self, db: Any, owner_kind: str, owner_id: str, operator: bool = False
+    ) -> None:
         self._db = db
         self._owner_kind = owner_kind
         self._owner_id = owner_id
+        self._operator = operator
         self._ensure_indexes()
 
     def _ensure_indexes(self) -> None:
         if TABLE not in self._db.tables:
             return
         table = self._db[TABLE]
-        if not table.has_column("deleted_at"):
-            table.create_column_by_example("deleted_at", "")
-        if not table.has_column("deleted_by"):
-            table.create_column_by_example("deleted_by", "")
-        if not table.has_column("notify"):
-            table.create_column_by_example("notify", 0)
-        if not table.has_column("tz"):
-            table.create_column_by_example("tz", "")
+        for column, example in ADDED_COLUMNS:
+            if not table.has_column(column):
+                table.create_column_by_example(column, example)
         for columns in INDEXED_COLUMNS:
             table.create_index(columns)
 
+    @property
+    def db(self) -> Any:
+        return self._db
+
+    def automation_allowed(self) -> bool:
+        if self._operator:
+            return True
+        return automation_allowed(self._owner_kind, self._owner_id)
+
+    def require_automation(self) -> None:
+        if not self.automation_allowed():
+            raise AutomationDenied(REASON_NOT_ADMIN)
+
+    def count_active(self) -> int:
+        rows = self._table.find(enabled=True, deleted_at=None, **self._scope)
+        return sum(1 for row in rows if row.get("status") in ACTIVE_STATUSES)
+
     @property
     def _table(self) -> Any:
         return self._db[TABLE]
@@ -73,8 +106,15 @@ class TaskStore:
         return {"owner_kind": self._owner_kind, "owner_id": self._owner_id}
 
     def create(self, record: dict[str, Any]) -> None:
+        self.require_automation()
         self._table.insert(
-            {"deleted_at": None, "deleted_by": None, **record, **self._scope}
+            {
+                "deleted_at": None,
+                "deleted_by": None,
+                "failure_count": 0,
+                **record,
+                **self._scope,
+            }
         )
         logger.info(
             "Task created uid=%s owner=%s/%s",
@@ -98,6 +138,8 @@ class TaskStore:
         return list(self._table.find(**criteria))
 
     def update(self, uid: str, changes: dict[str, Any]) -> None:
+        if changes.get("enabled"):
+            self.require_automation()
         changes = {**changes, "uid": uid, **self._scope}
         self._table.update(changes, ["uid", "owner_kind", "owner_id"])
         logger.debug("Task updated uid=%s changes=%s", uid, list(changes))