Molodetz blogrol volgens DPP-template
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
@@ -0,0 +1,204 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import tarfile
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import brotli
|
||||
import zstandard
|
||||
|
||||
from molodetz import config
|
||||
from molodetz.database import (
|
||||
admin_uids_ordered,
|
||||
delete_backup_rows,
|
||||
enqueue_backup,
|
||||
list_backups,
|
||||
now_iso,
|
||||
parse_iso,
|
||||
pending_backups,
|
||||
shard_path,
|
||||
update_backup,
|
||||
)
|
||||
from molodetz.services.base import BaseService, ConfigField
|
||||
from molodetz.utils.audit import record_system
|
||||
from molodetz.utils.notifications import create_notification
|
||||
|
||||
TARGETS = ("database", "uploads", "keys", "full")
|
||||
|
||||
|
||||
def database_file():
|
||||
prefix = "sqlite:///"
|
||||
if not config.DATABASE_URL.startswith(prefix):
|
||||
raise RuntimeError("backups support SQLite databases only")
|
||||
return Path(config.DATABASE_URL[len(prefix):])
|
||||
|
||||
|
||||
def _snapshot_database(staging):
|
||||
target = staging / "molodetz.db"
|
||||
source = sqlite3.connect(str(database_file()))
|
||||
try:
|
||||
destination = sqlite3.connect(str(target))
|
||||
try:
|
||||
source.backup(destination)
|
||||
finally:
|
||||
destination.close()
|
||||
finally:
|
||||
source.close()
|
||||
return target
|
||||
|
||||
|
||||
def build_archive(uid, target, codec="zstd"):
|
||||
if target not in TARGETS:
|
||||
raise ValueError(f"unknown backup target {target}")
|
||||
staging = config.DATA_PATHS["backup_staging"] / uid
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
buffer = io.BytesIO()
|
||||
with tarfile.open(fileobj=buffer, mode="w") as archive:
|
||||
if target in ("database", "full"):
|
||||
archive.add(_snapshot_database(staging), arcname="molodetz.db")
|
||||
if target in ("uploads", "full") and config.DATA_PATHS["uploads"].exists():
|
||||
archive.add(config.DATA_PATHS["uploads"], arcname="uploads")
|
||||
if target in ("keys", "full") and config.DATA_PATHS["keys"].exists():
|
||||
archive.add(config.DATA_PATHS["keys"], arcname="keys")
|
||||
raw = buffer.getvalue()
|
||||
if codec == "brotli":
|
||||
data = brotli.compress(raw, quality=6)
|
||||
suffix = "tar.br"
|
||||
else:
|
||||
data = zstandard.ZstdCompressor(level=10).compress(raw)
|
||||
suffix = "tar.zst"
|
||||
checksum = hashlib.sha256(data).hexdigest()[:16]
|
||||
directory = f"{shard_path(uid)}"
|
||||
stored_name = f"{target}-{uid.replace('-', '')}-{checksum}.{suffix}"
|
||||
path = config.DATA_PATHS["backups"] / directory / stored_name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
return directory, stored_name, len(data)
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def backup_path(row):
|
||||
root = config.DATA_PATHS["backups"].resolve()
|
||||
path = (root / row["directory"] / row["stored_name"]).resolve()
|
||||
if root not in path.parents:
|
||||
raise ValueError("backup path escapes the backup root")
|
||||
return path
|
||||
|
||||
|
||||
def prune_backups(keep_last, dry_run=False):
|
||||
done = [row for row in list_backups(limit=10000) if row["status"] == "done"]
|
||||
victims = done[keep_last:]
|
||||
if not dry_run:
|
||||
for row in victims:
|
||||
try:
|
||||
backup_path(row).unlink(missing_ok=True)
|
||||
except ValueError:
|
||||
pass
|
||||
delete_backup_rows([row["uid"] for row in victims])
|
||||
return len(victims)
|
||||
|
||||
|
||||
def clear_backups(dry_run=False):
|
||||
rows = list_backups(limit=100000)
|
||||
if not dry_run:
|
||||
for row in rows:
|
||||
if row.get("directory") and row.get("stored_name"):
|
||||
try:
|
||||
backup_path(row).unlink(missing_ok=True)
|
||||
except ValueError:
|
||||
pass
|
||||
delete_backup_rows([row["uid"] for row in rows])
|
||||
return len(rows)
|
||||
|
||||
|
||||
def offload(path):
|
||||
remote = config.BACKUP_OFFLOAD_REMOTE
|
||||
if not remote or shutil.which(config.RCLONE_BIN) is None:
|
||||
return "skipped"
|
||||
result = subprocess.run(
|
||||
[config.RCLONE_BIN, "--config", config.RCLONE_CONFIG, "copy", str(path), remote],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=900,
|
||||
)
|
||||
return "ok" if result.returncode == 0 else f"failed: {result.stderr.strip()[:200]}"
|
||||
|
||||
|
||||
def process_backup(row, codec):
|
||||
try:
|
||||
directory, stored_name, size = build_archive(row["uid"], row["target"], codec)
|
||||
except Exception as exc:
|
||||
update_backup(row["uid"], status="failed", error=str(exc)[:500], finished_at=now_iso())
|
||||
record_system("backup.failed", actor_kind="service", origin="service", result="error", message=str(exc)[:200])
|
||||
return False
|
||||
update_backup(row["uid"], status="done", directory=directory, stored_name=stored_name, size_bytes=size, finished_at=now_iso())
|
||||
record_system("backup.finished", actor_kind="service", origin="service", payload={"uid": row["uid"], "size": size})
|
||||
return True
|
||||
|
||||
|
||||
class BackupService(BaseService):
|
||||
name = "backup"
|
||||
title = "Back-ups"
|
||||
description = "Verwerkt de back-upwachtrij, plant automatische back-ups en snoeit oude archieven."
|
||||
default_interval = 10
|
||||
min_interval = 5
|
||||
metrics_interval = 30
|
||||
config_fields = [
|
||||
ConfigField("schedule_hours", "int", 24, "Automatisch elke N uur (0 = uit)", minimum=0, maximum=720, group="Planning"),
|
||||
ConfigField("schedule_target", "select", "database", "Automatisch doel", options=list(TARGETS), group="Planning"),
|
||||
ConfigField("keep_last", "int", 14, "Bewaar laatste N", minimum=1, maximum=1000, group="Bewaren"),
|
||||
ConfigField("codec", "select", "zstd", "Compressie", options=["zstd", "brotli"], group="Archief"),
|
||||
ConfigField("offload", "bool", "0", "Offload via rclone", group="Archief"),
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.processed = 0
|
||||
|
||||
def _schedule_due(self, hours):
|
||||
if hours <= 0:
|
||||
return False
|
||||
rows = list_backups(limit=1)
|
||||
if not rows:
|
||||
return True
|
||||
last = parse_iso(rows[0]["created_at"])
|
||||
return last is None or datetime.now(timezone.utc) - last >= timedelta(hours=hours)
|
||||
|
||||
async def run_once(self):
|
||||
cfg = self.get_config()
|
||||
if self._schedule_due(int(cfg["schedule_hours"])):
|
||||
enqueue_backup(cfg["schedule_target"], "system")
|
||||
self.log(f"automatische back-up ingepland: {cfg['schedule_target']}")
|
||||
for row in pending_backups():
|
||||
update_backup(row["uid"], status="running")
|
||||
ok = await asyncio.to_thread(process_backup, row, cfg["codec"])
|
||||
self.processed += 1
|
||||
self.log(f"back-up {row['target']} {'klaar' if ok else 'mislukt'}")
|
||||
primary = admin_uids_ordered()[:1]
|
||||
for uid in primary:
|
||||
create_notification(uid, "backup.finished" if ok else "backup.failed", f"Back-up {row['target']} {'klaar' if ok else 'mislukt'}", "/admin/backups")
|
||||
if ok and cfg["offload"] == "1":
|
||||
done = [item for item in list_backups(limit=5) if item["uid"] == row["uid"]]
|
||||
if done:
|
||||
self.log(f"offload: {offload(backup_path(done[0]))}")
|
||||
pruned = prune_backups(int(cfg["keep_last"]))
|
||||
if pruned:
|
||||
self.log(f"{pruned} oude back-ups gesnoeid")
|
||||
|
||||
def collect_metrics(self):
|
||||
rows = list_backups(limit=200)
|
||||
return {
|
||||
"cards": [
|
||||
{"label": "Archieven", "value": sum(1 for row in rows if row["status"] == "done")},
|
||||
{"label": "Wachtrij", "value": sum(1 for row in rows if row["status"] == "pending")},
|
||||
{"label": "Verwerkt sinds start", "value": self.processed},
|
||||
],
|
||||
"table": None,
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from collections import deque
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from molodetz.database import (
|
||||
ensure_service_state,
|
||||
get_service_state,
|
||||
get_setting,
|
||||
now_iso,
|
||||
parse_iso,
|
||||
set_setting,
|
||||
write_service_state,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEARTBEAT_SECONDS = 8
|
||||
LIVENESS_SECONDS = 15
|
||||
METRICS_SECONDS = 15
|
||||
FIELD_TYPES = ("int", "str", "url", "text", "password", "bool", "select")
|
||||
|
||||
|
||||
class ConfigField:
|
||||
def __init__(self, key, type="str", default="", label="", help="", minimum=None, maximum=None, options=None, secret=False, group="Algemeen"):
|
||||
if type not in FIELD_TYPES:
|
||||
raise ValueError(f"unknown field type {type}")
|
||||
self.key = key
|
||||
self.type = type
|
||||
self.default = default
|
||||
self.label = label or key
|
||||
self.help = help
|
||||
self.minimum = minimum
|
||||
self.maximum = maximum
|
||||
self.options = options or []
|
||||
self.secret = secret or type == "password"
|
||||
self.group = group
|
||||
|
||||
def coerce(self, raw):
|
||||
value = "" if raw is None else str(raw).strip()
|
||||
if self.type == "int":
|
||||
number = int(value)
|
||||
if self.minimum is not None and number < self.minimum:
|
||||
raise ValueError(f"{self.label} must be at least {self.minimum}")
|
||||
if self.maximum is not None and number > self.maximum:
|
||||
raise ValueError(f"{self.label} must be at most {self.maximum}")
|
||||
return str(number)
|
||||
if self.type == "bool":
|
||||
if value not in ("0", "1"):
|
||||
raise ValueError(f"{self.label} must be 0 or 1")
|
||||
return value
|
||||
if self.type == "select":
|
||||
if value not in [str(option) for option in self.options]:
|
||||
raise ValueError(f"{self.label} must be one of {self.options}")
|
||||
return value
|
||||
if self.type == "url" and value and not value.startswith(("http://", "https://")):
|
||||
raise ValueError(f"{self.label} must be an http(s) URL")
|
||||
return value
|
||||
|
||||
def read(self, service_name):
|
||||
raw = get_setting(f"service_{service_name}_{self.key}", str(self.default))
|
||||
try:
|
||||
return self.coerce(raw)
|
||||
except (TypeError, ValueError):
|
||||
return str(self.default)
|
||||
|
||||
def spec(self, service_name):
|
||||
value = self.read(service_name)
|
||||
return {
|
||||
"key": self.key,
|
||||
"type": self.type,
|
||||
"label": self.label,
|
||||
"help": self.help,
|
||||
"group": self.group,
|
||||
"options": self.options,
|
||||
"minimum": self.minimum,
|
||||
"maximum": self.maximum,
|
||||
"secret": self.secret,
|
||||
"value": "" if self.secret and value else value,
|
||||
"has_value": bool(value),
|
||||
}
|
||||
|
||||
|
||||
class BaseService:
|
||||
name = "base"
|
||||
title = "Service"
|
||||
description = ""
|
||||
default_enabled = True
|
||||
default_interval = 60
|
||||
min_interval = 1
|
||||
config_fields: list[ConfigField] = []
|
||||
metrics_interval = METRICS_SECONDS
|
||||
|
||||
def __init__(self):
|
||||
self.logs = deque(maxlen=self.log_size())
|
||||
self._task = None
|
||||
self._row_id = None
|
||||
self._last_heartbeat = 0.0
|
||||
self._last_metrics = 0.0
|
||||
self._last_command = None
|
||||
self._was_enabled = None
|
||||
self._next_run = None
|
||||
|
||||
@property
|
||||
def interval_key(self):
|
||||
return f"service_{self.name}_interval"
|
||||
|
||||
def fields(self):
|
||||
framework = [
|
||||
ConfigField("enabled", "bool", "1" if self.default_enabled else "0", "Ingeschakeld", group="Algemeen"),
|
||||
ConfigField("interval", "int", self.default_interval, "Interval (s)", minimum=self.min_interval, group="Algemeen"),
|
||||
]
|
||||
tail = [ConfigField("log_size", "int", 200, "Logregels", minimum=10, maximum=5000, group="Logs")]
|
||||
return framework + list(self.config_fields) + tail
|
||||
|
||||
def field(self, key):
|
||||
for field in self.fields():
|
||||
if field.key == key:
|
||||
return field
|
||||
raise KeyError(key)
|
||||
|
||||
def get_config(self):
|
||||
return {field.key: field.read(self.name) for field in self.fields()}
|
||||
|
||||
def is_enabled(self):
|
||||
return self.field("enabled").read(self.name) == "1"
|
||||
|
||||
def current_interval(self):
|
||||
return max(self.min_interval, int(self.field("interval").read(self.name)))
|
||||
|
||||
def log_size(self):
|
||||
try:
|
||||
return max(10, int(get_setting(f"service_{self.name}_log_size", "200")))
|
||||
except ValueError:
|
||||
return 200
|
||||
|
||||
def log(self, message):
|
||||
line = f"{datetime.now(timezone.utc).strftime('%d/%m/%Y %H:%M:%S')} {message}"
|
||||
self.logs.append(line)
|
||||
logger.info("[%s] %s", self.name, message)
|
||||
|
||||
async def run_once(self):
|
||||
raise NotImplementedError
|
||||
|
||||
async def on_enable(self):
|
||||
return None
|
||||
|
||||
async def on_disable(self):
|
||||
return None
|
||||
|
||||
def collect_metrics(self):
|
||||
return {"cards": [], "table": None}
|
||||
|
||||
def _command(self):
|
||||
return get_setting(f"service_{self.name}_command", "")
|
||||
|
||||
def _persist(self, status, force_metrics=False):
|
||||
loop_time = asyncio.get_running_loop().time()
|
||||
fields = {"status": status, "heartbeat": now_iso(), "logs": list(self.logs)}
|
||||
if self._next_run:
|
||||
fields["next_run"] = self._next_run.isoformat()
|
||||
if force_metrics or loop_time - self._last_metrics >= self.metrics_interval:
|
||||
fields["metrics"] = self.collect_metrics()
|
||||
self._last_metrics = loop_time
|
||||
write_service_state(self._row_id, **fields)
|
||||
self._last_heartbeat = loop_time
|
||||
|
||||
async def _tick(self):
|
||||
enabled = self.is_enabled()
|
||||
if enabled != self._was_enabled:
|
||||
if enabled:
|
||||
write_service_state(self._row_id, started_at=now_iso())
|
||||
await self.on_enable()
|
||||
self.log("ingeschakeld")
|
||||
elif self._was_enabled is not None:
|
||||
await self.on_disable()
|
||||
self.log("uitgeschakeld")
|
||||
self._was_enabled = enabled
|
||||
self._persist("running" if enabled else "stopped", force_metrics=True)
|
||||
command = self._command()
|
||||
run_now = False
|
||||
if command and command != self._last_command:
|
||||
if self._last_command is not None:
|
||||
verb = command.split(":", 1)[0]
|
||||
if verb == "run":
|
||||
run_now = True
|
||||
elif verb == "clear":
|
||||
self.logs.clear()
|
||||
self._last_command = command
|
||||
if self.logs.maxlen != self.log_size():
|
||||
self.logs = deque(self.logs, maxlen=self.log_size())
|
||||
now = datetime.now(timezone.utc)
|
||||
if enabled and (run_now or self._next_run is None or now >= self._next_run):
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception as exc:
|
||||
self.log(f"fout: {exc}")
|
||||
logger.exception("service %s failed", self.name)
|
||||
self._next_run = datetime.now(timezone.utc) + timedelta(seconds=self.current_interval())
|
||||
write_service_state(self._row_id, last_run=now_iso())
|
||||
self._persist("running", force_metrics=True)
|
||||
elif asyncio.get_running_loop().time() - self._last_heartbeat >= HEARTBEAT_SECONDS:
|
||||
self._persist("running" if enabled else "stopped")
|
||||
|
||||
async def loop(self):
|
||||
self._row_id = ensure_service_state(self.name)
|
||||
self._last_command = None
|
||||
while True:
|
||||
try:
|
||||
await self._tick()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("service loop %s crashed a tick", self.name)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
def start(self):
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.get_running_loop().create_task(self.loop())
|
||||
|
||||
async def stop(self):
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._task = None
|
||||
if self._row_id is not None:
|
||||
write_service_state(self._row_id, status="stopped")
|
||||
|
||||
|
||||
def derived_status(service, row):
|
||||
if not service.is_enabled():
|
||||
return "stopped"
|
||||
if row is None:
|
||||
return "stalled"
|
||||
heartbeat = parse_iso(row.get("heartbeat"))
|
||||
if heartbeat and (datetime.now(timezone.utc) - heartbeat).total_seconds() <= LIVENESS_SECONDS:
|
||||
return "running"
|
||||
return "stalled"
|
||||
|
||||
|
||||
class ServiceManager:
|
||||
def __init__(self):
|
||||
self.services = {}
|
||||
self.supervising = False
|
||||
|
||||
def register(self, service):
|
||||
self.services[service.name] = service
|
||||
return service
|
||||
|
||||
def get(self, name):
|
||||
return self.services.get(name)
|
||||
|
||||
def describe(self, service):
|
||||
row = get_service_state(service.name)
|
||||
metrics = {}
|
||||
logs = []
|
||||
if row:
|
||||
try:
|
||||
metrics = json.loads(row.get("metrics") or "{}")
|
||||
except ValueError:
|
||||
metrics = {}
|
||||
try:
|
||||
logs = json.loads(row.get("logs") or "[]")
|
||||
except ValueError:
|
||||
logs = []
|
||||
return {
|
||||
"name": service.name,
|
||||
"title": service.title,
|
||||
"description": service.description,
|
||||
"status": derived_status(service, row),
|
||||
"enabled": service.is_enabled(),
|
||||
"last_run": row.get("last_run") if row else None,
|
||||
"next_run": row.get("next_run") if row else None,
|
||||
"heartbeat": row.get("heartbeat") if row else None,
|
||||
"metrics": metrics,
|
||||
"logs": logs,
|
||||
"fields": [field.spec(service.name) for field in service.fields()],
|
||||
}
|
||||
|
||||
def describe_all(self):
|
||||
return [self.describe(service) for service in self.services.values()]
|
||||
|
||||
def set_enabled(self, name, enabled):
|
||||
set_setting(f"service_{name}_enabled", "1" if enabled else "0")
|
||||
|
||||
def send_command(self, name, verb):
|
||||
current = get_setting(f"service_{name}_command", "")
|
||||
counter = 0
|
||||
if ":" in current:
|
||||
try:
|
||||
counter = int(current.split(":", 1)[1])
|
||||
except ValueError:
|
||||
counter = 0
|
||||
set_setting(f"service_{name}_command", f"{verb}:{counter + 1}")
|
||||
|
||||
def save_config(self, name, values):
|
||||
service = self.services[name]
|
||||
updates = {}
|
||||
for field in service.fields():
|
||||
if field.key not in values:
|
||||
continue
|
||||
raw = values[field.key]
|
||||
if field.secret and (raw is None or str(raw).strip() == ""):
|
||||
continue
|
||||
updates[field.key] = field.coerce(raw)
|
||||
for key, value in updates.items():
|
||||
set_setting(f"service_{name}_{key}", value)
|
||||
return updates
|
||||
|
||||
def supervise(self):
|
||||
self.supervising = True
|
||||
for service in self.services.values():
|
||||
service.start()
|
||||
|
||||
async def shutdown_all(self):
|
||||
for service in self.services.values():
|
||||
await service.stop()
|
||||
self.supervising = False
|
||||
|
||||
|
||||
manager = ServiceManager()
|
||||
@@ -0,0 +1,36 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from molodetz.database import count_audit, get_int_setting, prune_audit, prune_expired_sessions
|
||||
from molodetz.services.base import BaseService
|
||||
|
||||
|
||||
class HousekeepingService(BaseService):
|
||||
name = "housekeeping"
|
||||
title = "Housekeeping"
|
||||
description = "Ruimt verlopen sessies op en snoeit het auditlog volgens de bewaartermijn."
|
||||
default_interval = 3600
|
||||
min_interval = 60
|
||||
metrics_interval = 300
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.last_sessions = 0
|
||||
self.last_audit = 0
|
||||
|
||||
async def run_once(self):
|
||||
self.last_sessions = prune_expired_sessions()
|
||||
days = max(1, get_int_setting("audit_retention_days", 365))
|
||||
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat()
|
||||
self.last_audit = prune_audit(cutoff)
|
||||
self.log(f"sessies verwijderd: {self.last_sessions}, auditregels gesnoeid: {self.last_audit}")
|
||||
|
||||
def collect_metrics(self):
|
||||
return {
|
||||
"cards": [
|
||||
{"label": "Auditregels", "value": count_audit()},
|
||||
{"label": "Laatste sessie-opruiming", "value": self.last_sessions},
|
||||
{"label": "Laatste auditsnoei", "value": self.last_audit},
|
||||
],
|
||||
"table": None,
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from molodetz import config
|
||||
from molodetz.services.base import BaseService
|
||||
from molodetz.utils.presence import compute_roster
|
||||
|
||||
|
||||
class PresenceService(BaseService):
|
||||
name = "presence"
|
||||
title = "Presence"
|
||||
description = "Publiceert de online-set uit de laatste activiteit van accounts."
|
||||
default_interval = 5
|
||||
min_interval = 2
|
||||
metrics_interval = 5
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.roster = []
|
||||
|
||||
async def run_once(self):
|
||||
self.roster = compute_roster()
|
||||
|
||||
def collect_metrics(self):
|
||||
return {
|
||||
"roster": self.roster,
|
||||
"cards": [
|
||||
{"label": "Online", "value": len(self.roster[: config.PRESENCE_ONLINE_LIMIT])},
|
||||
{"label": "Gevolgd", "value": len(self.roster)},
|
||||
],
|
||||
"table": None,
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BackgroundQueue:
|
||||
def __init__(self, max_size=10000):
|
||||
self.max_size = max_size
|
||||
self._queue = None
|
||||
self._task = None
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._task is not None and not self._task.done()
|
||||
|
||||
def start(self):
|
||||
if self.running:
|
||||
return
|
||||
self._queue = asyncio.Queue(maxsize=self.max_size)
|
||||
self._task = asyncio.get_running_loop().create_task(self._consume())
|
||||
|
||||
def submit(self, fn, *args, **kwargs):
|
||||
if not self.running:
|
||||
self._run(fn, args, kwargs)
|
||||
return
|
||||
try:
|
||||
self._queue.put_nowait((fn, args, kwargs))
|
||||
except asyncio.QueueFull:
|
||||
self._run(fn, args, kwargs)
|
||||
|
||||
async def stop(self):
|
||||
if self._task is None:
|
||||
return
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._task = None
|
||||
self.drain()
|
||||
|
||||
def drain(self):
|
||||
if self._queue is None:
|
||||
return
|
||||
while not self._queue.empty():
|
||||
fn, args, kwargs = self._queue.get_nowait()
|
||||
self._run(fn, args, kwargs)
|
||||
|
||||
async def _consume(self):
|
||||
while True:
|
||||
fn, args, kwargs = await self._queue.get()
|
||||
self._run(fn, args, kwargs)
|
||||
self._queue.task_done()
|
||||
|
||||
@staticmethod
|
||||
def _run(fn, args, kwargs):
|
||||
try:
|
||||
fn(*args, **kwargs)
|
||||
except Exception:
|
||||
logger.exception("background task %s failed", getattr(fn, "__name__", fn))
|
||||
|
||||
|
||||
background = BackgroundQueue()
|
||||
@@ -0,0 +1,10 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
from molodetz.services.backup import BackupService
|
||||
from molodetz.services.housekeeping import HousekeepingService
|
||||
from molodetz.services.presence import PresenceService
|
||||
|
||||
SERVICE_CLASSES = (PresenceService, HousekeepingService, BackupService)
|
||||
|
||||
|
||||
def build_services():
|
||||
return [cls() for cls in SERVICE_CLASSES]
|
||||
Reference in New Issue
Block a user