diff --git a/devplacepy/services/backup/service.py b/devplacepy/services/backup/service.py index 88892f8b..23fb09be 100644 --- a/devplacepy/services/backup/service.py +++ b/devplacepy/services/backup/service.py @@ -125,6 +125,15 @@ class BackupService(JobService): summary=f"backup {target} completed ({_human_bytes(stats['bytes_out'])})", links=[audit.job(job_uid)], ) + if schedule_uid: + try: + store.check_disk_and_alert() + except Exception as exc: + logger.warning("disk alert check failed: %s", exc) + try: + store.check_db_growth() + except Exception as exc: + logger.warning("db growth check failed: %s", exc) return { "backup_uid": backup_uid, "target": target, diff --git a/devplacepy/services/backup/store.py b/devplacepy/services/backup/store.py index f0cb8bdf..4a63f818 100644 --- a/devplacepy/services/backup/store.py +++ b/devplacepy/services/backup/store.py @@ -1,5 +1,7 @@ # retoor +import json +import logging import shutil import time from datetime import datetime, timezone @@ -9,6 +11,8 @@ from devplacepy import config from devplacepy.database import _index, db, get_table from devplacepy.utils import generate_uid +logger = logging.getLogger(__name__) + BACKUP_TARGETS: dict[str, dict] = { "database": { "label": "Database", @@ -487,3 +491,131 @@ def compute_storage_stats() -> dict: _storage_cache["data"] = data _storage_cache["at"] = now return data + + +DB_GROWTH_HISTORY_FILE = None + + +def _db_growth_path() -> Path: + global DB_GROWTH_HISTORY_FILE + if DB_GROWTH_HISTORY_FILE is None: + DB_GROWTH_HISTORY_FILE = config.BACKUPS_DIR / "db_growth_history.json" + return DB_GROWTH_HISTORY_FILE + + +def _load_db_sizes() -> list[dict]: + path = _db_growth_path() + if not path.exists(): + return [] + try: + return json.loads(path.read_text()) + except (ValueError, OSError): + return [] + + +def _save_db_sizes(sizes: list[dict]) -> None: + path = _db_growth_path() + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(sizes)) + except OSError as exc: + logger.warning("failed to write db growth history: %s", exc) + + +def _database_file_size() -> int: + database_file = Path(str(config.DATABASE_URL).replace("sqlite:///", "")) + try: + return database_file.stat().st_size if database_file.exists() else 0 + except OSError: + return 0 + + +def check_db_growth() -> None: + from devplacepy.services.audit import record as audit + + current = _database_file_size() + if current <= 0: + return + + sizes = _load_db_sizes() + last_entry = sizes[-1] if sizes else None + now = now_iso() + + sizes.append({"at": now, "size_bytes": current}) + _save_db_sizes(sizes) + + if last_entry and last_entry.get("size_bytes", 0) > 0: + try: + last_at = datetime.fromisoformat(last_entry["at"]) + current_at = datetime.fromisoformat(now) + days = (current_at - last_at).total_seconds() / 86400 + if days > 0: + growth_pct = (current - last_entry["size_bytes"]) / last_entry["size_bytes"] + daily_pct = growth_pct / days + if daily_pct > 0.10: + logger.warning( + "database growth %.1f%%/day (%.0f MB -> %.0f MB over %.1f days)", + daily_pct * 100, + last_entry["size_bytes"] / (1024 * 1024), + current / (1024 * 1024), + days, + ) + try: + audit.record_system( + "backup.db_growth_high", + actor_kind="service", + origin="scheduler", + target_type="system", + metadata={ + "size_bytes": current, + "previous_size_bytes": last_entry["size_bytes"], + "daily_growth_pct": round(daily_pct * 100, 1), + "span_days": round(days, 2), + }, + summary=( + f"database growth {daily_pct * 100:.1f}%/day " + f"({human_bytes(last_entry['size_bytes'])} -> {human_bytes(current)})" + ), + ) + except Exception as exc: + logger.warning("db growth audit failed: %s", exc) + except (ValueError, TypeError) as exc: + logger.warning("failed to parse db growth dates: %s", exc) + + +DISK_ALERT_THRESHOLD = 85 + + +def check_disk_and_alert() -> None: + from devplacepy.services.audit import record as audit + + stats = compute_storage_stats() + disk = stats.get("disk", {}) + used_pct = disk.get("used_percent", 0.0) + if used_pct < DISK_ALERT_THRESHOLD: + return + logger.warning( + "disk usage at %.1f%% (threshold %d%%)", + used_pct, + DISK_ALERT_THRESHOLD, + ) + try: + audit.record_system( + "backup.disk.critical", + actor_kind="service", + origin="scheduler", + target_type="system", + metadata={ + "used_percent": used_pct, + "used_bytes": disk.get("used_bytes", 0), + "total_bytes": disk.get("total_bytes", 0), + "free_bytes": disk.get("free_bytes", 0), + }, + summary=( + f"disk usage at {used_pct:.1f}% " + f"({human_bytes(disk.get('used_bytes', 0))} / " + f"{human_bytes(disk.get('total_bytes', 0))})" + ), + ) + except Exception as exc: + logger.warning("disk alert audit failed: %s", exc) diff --git a/devplacepy/services/containers/service.py b/devplacepy/services/containers/service.py index 85204d96..2f5d0ffe 100644 --- a/devplacepy/services/containers/service.py +++ b/devplacepy/services/containers/service.py @@ -81,6 +81,7 @@ class ContainerService(BaseService): def __init__(self): super().__init__(name="containers", interval_seconds=5) self._metric_tick = 0 + self._orphan_sweep_tick = 0 self._booted = False self._last_sync_at = 0.0 @@ -107,19 +108,6 @@ class ContainerService(BaseService): await self._periodic_sync(instances, by_uid) - for uid, row in by_uid.items(): - if uid not in known: - try: - await backend.rm(row.container_id, force=True) - self.log(f"reaped orphan container {row.name}") - _audit_reconcile( - {"uid": uid, "name": row.name}, - "orphan_reap", - f"reconciler reaped orphan container {row.name}", - ) - except Exception as exc: - self.log(f"orphan rm failed for {row.name}: {exc}") - for inst in instances: try: await self._reconcile(backend, inst, by_uid.get(inst["uid"])) @@ -128,6 +116,11 @@ class ContainerService(BaseService): await self._fire_schedules() + try: + await self._sweep_orphaned_containers(backend, by_uid, known) + except Exception as exc: + self.log(f"orphan sweep failed: {exc}") + try: await self._cleanup_stale_workspaces(backend) except Exception as exc: @@ -162,6 +155,28 @@ class ContainerService(BaseService): self.log(f"failed to remove workspace {uid}: {exc}") await backend.image_prune() + async def _sweep_orphaned_containers(self, backend, by_uid: dict, known: set) -> None: + if self._orphan_sweep_tick % 60 != 0: + self._orphan_sweep_tick += 1 + return + self._orphan_sweep_tick += 1 + for uid, row in by_uid.items(): + if uid not in known: + try: + await backend.stop(row.container_id, timeout=10) + except Exception: + pass + try: + await backend.rm(row.container_id, force=True) + self.log(f"removed orphan container {row.name}") + _audit_reconcile( + {"uid": uid, "name": row.name}, + "orphan_reap", + f"sweeper removed orphan container {row.name}", + ) + except Exception as exc: + self.log(f"orphan rm failed for {row.name}: {exc}") + async def _reconcile(self, backend, inst, ps) -> None: uid = inst["uid"] desired = inst["desired_state"] diff --git a/dpc.log b/dpc.log index de366762..f21f0cb7 100644 --- a/dpc.log +++ b/dpc.log @@ -26,3 +26,7 @@ 2026-07-19T19:02:05 DEBUG model=molodetz-pro fps=30 2026-07-19T19:02:05 INFO read task from file: /workspace/prompts/execution-2.txt 2026-07-19T19:02:05 INFO settings merged: model= allow=0 deny=0 ask=0 +2026-07-23T01:50:04 INFO logging initialised at /workspace/repo/dpc.log +2026-07-23T01:50:04 DEBUG model=molodetz-pro fps=30 +2026-07-23T01:50:04 INFO read task from file: /workspace/prompts/execution-2.txt +2026-07-23T01:50:04 INFO settings merged: model= allow=0 deny=0 ask=0