ticket #73 attempt 1

devplacepy/services/backup/service.py

@@ -125,6 +125,15 @@ class BackupService(JobService):
             summary=f"backup {target} completed ({_human_bytes(stats['bytes_out'])})",
             links=[audit.job(job_uid)],
         )
+        if schedule_uid:
+            try:
+                store.check_disk_and_alert()
+            except Exception as exc:
+                logger.warning("disk alert check failed: %s", exc)
+            try:
+                store.check_db_growth()
+            except Exception as exc:
+                logger.warning("db growth check failed: %s", exc)
         return {
             "backup_uid": backup_uid,
             "target": target,

devplacepy/services/backup/store.py

@@ -1,5 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
+import json
+import logging
 import shutil
 import time
 from datetime import datetime, timezone
@@ -9,6 +11,8 @@ from devplacepy import config
 from devplacepy.database import _index, db, get_table
 from devplacepy.utils import generate_uid
 
+logger = logging.getLogger(__name__)
+
 BACKUP_TARGETS: dict[str, dict] = {
     "database": {
         "label": "Database",
@@ -487,3 +491,131 @@ def compute_storage_stats() -> dict:
     _storage_cache["data"] = data
     _storage_cache["at"] = now
     return data
+
+
+DB_GROWTH_HISTORY_FILE = None
+
+
+def _db_growth_path() -> Path:
+    global DB_GROWTH_HISTORY_FILE
+    if DB_GROWTH_HISTORY_FILE is None:
+        DB_GROWTH_HISTORY_FILE = config.BACKUPS_DIR / "db_growth_history.json"
+    return DB_GROWTH_HISTORY_FILE
+
+
+def _load_db_sizes() -> list[dict]:
+    path = _db_growth_path()
+    if not path.exists():
+        return []
+    try:
+        return json.loads(path.read_text())
+    except (ValueError, OSError):
+        return []
+
+
+def _save_db_sizes(sizes: list[dict]) -> None:
+    path = _db_growth_path()
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(sizes))
+    except OSError as exc:
+        logger.warning("failed to write db growth history: %s", exc)
+
+
+def _database_file_size() -> int:
+    database_file = Path(str(config.DATABASE_URL).replace("sqlite:///", ""))
+    try:
+        return database_file.stat().st_size if database_file.exists() else 0
+    except OSError:
+        return 0
+
+
+def check_db_growth() -> None:
+    from devplacepy.services.audit import record as audit
+
+    current = _database_file_size()
+    if current <= 0:
+        return
+
+    sizes = _load_db_sizes()
+    last_entry = sizes[-1] if sizes else None
+    now = now_iso()
+
+    sizes.append({"at": now, "size_bytes": current})
+    _save_db_sizes(sizes)
+
+    if last_entry and last_entry.get("size_bytes", 0) > 0:
+        try:
+            last_at = datetime.fromisoformat(last_entry["at"])
+            current_at = datetime.fromisoformat(now)
+            days = (current_at - last_at).total_seconds() / 86400
+            if days > 0:
+                growth_pct = (current - last_entry["size_bytes"]) / last_entry["size_bytes"]
+                daily_pct = growth_pct / days
+                if daily_pct > 0.10:
+                    logger.warning(
+                        "database growth %.1f%%/day (%.0f MB -> %.0f MB over %.1f days)",
+                        daily_pct * 100,
+                        last_entry["size_bytes"] / (1024 * 1024),
+                        current / (1024 * 1024),
+                        days,
+                    )
+                    try:
+                        audit.record_system(
+                            "backup.db_growth_high",
+                            actor_kind="service",
+                            origin="scheduler",
+                            target_type="system",
+                            metadata={
+                                "size_bytes": current,
+                                "previous_size_bytes": last_entry["size_bytes"],
+                                "daily_growth_pct": round(daily_pct * 100, 1),
+                                "span_days": round(days, 2),
+                            },
+                            summary=(
+                                f"database growth {daily_pct * 100:.1f}%/day "
+                                f"({human_bytes(last_entry['size_bytes'])} -> {human_bytes(current)})"
+                            ),
+                        )
+                    except Exception as exc:
+                        logger.warning("db growth audit failed: %s", exc)
+        except (ValueError, TypeError) as exc:
+            logger.warning("failed to parse db growth dates: %s", exc)
+
+
+DISK_ALERT_THRESHOLD = 85
+
+
+def check_disk_and_alert() -> None:
+    from devplacepy.services.audit import record as audit
+
+    stats = compute_storage_stats()
+    disk = stats.get("disk", {})
+    used_pct = disk.get("used_percent", 0.0)
+    if used_pct < DISK_ALERT_THRESHOLD:
+        return
+    logger.warning(
+        "disk usage at %.1f%% (threshold %d%%)",
+        used_pct,
+        DISK_ALERT_THRESHOLD,
+    )
+    try:
+        audit.record_system(
+            "backup.disk.critical",
+            actor_kind="service",
+            origin="scheduler",
+            target_type="system",
+            metadata={
+                "used_percent": used_pct,
+                "used_bytes": disk.get("used_bytes", 0),
+                "total_bytes": disk.get("total_bytes", 0),
+                "free_bytes": disk.get("free_bytes", 0),
+            },
+            summary=(
+                f"disk usage at {used_pct:.1f}% "
+                f"({human_bytes(disk.get('used_bytes', 0))} / "
+                f"{human_bytes(disk.get('total_bytes', 0))})"
+            ),
+        )
+    except Exception as exc:
+        logger.warning("disk alert audit failed: %s", exc)

devplacepy/services/containers/service.py

@@ -81,6 +81,7 @@ class ContainerService(BaseService):
     def __init__(self):
         super().__init__(name="containers", interval_seconds=5)
         self._metric_tick = 0
+        self._orphan_sweep_tick = 0
         self._booted = False
         self._last_sync_at = 0.0
 
@@ -107,19 +108,6 @@ class ContainerService(BaseService):
 
         await self._periodic_sync(instances, by_uid)
 
-        for uid, row in by_uid.items():
-            if uid not in known:
-                try:
-                    await backend.rm(row.container_id, force=True)
-                    self.log(f"reaped orphan container {row.name}")
-                    _audit_reconcile(
-                        {"uid": uid, "name": row.name},
-                        "orphan_reap",
-                        f"reconciler reaped orphan container {row.name}",
-                    )
-                except Exception as exc:
-                    self.log(f"orphan rm failed for {row.name}: {exc}")
-
         for inst in instances:
             try:
                 await self._reconcile(backend, inst, by_uid.get(inst["uid"]))
@@ -128,6 +116,11 @@ class ContainerService(BaseService):
 
         await self._fire_schedules()
 
+        try:
+            await self._sweep_orphaned_containers(backend, by_uid, known)
+        except Exception as exc:
+            self.log(f"orphan sweep failed: {exc}")
+
         try:
             await self._cleanup_stale_workspaces(backend)
         except Exception as exc:
@@ -162,6 +155,28 @@ class ContainerService(BaseService):
                 self.log(f"failed to remove workspace {uid}: {exc}")
         await backend.image_prune()
 
+    async def _sweep_orphaned_containers(self, backend, by_uid: dict, known: set) -> None:
+        if self._orphan_sweep_tick % 60 != 0:
+            self._orphan_sweep_tick += 1
+            return
+        self._orphan_sweep_tick += 1
+        for uid, row in by_uid.items():
+            if uid not in known:
+                try:
+                    await backend.stop(row.container_id, timeout=10)
+                except Exception:
+                    pass
+                try:
+                    await backend.rm(row.container_id, force=True)
+                    self.log(f"removed orphan container {row.name}")
+                    _audit_reconcile(
+                        {"uid": uid, "name": row.name},
+                        "orphan_reap",
+                        f"sweeper removed orphan container {row.name}",
+                    )
+                except Exception as exc:
+                    self.log(f"orphan rm failed for {row.name}: {exc}")
+
     async def _reconcile(self, backend, inst, ps) -> None:
         uid = inst["uid"]
         desired = inst["desired_state"]

dpc.log

@@ -26,3 +26,7 @@
 2026-07-19T19:02:05 DEBUG model=molodetz-pro fps=30
 2026-07-19T19:02:05 INFO read task from file: /workspace/prompts/execution-2.txt
 2026-07-19T19:02:05 INFO settings merged: model=<default> allow=0 deny=0 ask=0
+2026-07-23T01:50:04 INFO logging initialised at /workspace/repo/dpc.log
+2026-07-23T01:50:04 DEBUG model=molodetz-pro fps=30
+2026-07-23T01:50:04 INFO read task from file: /workspace/prompts/execution-2.txt
+2026-07-23T01:50:04 INFO settings merged: model=<default> allow=0 deny=0 ask=0