devplacepy/routers/admin/backups.py

@@ -1,6 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
 import logging
+import os
 from pathlib import Path
 from typing import Annotated
 
@@ -64,6 +65,7 @@ def _targets() -> list[dict]:
 def _dashboard(can_download: bool) -> dict:
     backups = [_backup_payload(row, can_download) for row in store.list_backups()]
     schedules = store.list_schedules()
+    threshold = int(os.environ.get("DISK_WARNING_PERCENT", "85"))
     return {
         "storage": store.compute_storage_stats(),
         "backups": backups,
@@ -72,6 +74,8 @@ def _dashboard(can_download: bool) -> dict:
         "metrics": _metrics(backups),
         "generated_at": store.now_iso(),
         "can_download_backups": can_download,
+        "disk_warnings": store.get_disk_warnings(threshold),
+        "disk_warning_threshold": threshold,
     }
 
 @router.get("/backups", response_class=HTMLResponse)

devplacepy/schemas/backups.py

@@ -53,6 +53,13 @@ class BackupJobOut(_Out):
     completed_at: Optional[str] = None
 
 
+class DiskWarningOut(_Out):
+    path: str = ""
+    used_pct: float = 0.0
+    total_gb: float = 0.0
+    used_gb: float = 0.0
+
+
 class BackupScheduleOut(_Out):
     uid: str = ""
     name: str = ""
@@ -78,3 +85,5 @@ class BackupDashboardOut(_Out):
     generated_at: Optional[str] = None
     can_download_backups: bool = False
     admin_section: Optional[str] = None
+    disk_warnings: list[DiskWarningOut] = []
+    disk_warning_threshold: int = 85

devplacepy/services/backup/service.py

@@ -34,6 +34,10 @@ class BackupService(JobService):
 
     def __init__(self):
         super().__init__(name="backup", interval_seconds=15)
+        try:
+            store.seed_default_schedule()
+        except Exception as exc:
+            logger.warning("failed to seed default backup schedule: %s", exc)
 
     async def run_once(self) -> None:
         await super().run_once()
@@ -121,6 +125,15 @@ class BackupService(JobService):
             summary=f"backup {target} completed ({_human_bytes(stats['bytes_out'])})",
             links=[audit.job(job_uid)],
         )
+        if schedule_uid:
+            try:
+                store.check_disk_and_alert()
+            except Exception as exc:
+                logger.warning("disk alert check failed: %s", exc)
+            try:
+                store.check_db_growth()
+            except Exception as exc:
+                logger.warning("db growth check failed: %s", exc)
         return {
             "backup_uid": backup_uid,
             "target": target,

devplacepy/services/backup/store.py

@@ -1,5 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
+import json
+import logging
 import shutil
 import time
 from datetime import datetime, timezone
@@ -9,6 +11,8 @@ from devplacepy import config
 from devplacepy.database import _index, db, get_table
 from devplacepy.utils import generate_uid
 
+logger = logging.getLogger(__name__)
+
 BACKUP_TARGETS: dict[str, dict] = {
     "database": {
         "label": "Database",
@@ -375,6 +379,58 @@ def _storage_paths() -> list[tuple[str, str, Path]]:
     ]
 
 
+def seed_default_schedule() -> None:
+    if "backup_schedules" not in db.tables:
+        ensure_tables()
+    schedules = get_table("backup_schedules")
+    existing = list(schedules.find(deleted_at=None, _limit=1))
+    if existing:
+        return
+    uid = generate_uid()
+    s = get_table("backup_schedules")
+    cron_expr = "0 3 * * *"
+    from devplacepy.services.devii.tasks.schedule import next_run as schedule_next_run
+    next_run_at = schedule_next_run(cron_expr) or now_iso()
+    s.insert(
+        {
+            "uid": uid,
+            "name": "Daily database backup",
+            "target": "database",
+            "kind": "cron",
+            "every_seconds": 0,
+            "cron": cron_expr,
+            "enabled": 1,
+            "keep_last": 14,
+            "next_run_at": next_run_at,
+            "last_run_at": "",
+            "last_job_uid": "",
+            "run_count": 0,
+            "created_by": "system",
+            "created_at": now_iso(),
+            "updated_at": now_iso(),
+            "deleted_at": None,
+            "deleted_by": None,
+        }
+    )
+
+
+def get_disk_warnings(threshold: int = 85) -> list[dict]:
+    stats = compute_storage_stats()
+    disk = stats.get("disk", {})
+    used_pct = disk.get("used_percent", 0.0)
+    warnings = []
+    if used_pct >= threshold:
+        warnings.append(
+            {
+                "path": stats.get("data_dir", {}).get("path", str(config.DATA_DIR)),
+                "used_pct": used_pct,
+                "total_gb": round(disk.get("total_bytes", 0) / (1024**3), 1),
+                "used_gb": round(disk.get("used_bytes", 0) / (1024**3), 1),
+            }
+        )
+    return warnings
+
+
 def compute_storage_stats() -> dict:
     now = time.monotonic()
     if (
@@ -435,3 +491,131 @@ def compute_storage_stats() -> dict:
     _storage_cache["data"] = data
     _storage_cache["at"] = now
     return data
+
+
+DB_GROWTH_HISTORY_FILE = None
+
+
+def _db_growth_path() -> Path:
+    global DB_GROWTH_HISTORY_FILE
+    if DB_GROWTH_HISTORY_FILE is None:
+        DB_GROWTH_HISTORY_FILE = config.BACKUPS_DIR / "db_growth_history.json"
+    return DB_GROWTH_HISTORY_FILE
+
+
+def _load_db_sizes() -> list[dict]:
+    path = _db_growth_path()
+    if not path.exists():
+        return []
+    try:
+        return json.loads(path.read_text())
+    except (ValueError, OSError):
+        return []
+
+
+def _save_db_sizes(sizes: list[dict]) -> None:
+    path = _db_growth_path()
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(sizes))
+    except OSError as exc:
+        logger.warning("failed to write db growth history: %s", exc)
+
+
+def _database_file_size() -> int:
+    database_file = Path(str(config.DATABASE_URL).replace("sqlite:///", ""))
+    try:
+        return database_file.stat().st_size if database_file.exists() else 0
+    except OSError:
+        return 0
+
+
+def check_db_growth() -> None:
+    from devplacepy.services.audit import record as audit
+
+    current = _database_file_size()
+    if current <= 0:
+        return
+
+    sizes = _load_db_sizes()
+    last_entry = sizes[-1] if sizes else None
+    now = now_iso()
+
+    sizes.append({"at": now, "size_bytes": current})
+    _save_db_sizes(sizes)
+
+    if last_entry and last_entry.get("size_bytes", 0) > 0:
+        try:
+            last_at = datetime.fromisoformat(last_entry["at"])
+            current_at = datetime.fromisoformat(now)
+            days = (current_at - last_at).total_seconds() / 86400
+            if days > 0:
+                growth_pct = (current - last_entry["size_bytes"]) / last_entry["size_bytes"]
+                daily_pct = growth_pct / days
+                if daily_pct > 0.10:
+                    logger.warning(
+                        "database growth %.1f%%/day (%.0f MB -> %.0f MB over %.1f days)",
+                        daily_pct * 100,
+                        last_entry["size_bytes"] / (1024 * 1024),
+                        current / (1024 * 1024),
+                        days,
+                    )
+                    try:
+                        audit.record_system(
+                            "backup.db_growth_high",
+                            actor_kind="service",
+                            origin="scheduler",
+                            target_type="system",
+                            metadata={
+                                "size_bytes": current,
+                                "previous_size_bytes": last_entry["size_bytes"],
+                                "daily_growth_pct": round(daily_pct * 100, 1),
+                                "span_days": round(days, 2),
+                            },
+                            summary=(
+                                f"database growth {daily_pct * 100:.1f}%/day "
+                                f"({human_bytes(last_entry['size_bytes'])} -> {human_bytes(current)})"
+                            ),
+                        )
+                    except Exception as exc:
+                        logger.warning("db growth audit failed: %s", exc)
+        except (ValueError, TypeError) as exc:
+            logger.warning("failed to parse db growth dates: %s", exc)
+
+
+DISK_ALERT_THRESHOLD = 85
+
+
+def check_disk_and_alert() -> None:
+    from devplacepy.services.audit import record as audit
+
+    stats = compute_storage_stats()
+    disk = stats.get("disk", {})
+    used_pct = disk.get("used_percent", 0.0)
+    if used_pct < DISK_ALERT_THRESHOLD:
+        return
+    logger.warning(
+        "disk usage at %.1f%% (threshold %d%%)",
+        used_pct,
+        DISK_ALERT_THRESHOLD,
+    )
+    try:
+        audit.record_system(
+            "backup.disk.critical",
+            actor_kind="service",
+            origin="scheduler",
+            target_type="system",
+            metadata={
+                "used_percent": used_pct,
+                "used_bytes": disk.get("used_bytes", 0),
+                "total_bytes": disk.get("total_bytes", 0),
+                "free_bytes": disk.get("free_bytes", 0),
+            },
+            summary=(
+                f"disk usage at {used_pct:.1f}% "
+                f"({human_bytes(disk.get('used_bytes', 0))} / "
+                f"{human_bytes(disk.get('total_bytes', 0))})"
+            ),
+        )
+    except Exception as exc:
+        logger.warning("disk alert audit failed: %s", exc)

devplacepy/services/containers/service.py

@@ -81,6 +81,7 @@ class ContainerService(BaseService):
     def __init__(self):
         super().__init__(name="containers", interval_seconds=5)
         self._metric_tick = 0
+        self._orphan_sweep_tick = 0
         self._booted = False
         self._last_sync_at = 0.0
 
@@ -107,19 +108,6 @@ class ContainerService(BaseService):
 
         await self._periodic_sync(instances, by_uid)
 
-        for uid, row in by_uid.items():
-            if uid not in known:
-                try:
-                    await backend.rm(row.container_id, force=True)
-                    self.log(f"reaped orphan container {row.name}")
-                    _audit_reconcile(
-                        {"uid": uid, "name": row.name},
-                        "orphan_reap",
-                        f"reconciler reaped orphan container {row.name}",
-                    )
-                except Exception as exc:
-                    self.log(f"orphan rm failed for {row.name}: {exc}")
-
         for inst in instances:
             try:
                 await self._reconcile(backend, inst, by_uid.get(inst["uid"]))
@@ -128,6 +116,16 @@ class ContainerService(BaseService):
 
         await self._fire_schedules()
 
+        try:
+            await self._sweep_orphaned_containers(backend, by_uid, known)
+        except Exception as exc:
+            self.log(f"orphan sweep failed: {exc}")
+
+        try:
+            await self._cleanup_stale_workspaces(backend)
+        except Exception as exc:
+            self.log(f"workspace cleanup failed: {exc}")
+
         self._metric_tick += 1
         if (
             self._metric_tick
@@ -136,6 +134,49 @@ class ContainerService(BaseService):
         ):
             await self._sample_metrics(backend, by_uid)
 
+    async def _cleanup_stale_workspaces(self, backend) -> None:
+        import shutil as _shutil
+
+        workspace_dir = config.CONTAINER_WORKSPACES_DIR
+        if not workspace_dir.is_dir():
+            return
+        instances = store.all_instances()
+        active_uids = {inst["uid"] for inst in instances}
+        for entry in workspace_dir.iterdir():
+            if not entry.is_dir():
+                continue
+            uid = entry.name
+            if uid in active_uids:
+                continue
+            try:
+                _shutil.rmtree(entry)
+                self.log(f"removed stale workspace {uid}")
+            except Exception as exc:
+                self.log(f"failed to remove workspace {uid}: {exc}")
+        await backend.image_prune()
+
+    async def _sweep_orphaned_containers(self, backend, by_uid: dict, known: set) -> None:
+        if self._orphan_sweep_tick % 60 != 0:
+            self._orphan_sweep_tick += 1
+            return
+        self._orphan_sweep_tick += 1
+        for uid, row in by_uid.items():
+            if uid not in known:
+                try:
+                    await backend.stop(row.container_id, timeout=10)
+                except Exception:
+                    pass
+                try:
+                    await backend.rm(row.container_id, force=True)
+                    self.log(f"removed orphan container {row.name}")
+                    _audit_reconcile(
+                        {"uid": uid, "name": row.name},
+                        "orphan_reap",
+                        f"sweeper removed orphan container {row.name}",
+                    )
+                except Exception as exc:
+                    self.log(f"orphan rm failed for {row.name}: {exc}")
+
     async def _reconcile(self, backend, inst, ps) -> None:
         uid = inst["uid"]
         desired = inst["desired_state"]

devplacepy/templates/admin_backups.html

@@ -3,8 +3,34 @@
 {{ super() }}
 <link rel="stylesheet" href="{{ static_url('/static/css/services.css') }}">
 <link rel="stylesheet" href="{{ static_url('/static/css/backups.css') }}">
+<style>
+.disk-warning {
+    background: #fff3cd;
+    border: 1px solid #ffc107;
+    border-radius: 6px;
+    padding: 12px 16px;
+    margin-bottom: 16px;
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    color: #856404;
+}
+.disk-warning .icon {
+    font-size: 1.4em;
+    flex-shrink: 0;
+}
+.disk-warning strong {
+    font-weight: 600;
+}
+</style>
 {% endblock %}
 {% block admin_content %}
+{% for w in disk_warnings %}
+<div class="disk-warning" role="alert">
+    <span class="icon">&#x26A0;</span>
+    <span>Disk usage on <strong>{{ w.path }}</strong> is <strong>{{ w.used_pct }}%</strong> ({{ w.used_gb }} GB / {{ w.total_gb }} GB) - above {{ disk_warning_threshold }}% threshold. Consider cleaning up old data or increasing disk capacity.</span>
+</div>
+{% endfor %}
 <div class="admin-toolbar">
     <h2>Backups</h2>
     <div class="backup-controls">

dpc.log

@@ -0,0 +1,32 @@
+2026-07-19T09:01:55 INFO logging initialised at /workspace/repo/dpc.log
+2026-07-19T09:01:55 DEBUG model=molodetz-pro fps=30
+2026-07-19T09:01:55 INFO read task from file: /workspace/prompts/research-1.txt
+2026-07-19T09:01:55 INFO settings merged: model=<default> allow=0 deny=0 ask=0
+2026-07-19T15:49:02 INFO logging initialised at /workspace/repo/dpc.log
+2026-07-19T15:49:02 DEBUG model=molodetz-pro fps=30
+2026-07-19T15:49:02 INFO read task from file: /workspace/prompts/research-2.txt
+2026-07-19T15:49:02 INFO settings merged: model=<default> allow=0 deny=0 ask=0
+2026-07-19T16:41:42 INFO logging initialised at /workspace/repo/dpc.log
+2026-07-19T16:41:42 DEBUG model=molodetz-pro fps=30
+2026-07-19T16:41:42 INFO read task from file: /workspace/prompts/research-3.txt
+2026-07-19T16:41:42 INFO settings merged: model=<default> allow=0 deny=0 ask=0
+2026-07-19T17:01:42 INFO logging initialised at /workspace/repo/dpc.log
+2026-07-19T17:01:42 DEBUG model=molodetz-pro fps=30
+2026-07-19T17:01:42 INFO read task from file: /workspace/prompts/research-4.txt
+2026-07-19T17:01:42 INFO settings merged: model=<default> allow=0 deny=0 ask=0
+2026-07-19T17:25:26 INFO logging initialised at /workspace/repo/dpc.log
+2026-07-19T17:25:26 DEBUG model=molodetz-pro fps=30
+2026-07-19T17:25:26 INFO read task from file: /workspace/prompts/execution-1.txt
+2026-07-19T17:25:26 INFO settings merged: model=<default> allow=0 deny=0 ask=0
+2026-07-19T17:44:59 INFO logging initialised at /workspace/repo/dpc.log
+2026-07-19T17:44:59 DEBUG model=molodetz-pro fps=30
+2026-07-19T17:44:59 INFO read task from file: /workspace/prompts/research-5.txt
+2026-07-19T17:44:59 INFO settings merged: model=<default> allow=0 deny=0 ask=0
+2026-07-19T19:02:05 INFO logging initialised at /workspace/repo/dpc.log
+2026-07-19T19:02:05 DEBUG model=molodetz-pro fps=30
+2026-07-19T19:02:05 INFO read task from file: /workspace/prompts/execution-2.txt
+2026-07-19T19:02:05 INFO settings merged: model=<default> allow=0 deny=0 ask=0
+2026-07-23T01:50:04 INFO logging initialised at /workspace/repo/dpc.log
+2026-07-23T01:50:04 DEBUG model=molodetz-pro fps=30
+2026-07-23T01:50:04 INFO read task from file: /workspace/prompts/execution-2.txt
+2026-07-23T01:50:04 INFO settings merged: model=<default> allow=0 deny=0 ask=0