diff --git a/devplacepy/routers/admin/backups.py b/devplacepy/routers/admin/backups.py index 9053ee90..da6954ea 100644 --- a/devplacepy/routers/admin/backups.py +++ b/devplacepy/routers/admin/backups.py @@ -1,6 +1,7 @@ # retoor import logging +import os from pathlib import Path from typing import Annotated @@ -64,6 +65,7 @@ def _targets() -> list[dict]: def _dashboard(can_download: bool) -> dict: backups = [_backup_payload(row, can_download) for row in store.list_backups()] schedules = store.list_schedules() + threshold = int(os.environ.get("DISK_WARNING_PERCENT", "85")) return { "storage": store.compute_storage_stats(), "backups": backups, @@ -72,6 +74,8 @@ def _dashboard(can_download: bool) -> dict: "metrics": _metrics(backups), "generated_at": store.now_iso(), "can_download_backups": can_download, + "disk_warnings": store.get_disk_warnings(threshold), + "disk_warning_threshold": threshold, } @router.get("/backups", response_class=HTMLResponse) diff --git a/devplacepy/schemas/backups.py b/devplacepy/schemas/backups.py index aa876624..455bfcba 100644 --- a/devplacepy/schemas/backups.py +++ b/devplacepy/schemas/backups.py @@ -53,6 +53,13 @@ class BackupJobOut(_Out): completed_at: Optional[str] = None +class DiskWarningOut(_Out): + path: str = "" + used_pct: float = 0.0 + total_gb: float = 0.0 + used_gb: float = 0.0 + + class BackupScheduleOut(_Out): uid: str = "" name: str = "" @@ -78,3 +85,5 @@ class BackupDashboardOut(_Out): generated_at: Optional[str] = None can_download_backups: bool = False admin_section: Optional[str] = None + disk_warnings: list[DiskWarningOut] = [] + disk_warning_threshold: int = 85 diff --git a/devplacepy/services/backup/service.py b/devplacepy/services/backup/service.py index 65678c15..23fb09be 100644 --- a/devplacepy/services/backup/service.py +++ b/devplacepy/services/backup/service.py @@ -34,6 +34,10 @@ class BackupService(JobService): def __init__(self): super().__init__(name="backup", interval_seconds=15) + try: + store.seed_default_schedule() + except Exception as exc: + logger.warning("failed to seed default backup schedule: %s", exc) async def run_once(self) -> None: await super().run_once() @@ -121,6 +125,15 @@ class BackupService(JobService): summary=f"backup {target} completed ({_human_bytes(stats['bytes_out'])})", links=[audit.job(job_uid)], ) + if schedule_uid: + try: + store.check_disk_and_alert() + except Exception as exc: + logger.warning("disk alert check failed: %s", exc) + try: + store.check_db_growth() + except Exception as exc: + logger.warning("db growth check failed: %s", exc) return { "backup_uid": backup_uid, "target": target, diff --git a/devplacepy/services/backup/store.py b/devplacepy/services/backup/store.py index 85ce14c8..4a63f818 100644 --- a/devplacepy/services/backup/store.py +++ b/devplacepy/services/backup/store.py @@ -1,5 +1,7 @@ # retoor +import json +import logging import shutil import time from datetime import datetime, timezone @@ -9,6 +11,8 @@ from devplacepy import config from devplacepy.database import _index, db, get_table from devplacepy.utils import generate_uid +logger = logging.getLogger(__name__) + BACKUP_TARGETS: dict[str, dict] = { "database": { "label": "Database", @@ -375,6 +379,58 @@ def _storage_paths() -> list[tuple[str, str, Path]]: ] +def seed_default_schedule() -> None: + if "backup_schedules" not in db.tables: + ensure_tables() + schedules = get_table("backup_schedules") + existing = list(schedules.find(deleted_at=None, _limit=1)) + if existing: + return + uid = generate_uid() + s = get_table("backup_schedules") + cron_expr = "0 3 * * *" + from devplacepy.services.devii.tasks.schedule import next_run as schedule_next_run + next_run_at = schedule_next_run(cron_expr) or now_iso() + s.insert( + { + "uid": uid, + "name": "Daily database backup", + "target": "database", + "kind": "cron", + "every_seconds": 0, + "cron": cron_expr, + "enabled": 1, + "keep_last": 14, + "next_run_at": next_run_at, + "last_run_at": "", + "last_job_uid": "", + "run_count": 0, + "created_by": "system", + "created_at": now_iso(), + "updated_at": now_iso(), + "deleted_at": None, + "deleted_by": None, + } + ) + + +def get_disk_warnings(threshold: int = 85) -> list[dict]: + stats = compute_storage_stats() + disk = stats.get("disk", {}) + used_pct = disk.get("used_percent", 0.0) + warnings = [] + if used_pct >= threshold: + warnings.append( + { + "path": stats.get("data_dir", {}).get("path", str(config.DATA_DIR)), + "used_pct": used_pct, + "total_gb": round(disk.get("total_bytes", 0) / (1024**3), 1), + "used_gb": round(disk.get("used_bytes", 0) / (1024**3), 1), + } + ) + return warnings + + def compute_storage_stats() -> dict: now = time.monotonic() if ( @@ -435,3 +491,131 @@ def compute_storage_stats() -> dict: _storage_cache["data"] = data _storage_cache["at"] = now return data + + +DB_GROWTH_HISTORY_FILE = None + + +def _db_growth_path() -> Path: + global DB_GROWTH_HISTORY_FILE + if DB_GROWTH_HISTORY_FILE is None: + DB_GROWTH_HISTORY_FILE = config.BACKUPS_DIR / "db_growth_history.json" + return DB_GROWTH_HISTORY_FILE + + +def _load_db_sizes() -> list[dict]: + path = _db_growth_path() + if not path.exists(): + return [] + try: + return json.loads(path.read_text()) + except (ValueError, OSError): + return [] + + +def _save_db_sizes(sizes: list[dict]) -> None: + path = _db_growth_path() + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(sizes)) + except OSError as exc: + logger.warning("failed to write db growth history: %s", exc) + + +def _database_file_size() -> int: + database_file = Path(str(config.DATABASE_URL).replace("sqlite:///", "")) + try: + return database_file.stat().st_size if database_file.exists() else 0 + except OSError: + return 0 + + +def check_db_growth() -> None: + from devplacepy.services.audit import record as audit + + current = _database_file_size() + if current <= 0: + return + + sizes = _load_db_sizes() + last_entry = sizes[-1] if sizes else None + now = now_iso() + + sizes.append({"at": now, "size_bytes": current}) + _save_db_sizes(sizes) + + if last_entry and last_entry.get("size_bytes", 0) > 0: + try: + last_at = datetime.fromisoformat(last_entry["at"]) + current_at = datetime.fromisoformat(now) + days = (current_at - last_at).total_seconds() / 86400 + if days > 0: + growth_pct = (current - last_entry["size_bytes"]) / last_entry["size_bytes"] + daily_pct = growth_pct / days + if daily_pct > 0.10: + logger.warning( + "database growth %.1f%%/day (%.0f MB -> %.0f MB over %.1f days)", + daily_pct * 100, + last_entry["size_bytes"] / (1024 * 1024), + current / (1024 * 1024), + days, + ) + try: + audit.record_system( + "backup.db_growth_high", + actor_kind="service", + origin="scheduler", + target_type="system", + metadata={ + "size_bytes": current, + "previous_size_bytes": last_entry["size_bytes"], + "daily_growth_pct": round(daily_pct * 100, 1), + "span_days": round(days, 2), + }, + summary=( + f"database growth {daily_pct * 100:.1f}%/day " + f"({human_bytes(last_entry['size_bytes'])} -> {human_bytes(current)})" + ), + ) + except Exception as exc: + logger.warning("db growth audit failed: %s", exc) + except (ValueError, TypeError) as exc: + logger.warning("failed to parse db growth dates: %s", exc) + + +DISK_ALERT_THRESHOLD = 85 + + +def check_disk_and_alert() -> None: + from devplacepy.services.audit import record as audit + + stats = compute_storage_stats() + disk = stats.get("disk", {}) + used_pct = disk.get("used_percent", 0.0) + if used_pct < DISK_ALERT_THRESHOLD: + return + logger.warning( + "disk usage at %.1f%% (threshold %d%%)", + used_pct, + DISK_ALERT_THRESHOLD, + ) + try: + audit.record_system( + "backup.disk.critical", + actor_kind="service", + origin="scheduler", + target_type="system", + metadata={ + "used_percent": used_pct, + "used_bytes": disk.get("used_bytes", 0), + "total_bytes": disk.get("total_bytes", 0), + "free_bytes": disk.get("free_bytes", 0), + }, + summary=( + f"disk usage at {used_pct:.1f}% " + f"({human_bytes(disk.get('used_bytes', 0))} / " + f"{human_bytes(disk.get('total_bytes', 0))})" + ), + ) + except Exception as exc: + logger.warning("disk alert audit failed: %s", exc) diff --git a/devplacepy/services/containers/service.py b/devplacepy/services/containers/service.py index 529eb4b0..2f5d0ffe 100644 --- a/devplacepy/services/containers/service.py +++ b/devplacepy/services/containers/service.py @@ -81,6 +81,7 @@ class ContainerService(BaseService): def __init__(self): super().__init__(name="containers", interval_seconds=5) self._metric_tick = 0 + self._orphan_sweep_tick = 0 self._booted = False self._last_sync_at = 0.0 @@ -107,19 +108,6 @@ class ContainerService(BaseService): await self._periodic_sync(instances, by_uid) - for uid, row in by_uid.items(): - if uid not in known: - try: - await backend.rm(row.container_id, force=True) - self.log(f"reaped orphan container {row.name}") - _audit_reconcile( - {"uid": uid, "name": row.name}, - "orphan_reap", - f"reconciler reaped orphan container {row.name}", - ) - except Exception as exc: - self.log(f"orphan rm failed for {row.name}: {exc}") - for inst in instances: try: await self._reconcile(backend, inst, by_uid.get(inst["uid"])) @@ -128,6 +116,16 @@ class ContainerService(BaseService): await self._fire_schedules() + try: + await self._sweep_orphaned_containers(backend, by_uid, known) + except Exception as exc: + self.log(f"orphan sweep failed: {exc}") + + try: + await self._cleanup_stale_workspaces(backend) + except Exception as exc: + self.log(f"workspace cleanup failed: {exc}") + self._metric_tick += 1 if ( self._metric_tick @@ -136,6 +134,49 @@ class ContainerService(BaseService): ): await self._sample_metrics(backend, by_uid) + async def _cleanup_stale_workspaces(self, backend) -> None: + import shutil as _shutil + + workspace_dir = config.CONTAINER_WORKSPACES_DIR + if not workspace_dir.is_dir(): + return + instances = store.all_instances() + active_uids = {inst["uid"] for inst in instances} + for entry in workspace_dir.iterdir(): + if not entry.is_dir(): + continue + uid = entry.name + if uid in active_uids: + continue + try: + _shutil.rmtree(entry) + self.log(f"removed stale workspace {uid}") + except Exception as exc: + self.log(f"failed to remove workspace {uid}: {exc}") + await backend.image_prune() + + async def _sweep_orphaned_containers(self, backend, by_uid: dict, known: set) -> None: + if self._orphan_sweep_tick % 60 != 0: + self._orphan_sweep_tick += 1 + return + self._orphan_sweep_tick += 1 + for uid, row in by_uid.items(): + if uid not in known: + try: + await backend.stop(row.container_id, timeout=10) + except Exception: + pass + try: + await backend.rm(row.container_id, force=True) + self.log(f"removed orphan container {row.name}") + _audit_reconcile( + {"uid": uid, "name": row.name}, + "orphan_reap", + f"sweeper removed orphan container {row.name}", + ) + except Exception as exc: + self.log(f"orphan rm failed for {row.name}: {exc}") + async def _reconcile(self, backend, inst, ps) -> None: uid = inst["uid"] desired = inst["desired_state"] diff --git a/devplacepy/templates/admin_backups.html b/devplacepy/templates/admin_backups.html index 67e56799..28c0cba7 100644 --- a/devplacepy/templates/admin_backups.html +++ b/devplacepy/templates/admin_backups.html @@ -3,8 +3,34 @@ {{ super() }} + {% endblock %} {% block admin_content %} +{% for w in disk_warnings %} + +{% endfor %}

Backups

diff --git a/dpc.log b/dpc.log new file mode 100644 index 00000000..f21f0cb7 --- /dev/null +++ b/dpc.log @@ -0,0 +1,32 @@ +2026-07-19T09:01:55 INFO logging initialised at /workspace/repo/dpc.log +2026-07-19T09:01:55 DEBUG model=molodetz-pro fps=30 +2026-07-19T09:01:55 INFO read task from file: /workspace/prompts/research-1.txt +2026-07-19T09:01:55 INFO settings merged: model= allow=0 deny=0 ask=0 +2026-07-19T15:49:02 INFO logging initialised at /workspace/repo/dpc.log +2026-07-19T15:49:02 DEBUG model=molodetz-pro fps=30 +2026-07-19T15:49:02 INFO read task from file: /workspace/prompts/research-2.txt +2026-07-19T15:49:02 INFO settings merged: model= allow=0 deny=0 ask=0 +2026-07-19T16:41:42 INFO logging initialised at /workspace/repo/dpc.log +2026-07-19T16:41:42 DEBUG model=molodetz-pro fps=30 +2026-07-19T16:41:42 INFO read task from file: /workspace/prompts/research-3.txt +2026-07-19T16:41:42 INFO settings merged: model= allow=0 deny=0 ask=0 +2026-07-19T17:01:42 INFO logging initialised at /workspace/repo/dpc.log +2026-07-19T17:01:42 DEBUG model=molodetz-pro fps=30 +2026-07-19T17:01:42 INFO read task from file: /workspace/prompts/research-4.txt +2026-07-19T17:01:42 INFO settings merged: model= allow=0 deny=0 ask=0 +2026-07-19T17:25:26 INFO logging initialised at /workspace/repo/dpc.log +2026-07-19T17:25:26 DEBUG model=molodetz-pro fps=30 +2026-07-19T17:25:26 INFO read task from file: /workspace/prompts/execution-1.txt +2026-07-19T17:25:26 INFO settings merged: model= allow=0 deny=0 ask=0 +2026-07-19T17:44:59 INFO logging initialised at /workspace/repo/dpc.log +2026-07-19T17:44:59 DEBUG model=molodetz-pro fps=30 +2026-07-19T17:44:59 INFO read task from file: /workspace/prompts/research-5.txt +2026-07-19T17:44:59 INFO settings merged: model= allow=0 deny=0 ask=0 +2026-07-19T19:02:05 INFO logging initialised at /workspace/repo/dpc.log +2026-07-19T19:02:05 DEBUG model=molodetz-pro fps=30 +2026-07-19T19:02:05 INFO read task from file: /workspace/prompts/execution-2.txt +2026-07-19T19:02:05 INFO settings merged: model= allow=0 deny=0 ask=0 +2026-07-23T01:50:04 INFO logging initialised at /workspace/repo/dpc.log +2026-07-23T01:50:04 DEBUG model=molodetz-pro fps=30 +2026-07-23T01:50:04 INFO read task from file: /workspace/prompts/execution-2.txt +2026-07-23T01:50:04 INFO settings merged: model= allow=0 deny=0 ask=0