ticket #106 attempt 1

devplacepy/models.py

@@ -470,9 +470,19 @@ class SeoRunForm(BaseModel):
         text = value.strip()
         if not text:
             raise ValueError("A URL is required")
+        if "://" in text:
+            scheme = text.split("://", 1)[0]
+            if scheme not in ("http", "https"):
+                raise ValueError(f"Only http and https URLs are allowed; got '{scheme}://'")
+        else:
+            text = f"https://{text}"
+        if not SEO_URL_PATTERN.match(text):
+            raise ValueError("URL must be a valid http or https source location")
         return text
 
 
+SEO_URL_PATTERN = re.compile(r"^https?://[a-zA-Z0-9][\w./:@~^?&#%=;-]*$")
+
 ISSLOP_URL_PATTERN = re.compile(r"^(https?://|git://|ssh://|git@)[\w./:@~^-]+$", re.IGNORECASE)
 ISSLOP_SINGLE_SLASH_PATTERN = re.compile(r"^(https?|git|ssh):/(?!/)", re.IGNORECASE)
 ISSLOP_SCHEME_PATTERN = re.compile(r"^[a-z][a-z0-9+.-]*://", re.IGNORECASE)

devplacepy/routers/admin/backups.py

@@ -1,7 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
 import logging
-import os
 from pathlib import Path
 from typing import Annotated
 
@@ -65,7 +64,6 @@ def _targets() -> list[dict]:
 def _dashboard(can_download: bool) -> dict:
     backups = [_backup_payload(row, can_download) for row in store.list_backups()]
     schedules = store.list_schedules()
-    threshold = int(os.environ.get("DISK_WARNING_PERCENT", "85"))
     return {
         "storage": store.compute_storage_stats(),
         "backups": backups,
@@ -74,8 +72,6 @@ def _dashboard(can_download: bool) -> dict:
         "metrics": _metrics(backups),
         "generated_at": store.now_iso(),
         "can_download_backups": can_download,
-        "disk_warnings": store.get_disk_warnings(threshold),
-        "disk_warning_threshold": threshold,
     }
 
 @router.get("/backups", response_class=HTMLResponse)

devplacepy/schemas/backups.py

@@ -53,13 +53,6 @@ class BackupJobOut(_Out):
     completed_at: Optional[str] = None
 
 
-class DiskWarningOut(_Out):
-    path: str = ""
-    used_pct: float = 0.0
-    total_gb: float = 0.0
-    used_gb: float = 0.0
-
-
 class BackupScheduleOut(_Out):
     uid: str = ""
     name: str = ""
@@ -85,5 +78,3 @@ class BackupDashboardOut(_Out):
     generated_at: Optional[str] = None
     can_download_backups: bool = False
     admin_section: Optional[str] = None
-    disk_warnings: list[DiskWarningOut] = []
-    disk_warning_threshold: int = 85

devplacepy/services/backup/service.py

@@ -34,10 +34,6 @@ class BackupService(JobService):
 
     def __init__(self):
         super().__init__(name="backup", interval_seconds=15)
-        try:
-            store.seed_default_schedule()
-        except Exception as exc:
-            logger.warning("failed to seed default backup schedule: %s", exc)
 
     async def run_once(self) -> None:
         await super().run_once()
@@ -125,15 +121,6 @@ class BackupService(JobService):
             summary=f"backup {target} completed ({_human_bytes(stats['bytes_out'])})",
             links=[audit.job(job_uid)],
         )
-        if schedule_uid:
-            try:
-                store.check_disk_and_alert()
-            except Exception as exc:
-                logger.warning("disk alert check failed: %s", exc)
-            try:
-                store.check_db_growth()
-            except Exception as exc:
-                logger.warning("db growth check failed: %s", exc)
         return {
             "backup_uid": backup_uid,
             "target": target,

devplacepy/services/backup/store.py

@@ -1,7 +1,5 @@
 # retoor <retoor@molodetz.nl>
 
-import json
-import logging
 import shutil
 import time
 from datetime import datetime, timezone
@@ -11,8 +9,6 @@ from devplacepy import config
 from devplacepy.database import _index, db, get_table
 from devplacepy.utils import generate_uid
 
-logger = logging.getLogger(__name__)
-
 BACKUP_TARGETS: dict[str, dict] = {
     "database": {
         "label": "Database",
@@ -379,58 +375,6 @@ def _storage_paths() -> list[tuple[str, str, Path]]:
     ]
 
 
-def seed_default_schedule() -> None:
-    if "backup_schedules" not in db.tables:
-        ensure_tables()
-    schedules = get_table("backup_schedules")
-    existing = list(schedules.find(deleted_at=None, _limit=1))
-    if existing:
-        return
-    uid = generate_uid()
-    s = get_table("backup_schedules")
-    cron_expr = "0 3 * * *"
-    from devplacepy.services.devii.tasks.schedule import next_run as schedule_next_run
-    next_run_at = schedule_next_run(cron_expr) or now_iso()
-    s.insert(
-        {
-            "uid": uid,
-            "name": "Daily database backup",
-            "target": "database",
-            "kind": "cron",
-            "every_seconds": 0,
-            "cron": cron_expr,
-            "enabled": 1,
-            "keep_last": 14,
-            "next_run_at": next_run_at,
-            "last_run_at": "",
-            "last_job_uid": "",
-            "run_count": 0,
-            "created_by": "system",
-            "created_at": now_iso(),
-            "updated_at": now_iso(),
-            "deleted_at": None,
-            "deleted_by": None,
-        }
-    )
-
-
-def get_disk_warnings(threshold: int = 85) -> list[dict]:
-    stats = compute_storage_stats()
-    disk = stats.get("disk", {})
-    used_pct = disk.get("used_percent", 0.0)
-    warnings = []
-    if used_pct >= threshold:
-        warnings.append(
-            {
-                "path": stats.get("data_dir", {}).get("path", str(config.DATA_DIR)),
-                "used_pct": used_pct,
-                "total_gb": round(disk.get("total_bytes", 0) / (1024**3), 1),
-                "used_gb": round(disk.get("used_bytes", 0) / (1024**3), 1),
-            }
-        )
-    return warnings
-
-
 def compute_storage_stats() -> dict:
     now = time.monotonic()
     if (
@@ -491,131 +435,3 @@ def compute_storage_stats() -> dict:
     _storage_cache["data"] = data
     _storage_cache["at"] = now
     return data
-
-
-DB_GROWTH_HISTORY_FILE = None
-
-
-def _db_growth_path() -> Path:
-    global DB_GROWTH_HISTORY_FILE
-    if DB_GROWTH_HISTORY_FILE is None:
-        DB_GROWTH_HISTORY_FILE = config.BACKUPS_DIR / "db_growth_history.json"
-    return DB_GROWTH_HISTORY_FILE
-
-
-def _load_db_sizes() -> list[dict]:
-    path = _db_growth_path()
-    if not path.exists():
-        return []
-    try:
-        return json.loads(path.read_text())
-    except (ValueError, OSError):
-        return []
-
-
-def _save_db_sizes(sizes: list[dict]) -> None:
-    path = _db_growth_path()
-    try:
-        path.parent.mkdir(parents=True, exist_ok=True)
-        path.write_text(json.dumps(sizes))
-    except OSError as exc:
-        logger.warning("failed to write db growth history: %s", exc)
-
-
-def _database_file_size() -> int:
-    database_file = Path(str(config.DATABASE_URL).replace("sqlite:///", ""))
-    try:
-        return database_file.stat().st_size if database_file.exists() else 0
-    except OSError:
-        return 0
-
-
-def check_db_growth() -> None:
-    from devplacepy.services.audit import record as audit
-
-    current = _database_file_size()
-    if current <= 0:
-        return
-
-    sizes = _load_db_sizes()
-    last_entry = sizes[-1] if sizes else None
-    now = now_iso()
-
-    sizes.append({"at": now, "size_bytes": current})
-    _save_db_sizes(sizes)
-
-    if last_entry and last_entry.get("size_bytes", 0) > 0:
-        try:
-            last_at = datetime.fromisoformat(last_entry["at"])
-            current_at = datetime.fromisoformat(now)
-            days = (current_at - last_at).total_seconds() / 86400
-            if days > 0:
-                growth_pct = (current - last_entry["size_bytes"]) / last_entry["size_bytes"]
-                daily_pct = growth_pct / days
-                if daily_pct > 0.10:
-                    logger.warning(
-                        "database growth %.1f%%/day (%.0f MB -> %.0f MB over %.1f days)",
-                        daily_pct * 100,
-                        last_entry["size_bytes"] / (1024 * 1024),
-                        current / (1024 * 1024),
-                        days,
-                    )
-                    try:
-                        audit.record_system(
-                            "backup.db_growth_high",
-                            actor_kind="service",
-                            origin="scheduler",
-                            target_type="system",
-                            metadata={
-                                "size_bytes": current,
-                                "previous_size_bytes": last_entry["size_bytes"],
-                                "daily_growth_pct": round(daily_pct * 100, 1),
-                                "span_days": round(days, 2),
-                            },
-                            summary=(
-                                f"database growth {daily_pct * 100:.1f}%/day "
-                                f"({human_bytes(last_entry['size_bytes'])} -> {human_bytes(current)})"
-                            ),
-                        )
-                    except Exception as exc:
-                        logger.warning("db growth audit failed: %s", exc)
-        except (ValueError, TypeError) as exc:
-            logger.warning("failed to parse db growth dates: %s", exc)
-
-
-DISK_ALERT_THRESHOLD = 85
-
-
-def check_disk_and_alert() -> None:
-    from devplacepy.services.audit import record as audit
-
-    stats = compute_storage_stats()
-    disk = stats.get("disk", {})
-    used_pct = disk.get("used_percent", 0.0)
-    if used_pct < DISK_ALERT_THRESHOLD:
-        return
-    logger.warning(
-        "disk usage at %.1f%% (threshold %d%%)",
-        used_pct,
-        DISK_ALERT_THRESHOLD,
-    )
-    try:
-        audit.record_system(
-            "backup.disk.critical",
-            actor_kind="service",
-            origin="scheduler",
-            target_type="system",
-            metadata={
-                "used_percent": used_pct,
-                "used_bytes": disk.get("used_bytes", 0),
-                "total_bytes": disk.get("total_bytes", 0),
-                "free_bytes": disk.get("free_bytes", 0),
-            },
-            summary=(
-                f"disk usage at {used_pct:.1f}% "
-                f"({human_bytes(disk.get('used_bytes', 0))} / "
-                f"{human_bytes(disk.get('total_bytes', 0))})"
-            ),
-        )
-    except Exception as exc:
-        logger.warning("disk alert audit failed: %s", exc)

devplacepy/services/containers/service.py

@@ -81,7 +81,6 @@ class ContainerService(BaseService):
     def __init__(self):
         super().__init__(name="containers", interval_seconds=5)
         self._metric_tick = 0
-        self._orphan_sweep_tick = 0
         self._booted = False
         self._last_sync_at = 0.0
 
@@ -108,6 +107,19 @@ class ContainerService(BaseService):
 
         await self._periodic_sync(instances, by_uid)
 
+        for uid, row in by_uid.items():
+            if uid not in known:
+                try:
+                    await backend.rm(row.container_id, force=True)
+                    self.log(f"reaped orphan container {row.name}")
+                    _audit_reconcile(
+                        {"uid": uid, "name": row.name},
+                        "orphan_reap",
+                        f"reconciler reaped orphan container {row.name}",
+                    )
+                except Exception as exc:
+                    self.log(f"orphan rm failed for {row.name}: {exc}")
+
         for inst in instances:
             try:
                 await self._reconcile(backend, inst, by_uid.get(inst["uid"]))
@@ -116,16 +128,6 @@ class ContainerService(BaseService):
 
         await self._fire_schedules()
 
-        try:
-            await self._sweep_orphaned_containers(backend, by_uid, known)
-        except Exception as exc:
-            self.log(f"orphan sweep failed: {exc}")
-
-        try:
-            await self._cleanup_stale_workspaces(backend)
-        except Exception as exc:
-            self.log(f"workspace cleanup failed: {exc}")
-
         self._metric_tick += 1
         if (
             self._metric_tick
@@ -134,49 +136,6 @@ class ContainerService(BaseService):
         ):
             await self._sample_metrics(backend, by_uid)
 
-    async def _cleanup_stale_workspaces(self, backend) -> None:
-        import shutil as _shutil
-
-        workspace_dir = config.CONTAINER_WORKSPACES_DIR
-        if not workspace_dir.is_dir():
-            return
-        instances = store.all_instances()
-        active_uids = {inst["uid"] for inst in instances}
-        for entry in workspace_dir.iterdir():
-            if not entry.is_dir():
-                continue
-            uid = entry.name
-            if uid in active_uids:
-                continue
-            try:
-                _shutil.rmtree(entry)
-                self.log(f"removed stale workspace {uid}")
-            except Exception as exc:
-                self.log(f"failed to remove workspace {uid}: {exc}")
-        await backend.image_prune()
-
-    async def _sweep_orphaned_containers(self, backend, by_uid: dict, known: set) -> None:
-        if self._orphan_sweep_tick % 60 != 0:
-            self._orphan_sweep_tick += 1
-            return
-        self._orphan_sweep_tick += 1
-        for uid, row in by_uid.items():
-            if uid not in known:
-                try:
-                    await backend.stop(row.container_id, timeout=10)
-                except Exception:
-                    pass
-                try:
-                    await backend.rm(row.container_id, force=True)
-                    self.log(f"removed orphan container {row.name}")
-                    _audit_reconcile(
-                        {"uid": uid, "name": row.name},
-                        "orphan_reap",
-                        f"sweeper removed orphan container {row.name}",
-                    )
-                except Exception as exc:
-                    self.log(f"orphan rm failed for {row.name}: {exc}")
-
     async def _reconcile(self, backend, inst, ps) -> None:
         uid = inst["uid"]
         desired = inst["desired_state"]

devplacepy/templates/admin_backups.html

@@ -3,34 +3,8 @@
 {{ super() }}
 <link rel="stylesheet" href="{{ static_url('/static/css/services.css') }}">
 <link rel="stylesheet" href="{{ static_url('/static/css/backups.css') }}">
-<style>
-.disk-warning {
-    background: #fff3cd;
-    border: 1px solid #ffc107;
-    border-radius: 6px;
-    padding: 12px 16px;
-    margin-bottom: 16px;
-    display: flex;
-    align-items: center;
-    gap: 10px;
-    color: #856404;
-}
-.disk-warning .icon {
-    font-size: 1.4em;
-    flex-shrink: 0;
-}
-.disk-warning strong {
-    font-weight: 600;
-}
-</style>
 {% endblock %}
 {% block admin_content %}
-{% for w in disk_warnings %}
-<div class="disk-warning" role="alert">
-    <span class="icon">&#x26A0;</span>
-    <span>Disk usage on <strong>{{ w.path }}</strong> is <strong>{{ w.used_pct }}%</strong> ({{ w.used_gb }} GB / {{ w.total_gb }} GB) - above {{ disk_warning_threshold }}% threshold. Consider cleaning up old data or increasing disk capacity.</span>
-</div>
-{% endfor %}
 <div class="admin-toolbar">
     <h2>Backups</h2>
     <div class="backup-controls">

dpc.log

@@ -1,32 +0,0 @@
-2026-07-19T09:01:55 INFO logging initialised at /workspace/repo/dpc.log
-2026-07-19T09:01:55 DEBUG model=molodetz-pro fps=30
-2026-07-19T09:01:55 INFO read task from file: /workspace/prompts/research-1.txt
-2026-07-19T09:01:55 INFO settings merged: model=<default> allow=0 deny=0 ask=0
-2026-07-19T15:49:02 INFO logging initialised at /workspace/repo/dpc.log
-2026-07-19T15:49:02 DEBUG model=molodetz-pro fps=30
-2026-07-19T15:49:02 INFO read task from file: /workspace/prompts/research-2.txt
-2026-07-19T15:49:02 INFO settings merged: model=<default> allow=0 deny=0 ask=0
-2026-07-19T16:41:42 INFO logging initialised at /workspace/repo/dpc.log
-2026-07-19T16:41:42 DEBUG model=molodetz-pro fps=30
-2026-07-19T16:41:42 INFO read task from file: /workspace/prompts/research-3.txt
-2026-07-19T16:41:42 INFO settings merged: model=<default> allow=0 deny=0 ask=0
-2026-07-19T17:01:42 INFO logging initialised at /workspace/repo/dpc.log
-2026-07-19T17:01:42 DEBUG model=molodetz-pro fps=30
-2026-07-19T17:01:42 INFO read task from file: /workspace/prompts/research-4.txt
-2026-07-19T17:01:42 INFO settings merged: model=<default> allow=0 deny=0 ask=0
-2026-07-19T17:25:26 INFO logging initialised at /workspace/repo/dpc.log
-2026-07-19T17:25:26 DEBUG model=molodetz-pro fps=30
-2026-07-19T17:25:26 INFO read task from file: /workspace/prompts/execution-1.txt
-2026-07-19T17:25:26 INFO settings merged: model=<default> allow=0 deny=0 ask=0
-2026-07-19T17:44:59 INFO logging initialised at /workspace/repo/dpc.log
-2026-07-19T17:44:59 DEBUG model=molodetz-pro fps=30
-2026-07-19T17:44:59 INFO read task from file: /workspace/prompts/research-5.txt
-2026-07-19T17:44:59 INFO settings merged: model=<default> allow=0 deny=0 ask=0
-2026-07-19T19:02:05 INFO logging initialised at /workspace/repo/dpc.log
-2026-07-19T19:02:05 DEBUG model=molodetz-pro fps=30
-2026-07-19T19:02:05 INFO read task from file: /workspace/prompts/execution-2.txt
-2026-07-19T19:02:05 INFO settings merged: model=<default> allow=0 deny=0 ask=0
-2026-07-23T01:50:04 INFO logging initialised at /workspace/repo/dpc.log
-2026-07-23T01:50:04 DEBUG model=molodetz-pro fps=30
-2026-07-23T01:50:04 INFO read task from file: /workspace/prompts/execution-2.txt
-2026-07-23T01:50:04 INFO settings merged: model=<default> allow=0 deny=0 ask=0

tests/api/tools/seo.py

@@ -89,6 +89,53 @@ def test_run_clamps_max_pages_above_cap(app_server):
         _clear_seo_jobs()
 
 
+def test_run_rejects_malformed_scheme(app_server):
+    r = requests.post(
+        f"{BASE_URL}/tools/seo/run",
+        headers=_json_headers(),
+        data={"url": "ahttps://devplace.net/sitem", "mode": "url"},
+        allow_redirects=False,
+    )
+    assert r.status_code in (400, 422), r.text
+
+
+def test_run_rejects_relative_path(app_server):
+    r = requests.post(
+        f"{BASE_URL}/tools/seo/run",
+        headers=_json_headers(),
+        data={"url": "/feed", "mode": "url"},
+        allow_redirects=False,
+    )
+    assert r.status_code in (400, 422), r.text
+
+
+def test_run_normalizes_missing_scheme(app_server):
+    try:
+        r = requests.post(
+            f"{BASE_URL}/tools/seo/run",
+            headers=_json_headers(),
+            data={"url": "example.com", "mode": "url", "max_pages": "5"},
+        )
+        assert r.status_code == 200, r.text
+        uid = r.json()["uid"]
+        refresh_snapshot()
+        job = queue.get_job(uid)
+        assert job is not None
+        assert job["payload"]["url"] == "https://example.com"
+    finally:
+        _clear_seo_jobs()
+
+
+def test_run_rejects_non_http_scheme(app_server):
+    r = requests.post(
+        f"{BASE_URL}/tools/seo/run",
+        headers=_json_headers(),
+        data={"url": "ftp://example.com", "mode": "url"},
+        allow_redirects=False,
+    )
+    assert r.status_code in (400, 422), r.text
+
+
 def test_run_enforces_one_active_job_per_owner(app_server):
     try:
         first = requests.post(