Add personal notes, DeepSearch history, backup offload, and gateway auth throttling

Also streamline the top navigation: drop the Tools dropdown, make Quizzes and Battles icon-only entries, and remove the Workspace, Containers and Editor entry points from the project detail page.
2026-09-12 20:32:31 +02:00
parent c4f7d01b2d
commit cae139ead9
158 changed files with 6156 additions and 377 deletions
@@ -0,0 +1,58 @@
+# retoor <retoor@molodetz.nl>
+
+import tempfile
+from pathlib import Path
+
+from tests.conftest import run_async
+from devplacepy.services.backup import offload, store
+
+
+def _make_backup(target):
+    job_uid = f"job-{store.generate_uid()}"
+    uid = store.create_backup(target=target, created_by="test", job_uid=job_uid)
+    path = Path(tempfile.gettempdir()) / f"backup-offload-test-{uid}.tar.gz"
+    path.write_bytes(b"x" * 10)
+    store.finalize_backup(
+        uid,
+        filename=path.name,
+        local_path=str(path),
+        stats={
+            "bytes_out": 10,
+            "bytes_in": 10,
+            "file_count": 1,
+            "dir_count": 0,
+            "sha256": "abc",
+        },
+    )
+    return uid, path
+
+
+def test_upload_pending_stops_after_auth_error(local_db, monkeypatch):
+    first_uid, first_path = _make_backup("database")
+    second_uid, second_path = _make_backup("uploads")
+    calls = []
+
+    async def fake_rclone(*args, timeout=1800.0):
+        calls.append(args)
+        return 1, "", "401 Unauthorized from Apache"
+
+    monkeypatch.setattr(offload, "_run_rclone", fake_rclone)
+    logs = []
+    try:
+        uploaded = run_async(offload.upload_pending(log=logs.append))
+        assert uploaded == 0
+        assert len(calls) == 1
+        assert any("halted" in line.lower() for line in logs)
+    finally:
+        first_path.unlink(missing_ok=True)
+        second_path.unlink(missing_ok=True)
+        store.delete_backup(first_uid)
+        store.delete_backup(second_uid)
+
+
+def test_is_auth_error_detects_rclone_html_401():
+    assert offload._is_auth_error(
+        'read metadata failed: <title>401 Unauthorized</title>: 401 Unauthorized'
+    )
+    assert offload._is_auth_error("didn't find section in config file")
+    assert not offload._is_auth_error("connection timed out")
@@ -0,0 +1,72 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.backup import service, store
+
+
+def _stats(used_percent):
+    return {
+        "disk": {
+            "total_bytes": 100,
+            "used_bytes": used_percent,
+            "free_bytes": 100 - used_percent,
+            "total_human": "100 B",
+            "used_human": f"{used_percent} B",
+            "free_human": f"{100 - used_percent} B",
+            "used_percent": used_percent,
+        }
+    }
+
+
+def _disk(used_percent):
+    return _stats(used_percent)["disk"]
+
+
+def test_check_disk_usage_warns_once_and_recovers_once(local_db, monkeypatch):
+    svc = service.BackupService()
+    assert svc.disk_warn_percent_field.default == service.DEFAULT_DISK_WARN_PERCENT
+    logs = []
+    monkeypatch.setattr(svc, "log", lambda message: logs.append(message))
+
+    monkeypatch.setattr(store, "disk_usage", lambda: _disk(50))
+    svc._check_disk_usage()
+    assert logs == []
+    assert svc._disk_warned is False
+
+    monkeypatch.setattr(store, "disk_usage", lambda: _disk(95))
+    svc._check_disk_usage()
+    assert len(logs) == 1
+    assert "critical" in logs[0].lower()
+    assert svc._disk_warned is True
+
+    svc._check_disk_usage()
+    assert len(logs) == 1
+
+    monkeypatch.setattr(store, "disk_usage", lambda: _disk(40))
+    svc._check_disk_usage()
+    assert len(logs) == 2
+    assert "under threshold" in logs[1].lower()
+    assert svc._disk_warned is False
+
+
+def test_collect_metrics_includes_disk_usage_stat(local_db, monkeypatch):
+    svc = service.BackupService()
+    monkeypatch.setattr(store, "disk_usage", lambda: _disk(62))
+
+    metrics = svc.collect_metrics()
+
+    labels = [stat["label"] for stat in metrics["stats"]]
+    assert "Disk usage" in labels
+    disk_stat = next(stat for stat in metrics["stats"] if stat["label"] == "Disk usage")
+    assert "62%" in disk_stat["value"]
+
+
+def test_collect_metrics_never_walks_storage_tree(local_db, monkeypatch):
+    svc = service.BackupService()
+    monkeypatch.setattr(store, "disk_usage", lambda: _disk(10))
+
+    def boom():
+        raise AssertionError("compute_storage_stats must not run on the service tick")
+
+    monkeypatch.setattr(store, "compute_storage_stats", boom)
+    svc.collect_metrics()
+    svc._check_disk_usage()
@@ -1,5 +1,6 @@
 # retoor <retoor@molodetz.nl>

+import os
 import tempfile
 from pathlib import Path

@@ -65,3 +66,90 @@ def test_rotate_schedule_never_removes_a_backup_without_a_confirmed_remote_copy(
    finally:
        store.delete_backup(older_uid)
        store.delete_backup(newer_uid)
+
+
+def _point_storage_at(monkeypatch, root: Path):
+    uploads = root / "uploads"
+    attachments = uploads / "attachments"
+    project_files = uploads / "project_files"
+    keys = root / "keys"
+    zips = root / "zips"
+    deepsearch = root / "deepsearch"
+    workspaces = root / "container_workspaces"
+    backups = root / "backups"
+    for path in (
+        attachments,
+        project_files,
+        keys,
+        zips,
+        deepsearch,
+        workspaces,
+        backups,
+    ):
+        path.mkdir(parents=True, exist_ok=True)
+    db_file = root / "devplace.db"
+    db_file.write_bytes(b"db")
+    tasks = root / "devii_tasks.db"
+    tasks.write_bytes(b"t")
+    lessons = root / "devii_lessons.db"
+    lessons.write_bytes(b"l")
+    (project_files / "a.bin").write_bytes(b"x" * 10)
+    (attachments / "b.bin").write_bytes(b"y" * 5)
+    monkeypatch.setattr(store.config, "DATA_DIR", root)
+    monkeypatch.setattr(store.config, "UPLOADS_DIR", uploads)
+    monkeypatch.setattr(store.config, "ATTACHMENTS_DIR", attachments)
+    monkeypatch.setattr(store.config, "PROJECT_FILES_DIR", project_files)
+    monkeypatch.setattr(store.config, "KEYS_DIR", keys)
+    monkeypatch.setattr(store.config, "ZIPS_DIR", zips)
+    monkeypatch.setattr(store.config, "DEEPSEARCH_DIR", deepsearch)
+    monkeypatch.setattr(store.config, "CONTAINER_WORKSPACES_DIR", workspaces)
+    monkeypatch.setattr(store.config, "BACKUPS_DIR", backups)
+    monkeypatch.setattr(store.config, "DEVII_TASKS_DB", tasks)
+    monkeypatch.setattr(store.config, "DEVII_LESSONS_DB", lessons)
+    monkeypatch.setattr(store.config, "DATABASE_URL", f"sqlite:///{db_file}")
+    store.clear_storage_stats_cache()
+    return {
+        "uploads": uploads,
+        "attachments": attachments,
+        "project_files": project_files,
+        "db_file": db_file,
+    }
+
+
+def test_disk_usage_does_not_walk(monkeypatch, tmp_path):
+    monkeypatch.setattr(store.config, "DATA_DIR", tmp_path)
+    store.clear_storage_stats_cache()
+
+    def boom(*_args, **_kwargs):
+        raise AssertionError("disk_usage must not walk the data tree")
+
+    monkeypatch.setattr(store.os, "walk", boom)
+    disk = store.disk_usage()
+    assert disk["total_bytes"] > 0
+    assert "used_percent" in disk
+
+
+def test_compute_storage_stats_walks_data_dir_once(local_db, monkeypatch, tmp_path):
+    root = tmp_path / "data"
+    _point_storage_at(monkeypatch, root)
+    walks = []
+    real_walk = os.walk
+
+    def counting_walk(path, *args, **kwargs):
+        walks.append(os.path.realpath(path))
+        return real_walk(path, *args, **kwargs)
+
+    monkeypatch.setattr(store.os, "walk", counting_walk)
+    stats = store.compute_storage_stats()
+    assert walks == [os.path.realpath(root)]
+    by_key = {row["key"]: row for row in stats["paths"]}
+    assert by_key["project_files"]["file_count"] == 1
+    assert by_key["project_files"]["size_bytes"] == 10
+    assert by_key["attachments"]["file_count"] == 1
+    assert by_key["uploads"]["file_count"] >= 2
+    assert by_key["database"]["file_count"] == 1
+    assert stats["data_dir"]["file_count"] >= 3
+    walks.clear()
+    again = store.compute_storage_stats()
+    assert walks == []
+    assert again["data_dir"]["file_count"] == stats["data_dir"]["file_count"]