Fix container sync races that leaked orphan blobs; add a system-prune CLI command

sync_workspace (user-triggered) and the reconciler's sync_bidirectional_sync could run concurrently for the same project, and store_upload's read-then- write on a changed path meant two racing imports each wrote their own blob while only one ever got referenced - the loser leaked forever. Combined with no build-artifact exclusion, an actively-compiling workspace hit this constantly and leaked 5.9M orphan blobs (~96GB) in production before it was caught. Closes it at the root: api._sync_dir_bidirectional_locked serializes both call sites per-project (non-blocking - a project already mid-sync is simply skipped until the next tick), and IMPORT_SKIP_NAMES/IMPORT_SKIP_EXTENSIONS keep build output (build/, dist/, *.o, *.pyc, ...) out of the walk entirely. Recovering what already leaked is a separate concern: a new CLI subcommand (plus matching make targets) sweeps soft-deleted attachment/project-file blobs and any blob with zero DB reference at all, plus orphaned container workspace directories. run_maintenance_cleanup.sh wraps the existing prune/clear commands for routine disk upkeep. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BWJy6PrMMt5hwWxQwia2rd
2026-09-08 03:43:49 +02:00
parent 85bd8fad47
commit 7880bf4b31
19 changed files with 1068 additions and 19 deletions
@@ -388,3 +388,144 @@ def test_docs_group_present():
    assert group is not None
    ids = {e["id"] for e in group["endpoints"]}
    assert "project-files-write" in ids and "project-files-list" in ids
+
+
+def _insert_binary_node(project_uid, directory, stored_name, deleted_at=None):
+    from devplacepy.utils import generate_uid
+
+    uid = generate_uid()
+    pf._table().insert(
+        {
+            "uid": uid,
+            "project_uid": project_uid,
+            "deleted_at": deleted_at,
+            "deleted_by": None,
+            "user_uid": "system-prune-test-user",
+            "path": f"/{stored_name}",
+            "name": stored_name,
+            "parent_path": "/",
+            "type": "file",
+            "content": None,
+            "is_binary": 1,
+            "stored_name": stored_name,
+            "directory": directory,
+            "mime_type": "application/octet-stream",
+            "size": 0,
+            "created_at": pf._now(),
+            "updated_at": pf._now(),
+        }
+    )
+    return uid
+
+
+def _write_project_blob(base, directory, stored_name, content=b"data"):
+    file_dir = base / directory
+    file_dir.mkdir(parents=True, exist_ok=True)
+    (file_dir / stored_name).write_bytes(content)
+
+
+def test_purge_soft_deleted_project_files_removes_row_and_blob(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = "obj.o"
+    _write_project_blob(tmp_path, directory, stored_name, b"x" * 40)
+    uid = _insert_binary_node(
+        "sys-prune-proj-1", directory, stored_name, deleted_at="2020-01-01T00:00:00+00:00"
+    )
+
+    removed, freed = pf.purge_soft_deleted_project_files()
+
+    assert removed >= 1
+    assert freed == 40
+    assert pf._table().find_one(uid=uid) is None
+    assert not (tmp_path / directory / stored_name).exists()
+
+
+def test_purge_soft_deleted_project_files_dry_run_changes_nothing(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = "obj2.o"
+    _write_project_blob(tmp_path, directory, stored_name, b"x" * 15)
+    uid = _insert_binary_node(
+        "sys-prune-proj-2", directory, stored_name, deleted_at="2020-01-01T00:00:00+00:00"
+    )
+
+    removed, freed = pf.purge_soft_deleted_project_files(dry_run=True)
+
+    assert removed >= 1
+    assert freed == 15
+    assert pf._table().find_one(uid=uid) is not None
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_purge_soft_deleted_project_files_ignores_live_and_text_rows(
+    local_db, tmp_path, monkeypatch
+):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = "live.o"
+    _write_project_blob(tmp_path, directory, stored_name)
+    live_uid = _insert_binary_node("sys-prune-proj-3", directory, stored_name, deleted_at=None)
+
+    pf.purge_soft_deleted_project_files()
+
+    assert pf._table().find_one(uid=live_uid) is not None
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_sweep_orphan_project_file_blobs_removes_unreferenced_file(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    orphan_name = "leftover.right"
+    _write_project_blob(tmp_path, directory, orphan_name, b"q" * 60)
+
+    removed, freed = pf.sweep_orphan_project_file_blobs()
+
+    assert removed == 1
+    assert freed == 60
+    assert not (tmp_path / directory / orphan_name).exists()
+
+
+def test_sweep_orphan_project_file_blobs_keeps_referenced_file(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = "kept.bin"
+    _write_project_blob(tmp_path, directory, stored_name)
+    _insert_binary_node("sys-prune-proj-4", directory, stored_name, deleted_at=None)
+
+    removed, freed = pf.sweep_orphan_project_file_blobs()
+
+    assert removed == 0
+    assert freed == 0
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_sweep_orphan_project_file_blobs_keeps_blob_referenced_only_by_soft_deleted_row(
+    local_db, tmp_path, monkeypatch
+):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = "still-soft-deleted.bin"
+    _write_project_blob(tmp_path, directory, stored_name)
+    _insert_binary_node(
+        "sys-prune-proj-5", directory, stored_name, deleted_at="2020-01-01T00:00:00+00:00"
+    )
+
+    removed, freed = pf.sweep_orphan_project_file_blobs()
+
+    assert removed == 0
+    assert freed == 0
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_sweep_orphan_project_file_blobs_dry_run_changes_nothing(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    orphan_name = "dry.right"
+    _write_project_blob(tmp_path, directory, orphan_name, b"w" * 12)
+
+    removed, freed = pf.sweep_orphan_project_file_blobs(dry_run=True)
+
+    assert removed == 1
+    assert freed == 12
+    assert (tmp_path / directory / orphan_name).exists()