Fix container sync races that leaked orphan blobs; add a system-prune CLI command

sync_workspace (user-triggered) and the reconciler's sync_bidirectional_sync
could run concurrently for the same project, and store_upload's read-then-
write on a changed path meant two racing imports each wrote their own blob
while only one ever got referenced - the loser leaked forever. Combined with
no build-artifact exclusion, an actively-compiling workspace hit this
constantly and leaked 5.9M orphan blobs (~96GB) in production before it was
caught.

Closes it at the root: api._sync_dir_bidirectional_locked serializes both
call sites per-project (non-blocking - a project already mid-sync is simply
skipped until the next tick), and IMPORT_SKIP_NAMES/IMPORT_SKIP_EXTENSIONS
keep build output (build/, dist/, *.o, *.pyc, ...) out of the walk entirely.

Recovering what already leaked is a separate concern: a new CLI subcommand
(plus matching make targets) sweeps soft-deleted attachment/project-file
blobs and any blob with zero DB reference at all, plus orphaned container
workspace directories. run_maintenance_cleanup.sh wraps the existing
prune/clear commands for routine disk upkeep.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BWJy6PrMMt5hwWxQwia2rd

tests/unit/attachments.py

@@ -0,0 +1,149 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy import attachments as att
+from devplacepy.database import get_table
+from devplacepy.utils import generate_uid
+
+
+def _make_attachment(directory, stored_name, deleted_at=None):
+    uid = generate_uid()
+    get_table("attachments").insert(
+        {
+            "uid": uid,
+            "deleted_at": deleted_at,
+            "deleted_by": None,
+            "directory": directory,
+            "stored_name": stored_name,
+            "target_type": "post",
+            "target_uid": generate_uid(),
+        }
+    )
+    return uid
+
+
+def _write_blob(base, directory, stored_name, content=b"data"):
+    file_dir = base / directory
+    file_dir.mkdir(parents=True, exist_ok=True)
+    (file_dir / stored_name).write_bytes(content)
+    return file_dir / stored_name
+
+
+def test_purge_soft_deleted_attachments_removes_row_and_blob(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(att, "ATTACHMENTS_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = f"{generate_uid()}.png"
+    _write_blob(tmp_path, directory, stored_name, b"x" * 100)
+    uid = _make_attachment(directory, stored_name, deleted_at="2020-01-01T00:00:00+00:00")
+
+    removed, freed = att.purge_soft_deleted_attachments()
+
+    assert removed >= 1
+    assert freed == 100
+    assert get_table("attachments").find_one(uid=uid) is None
+    assert not (tmp_path / directory / stored_name).exists()
+
+
+def test_purge_soft_deleted_attachments_dry_run_changes_nothing(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(att, "ATTACHMENTS_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = f"{generate_uid()}.png"
+    _write_blob(tmp_path, directory, stored_name, b"x" * 50)
+    uid = _make_attachment(directory, stored_name, deleted_at="2020-01-01T00:00:00+00:00")
+
+    removed, freed = att.purge_soft_deleted_attachments(dry_run=True)
+
+    assert removed >= 1
+    assert freed == 50
+    assert get_table("attachments").find_one(uid=uid) is not None
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_purge_soft_deleted_attachments_ignores_live_rows(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(att, "ATTACHMENTS_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = f"{generate_uid()}.png"
+    _write_blob(tmp_path, directory, stored_name)
+    uid = _make_attachment(directory, stored_name, deleted_at=None)
+
+    att.purge_soft_deleted_attachments()
+
+    assert get_table("attachments").find_one(uid=uid) is not None
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_purge_soft_deleted_attachments_also_removes_thumbnail(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(att, "ATTACHMENTS_DIR", tmp_path)
+    directory = "ab/cd"
+    stem = generate_uid()
+    stored_name = f"{stem}.jpg"
+    _write_blob(tmp_path, directory, stored_name, b"x" * 20)
+    _write_blob(tmp_path, directory, f"{stem}_thumb.jpg", b"y" * 5)
+    _make_attachment(directory, stored_name, deleted_at="2020-01-01T00:00:00+00:00")
+
+    removed, freed = att.purge_soft_deleted_attachments()
+
+    assert removed >= 1
+    assert freed == 25
+    assert not (tmp_path / directory / f"{stem}_thumb.jpg").exists()
+
+
+def test_sweep_orphan_attachment_blobs_removes_unreferenced_file(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(att, "ATTACHMENTS_DIR", tmp_path)
+    directory = "ab/cd"
+    orphan_name = f"{generate_uid()}.o"
+    _write_blob(tmp_path, directory, orphan_name, b"y" * 30)
+
+    removed, freed = att.sweep_orphan_attachment_blobs()
+
+    assert removed == 1
+    assert freed == 30
+    assert not (tmp_path / directory / orphan_name).exists()
+
+
+def test_sweep_orphan_attachment_blobs_keeps_referenced_file_and_its_thumbnail(
+    local_db, tmp_path, monkeypatch
+):
+    monkeypatch.setattr(att, "ATTACHMENTS_DIR", tmp_path)
+    directory = "ab/cd"
+    stem = generate_uid()
+    stored_name = f"{stem}.jpg"
+    thumb_name = f"{stem}_thumb.jpg"
+    _write_blob(tmp_path, directory, stored_name)
+    _write_blob(tmp_path, directory, thumb_name)
+    _make_attachment(directory, stored_name, deleted_at=None)
+
+    removed, freed = att.sweep_orphan_attachment_blobs()
+
+    assert removed == 0
+    assert freed == 0
+    assert (tmp_path / directory / stored_name).exists()
+    assert (tmp_path / directory / thumb_name).exists()
+
+
+def test_sweep_orphan_attachment_blobs_keeps_blob_referenced_only_by_soft_deleted_row(
+    local_db, tmp_path, monkeypatch
+):
+    monkeypatch.setattr(att, "ATTACHMENTS_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = f"{generate_uid()}.png"
+    _write_blob(tmp_path, directory, stored_name)
+    _make_attachment(directory, stored_name, deleted_at="2020-01-01T00:00:00+00:00")
+
+    removed, freed = att.sweep_orphan_attachment_blobs()
+
+    assert removed == 0
+    assert freed == 0
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_sweep_orphan_attachment_blobs_dry_run_changes_nothing(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(att, "ATTACHMENTS_DIR", tmp_path)
+    directory = "ab/cd"
+    orphan_name = f"{generate_uid()}.o"
+    _write_blob(tmp_path, directory, orphan_name, b"z" * 10)
+
+    removed, freed = att.sweep_orphan_attachment_blobs(dry_run=True)
+
+    assert removed == 1
+    assert freed == 10
+    assert (tmp_path / directory / orphan_name).exists()

tests/unit/cli.py

@@ -161,3 +161,44 @@ def test_main_dispatches_subcommand(local_db, monkeypatch, capsys):
     monkeypatch.setattr("sys.argv", ["devplace", "role", "get", username])
     cli.main()
     assert capsys.readouterr().out.strip() == "admin"
+
+
+def test_system_prune_reclaims_orphans_across_subsystems(local_db, tmp_path, capsys, monkeypatch):
+    from devplacepy import attachments as att
+    from devplacepy import project_files as pf
+
+    monkeypatch.setattr(att, "ATTACHMENTS_DIR", tmp_path / "attachments")
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path / "project_files")
+    monkeypatch.setattr("devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path / "workspaces")
+
+    attachments_dir = tmp_path / "attachments" / "ab" / "cd"
+    attachments_dir.mkdir(parents=True)
+    (attachments_dir / "orphan.png").write_bytes(b"a" * 10)
+
+    project_files_dir = tmp_path / "project_files" / "ab" / "cd"
+    project_files_dir.mkdir(parents=True)
+    (project_files_dir / "orphan.o").write_bytes(b"b" * 20)
+
+    (tmp_path / "workspaces").mkdir()
+    (tmp_path / "workspaces" / "orphan-project").mkdir()
+
+    cli.cmd_system_prune(argparse.Namespace(dry_run=True))
+    dry_output = capsys.readouterr().out
+    assert "DRY RUN" in dry_output
+    assert (attachments_dir / "orphan.png").exists()
+    assert (project_files_dir / "orphan.o").exists()
+    assert (tmp_path / "workspaces" / "orphan-project").exists()
+
+    cli.cmd_system_prune(argparse.Namespace(dry_run=False))
+    real_output = capsys.readouterr().out
+    assert "DRY RUN" not in real_output
+    assert not (attachments_dir / "orphan.png").exists()
+    assert not (project_files_dir / "orphan.o").exists()
+    assert not (tmp_path / "workspaces" / "orphan-project").exists()
+
+
+def test_system_prune_registered_in_parser():
+    parser = cli.build_parser()
+    args = parser.parse_args(["system", "prune", "--dry-run"])
+    assert args.func is cli.cmd_system_prune
+    assert args.dry_run is True

tests/unit/project_files.py

@@ -388,3 +388,144 @@ def test_docs_group_present():
     assert group is not None
     ids = {e["id"] for e in group["endpoints"]}
     assert "project-files-write" in ids and "project-files-list" in ids
+
+
+def _insert_binary_node(project_uid, directory, stored_name, deleted_at=None):
+    from devplacepy.utils import generate_uid
+
+    uid = generate_uid()
+    pf._table().insert(
+        {
+            "uid": uid,
+            "project_uid": project_uid,
+            "deleted_at": deleted_at,
+            "deleted_by": None,
+            "user_uid": "system-prune-test-user",
+            "path": f"/{stored_name}",
+            "name": stored_name,
+            "parent_path": "/",
+            "type": "file",
+            "content": None,
+            "is_binary": 1,
+            "stored_name": stored_name,
+            "directory": directory,
+            "mime_type": "application/octet-stream",
+            "size": 0,
+            "created_at": pf._now(),
+            "updated_at": pf._now(),
+        }
+    )
+    return uid
+
+
+def _write_project_blob(base, directory, stored_name, content=b"data"):
+    file_dir = base / directory
+    file_dir.mkdir(parents=True, exist_ok=True)
+    (file_dir / stored_name).write_bytes(content)
+
+
+def test_purge_soft_deleted_project_files_removes_row_and_blob(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = "obj.o"
+    _write_project_blob(tmp_path, directory, stored_name, b"x" * 40)
+    uid = _insert_binary_node(
+        "sys-prune-proj-1", directory, stored_name, deleted_at="2020-01-01T00:00:00+00:00"
+    )
+
+    removed, freed = pf.purge_soft_deleted_project_files()
+
+    assert removed >= 1
+    assert freed == 40
+    assert pf._table().find_one(uid=uid) is None
+    assert not (tmp_path / directory / stored_name).exists()
+
+
+def test_purge_soft_deleted_project_files_dry_run_changes_nothing(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = "obj2.o"
+    _write_project_blob(tmp_path, directory, stored_name, b"x" * 15)
+    uid = _insert_binary_node(
+        "sys-prune-proj-2", directory, stored_name, deleted_at="2020-01-01T00:00:00+00:00"
+    )
+
+    removed, freed = pf.purge_soft_deleted_project_files(dry_run=True)
+
+    assert removed >= 1
+    assert freed == 15
+    assert pf._table().find_one(uid=uid) is not None
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_purge_soft_deleted_project_files_ignores_live_and_text_rows(
+    local_db, tmp_path, monkeypatch
+):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = "live.o"
+    _write_project_blob(tmp_path, directory, stored_name)
+    live_uid = _insert_binary_node("sys-prune-proj-3", directory, stored_name, deleted_at=None)
+
+    pf.purge_soft_deleted_project_files()
+
+    assert pf._table().find_one(uid=live_uid) is not None
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_sweep_orphan_project_file_blobs_removes_unreferenced_file(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    orphan_name = "leftover.right"
+    _write_project_blob(tmp_path, directory, orphan_name, b"q" * 60)
+
+    removed, freed = pf.sweep_orphan_project_file_blobs()
+
+    assert removed == 1
+    assert freed == 60
+    assert not (tmp_path / directory / orphan_name).exists()
+
+
+def test_sweep_orphan_project_file_blobs_keeps_referenced_file(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = "kept.bin"
+    _write_project_blob(tmp_path, directory, stored_name)
+    _insert_binary_node("sys-prune-proj-4", directory, stored_name, deleted_at=None)
+
+    removed, freed = pf.sweep_orphan_project_file_blobs()
+
+    assert removed == 0
+    assert freed == 0
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_sweep_orphan_project_file_blobs_keeps_blob_referenced_only_by_soft_deleted_row(
+    local_db, tmp_path, monkeypatch
+):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    stored_name = "still-soft-deleted.bin"
+    _write_project_blob(tmp_path, directory, stored_name)
+    _insert_binary_node(
+        "sys-prune-proj-5", directory, stored_name, deleted_at="2020-01-01T00:00:00+00:00"
+    )
+
+    removed, freed = pf.sweep_orphan_project_file_blobs()
+
+    assert removed == 0
+    assert freed == 0
+    assert (tmp_path / directory / stored_name).exists()
+
+
+def test_sweep_orphan_project_file_blobs_dry_run_changes_nothing(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    directory = "ab/cd"
+    orphan_name = "dry.right"
+    _write_project_blob(tmp_path, directory, orphan_name, b"w" * 12)
+
+    removed, freed = pf.sweep_orphan_project_file_blobs(dry_run=True)
+
+    assert removed == 1
+    assert freed == 12
+    assert (tmp_path / directory / orphan_name).exists()

tests/unit/services/containers/api.py

@@ -570,6 +570,85 @@ def test_bidirectional_sync_manifest_is_pruned_after_both_sides_agree(env, tmp_p
     assert remaining == []
 
 
+def test_bidirectional_sync_skips_build_artifact_directories(env, tmp_path):
+    workspace = tmp_path / "sync-build-ws"
+    workspace.mkdir()
+    pid = env["project"]["uid"]
+    user = env["user"]
+    (workspace / "build").mkdir()
+    (workspace / "build" / "output.bin").write_text("junk")
+    (workspace / "dist").mkdir()
+    (workspace / "dist" / "bundle.js").write_text("junk")
+    (workspace / "real.txt").write_text("keep me\n")
+
+    counts = project_files.sync_dir_bidirectional(pid, str(workspace), user)
+
+    assert counts["imported"] == 1
+    assert project_files.get_node(pid, "real.txt") is not None
+    assert project_files.get_node(pid, "build/output.bin") is None
+    assert project_files.get_node(pid, "dist/bundle.js") is None
+
+
+def test_bidirectional_sync_skips_compiled_artifact_extensions(env, tmp_path):
+    workspace = tmp_path / "sync-ext-ws"
+    workspace.mkdir()
+    pid = env["project"]["uid"]
+    user = env["user"]
+    (workspace / "main.o").write_bytes(b"junk")
+    (workspace / "helper.pyc").write_bytes(b"junk")
+    (workspace / "main.c").write_text("int main() { return 0; }\n")
+
+    counts = project_files.sync_dir_bidirectional(pid, str(workspace), user)
+
+    assert counts["imported"] == 1
+    assert project_files.get_node(pid, "main.c") is not None
+    assert project_files.get_node(pid, "main.o") is None
+    assert project_files.get_node(pid, "helper.pyc") is None
+
+
+def test_sync_dir_bidirectional_locked_skips_concurrent_run_for_same_project(
+    env, tmp_path
+):
+    workspace = tmp_path / "sync-lock-ws"
+    workspace.mkdir()
+    pid = env["project"]["uid"]
+    user = env["user"]
+
+    lock = api._sync_lock_for(pid)
+    lock.acquire()
+    try:
+        counts = api._sync_dir_bidirectional_locked(pid, str(workspace), user)
+    finally:
+        lock.release()
+
+    assert counts == {
+        "exported": 0,
+        "imported": 0,
+        "deleted_in_project": 0,
+        "deleted_in_workspace": 0,
+    }
+    assert project_files.get_node(pid, "real.txt") is None
+
+
+def test_sync_dir_bidirectional_locked_runs_when_lock_is_free(env, tmp_path):
+    workspace = tmp_path / "sync-lock-free-ws"
+    workspace.mkdir()
+    pid = env["project"]["uid"]
+    user = env["user"]
+    (workspace / "real.txt").write_text("keep me\n")
+
+    counts = api._sync_dir_bidirectional_locked(pid, str(workspace), user)
+
+    assert counts["imported"] == 1
+    assert project_files.get_node(pid, "real.txt") is not None
+
+
+def test_sync_dir_bidirectional_locked_is_per_project(env, tmp_path):
+    other_pid = "sync-lock-other-project"
+    lock = api._sync_lock_for(other_pid)
+    assert lock is not api._sync_lock_for(env["project"]["uid"])
+
+
 
 def _proxy_scope(kind: str, headers: dict, scheme: str, query: str = "") -> dict:
     return {

tests/unit/services/containers/store.py

@@ -0,0 +1,76 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.database import get_table, init_db
+from devplacepy.services.containers import store
+from devplacepy.utils import generate_uid
+
+
+@pytest.fixture(autouse=True)
+def _init_db_containers_store():
+    init_db()
+    yield
+
+
+def _make_instance(project_uid, deleted_at=None):
+    uid = generate_uid()
+    get_table("instances").insert(
+        {
+            "uid": uid,
+            "project_uid": project_uid,
+            "deleted_at": deleted_at,
+            "deleted_by": None,
+            "name": f"inst-{uid[:8]}",
+            "status": "running",
+            "desired_state": "running",
+        }
+    )
+    return uid
+
+
+def test_gc_workspaces_removes_directories_with_no_live_instance(
+    local_db, tmp_path, monkeypatch
+):
+    monkeypatch.setattr("devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path)
+    active_project = "gcws-active-project"
+    orphan_project = "gcws-orphan-project"
+    (tmp_path / active_project).mkdir()
+    (tmp_path / orphan_project).mkdir()
+    _make_instance(active_project)
+
+    removed = store.gc_workspaces()
+
+    assert removed == 1
+    assert (tmp_path / active_project).exists()
+    assert not (tmp_path / orphan_project).exists()
+
+
+def test_gc_workspaces_ignores_soft_deleted_instances(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr("devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path)
+    project = "gcws-soft-deleted-project"
+    (tmp_path / project).mkdir()
+    _make_instance(project, deleted_at="2020-01-01T00:00:00+00:00")
+
+    removed = store.gc_workspaces()
+
+    assert removed == 1
+    assert not (tmp_path / project).exists()
+
+
+def test_gc_workspaces_dry_run_changes_nothing(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr("devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path)
+    orphan_project = "gcws-dry-run-project"
+    (tmp_path / orphan_project).mkdir()
+
+    removed = store.gc_workspaces(dry_run=True)
+
+    assert removed == 1
+    assert (tmp_path / orphan_project).exists()
+
+
+def test_gc_workspaces_missing_base_dir_is_a_noop(local_db, tmp_path, monkeypatch):
+    monkeypatch.setattr(
+        "devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path / "does-not-exist"
+    )
+    assert store.gc_workspaces() == 0