Fix container sync races that leaked orphan blobs; add a system-prune CLI command

sync_workspace (user-triggered) and the reconciler's sync_bidirectional_sync
could run concurrently for the same project, and store_upload's read-then-
write on a changed path meant two racing imports each wrote their own blob
while only one ever got referenced - the loser leaked forever. Combined with
no build-artifact exclusion, an actively-compiling workspace hit this
constantly and leaked 5.9M orphan blobs (~96GB) in production before it was
caught.

Closes it at the root: api._sync_dir_bidirectional_locked serializes both
call sites per-project (non-blocking - a project already mid-sync is simply
skipped until the next tick), and IMPORT_SKIP_NAMES/IMPORT_SKIP_EXTENSIONS
keep build output (build/, dist/, *.o, *.pyc, ...) out of the walk entirely.

Recovering what already leaked is a separate concern: a new CLI subcommand
(plus matching make targets) sweeps soft-deleted attachment/project-file
blobs and any blob with zero DB reference at all, plus orphaned container
workspace directories. run_maintenance_cleanup.sh wraps the existing
prune/clear commands for routine disk upkeep.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BWJy6PrMMt5hwWxQwia2rd
This commit is contained in:
2026-09-08 03:43:49 +02:00
co-authored by Claude Sonnet 5
parent 85bd8fad47
commit 7880bf4b31
19 changed files with 1068 additions and 19 deletions
+79
View File
@@ -570,6 +570,85 @@ def test_bidirectional_sync_manifest_is_pruned_after_both_sides_agree(env, tmp_p
assert remaining == []
def test_bidirectional_sync_skips_build_artifact_directories(env, tmp_path):
workspace = tmp_path / "sync-build-ws"
workspace.mkdir()
pid = env["project"]["uid"]
user = env["user"]
(workspace / "build").mkdir()
(workspace / "build" / "output.bin").write_text("junk")
(workspace / "dist").mkdir()
(workspace / "dist" / "bundle.js").write_text("junk")
(workspace / "real.txt").write_text("keep me\n")
counts = project_files.sync_dir_bidirectional(pid, str(workspace), user)
assert counts["imported"] == 1
assert project_files.get_node(pid, "real.txt") is not None
assert project_files.get_node(pid, "build/output.bin") is None
assert project_files.get_node(pid, "dist/bundle.js") is None
def test_bidirectional_sync_skips_compiled_artifact_extensions(env, tmp_path):
workspace = tmp_path / "sync-ext-ws"
workspace.mkdir()
pid = env["project"]["uid"]
user = env["user"]
(workspace / "main.o").write_bytes(b"junk")
(workspace / "helper.pyc").write_bytes(b"junk")
(workspace / "main.c").write_text("int main() { return 0; }\n")
counts = project_files.sync_dir_bidirectional(pid, str(workspace), user)
assert counts["imported"] == 1
assert project_files.get_node(pid, "main.c") is not None
assert project_files.get_node(pid, "main.o") is None
assert project_files.get_node(pid, "helper.pyc") is None
def test_sync_dir_bidirectional_locked_skips_concurrent_run_for_same_project(
env, tmp_path
):
workspace = tmp_path / "sync-lock-ws"
workspace.mkdir()
pid = env["project"]["uid"]
user = env["user"]
lock = api._sync_lock_for(pid)
lock.acquire()
try:
counts = api._sync_dir_bidirectional_locked(pid, str(workspace), user)
finally:
lock.release()
assert counts == {
"exported": 0,
"imported": 0,
"deleted_in_project": 0,
"deleted_in_workspace": 0,
}
assert project_files.get_node(pid, "real.txt") is None
def test_sync_dir_bidirectional_locked_runs_when_lock_is_free(env, tmp_path):
workspace = tmp_path / "sync-lock-free-ws"
workspace.mkdir()
pid = env["project"]["uid"]
user = env["user"]
(workspace / "real.txt").write_text("keep me\n")
counts = api._sync_dir_bidirectional_locked(pid, str(workspace), user)
assert counts["imported"] == 1
assert project_files.get_node(pid, "real.txt") is not None
def test_sync_dir_bidirectional_locked_is_per_project(env, tmp_path):
other_pid = "sync-lock-other-project"
lock = api._sync_lock_for(other_pid)
assert lock is not api._sync_lock_for(env["project"]["uid"])
def _proxy_scope(kind: str, headers: dict, scheme: str, query: str = "") -> dict:
return {