forked from retoor/devplacepy
Fix container sync races that leaked orphan blobs; add a system-prune CLI command
sync_workspace (user-triggered) and the reconciler's sync_bidirectional_sync could run concurrently for the same project, and store_upload's read-then- write on a changed path meant two racing imports each wrote their own blob while only one ever got referenced - the loser leaked forever. Combined with no build-artifact exclusion, an actively-compiling workspace hit this constantly and leaked 5.9M orphan blobs (~96GB) in production before it was caught. Closes it at the root: api._sync_dir_bidirectional_locked serializes both call sites per-project (non-blocking - a project already mid-sync is simply skipped until the next tick), and IMPORT_SKIP_NAMES/IMPORT_SKIP_EXTENSIONS keep build output (build/, dist/, *.o, *.pyc, ...) out of the walk entirely. Recovering what already leaked is a separate concern: a new CLI subcommand (plus matching make targets) sweeps soft-deleted attachment/project-file blobs and any blob with zero DB reference at all, plus orphaned container workspace directories. run_maintenance_cleanup.sh wraps the existing prune/clear commands for routine disk upkeep. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BWJy6PrMMt5hwWxQwia2rd
This commit is contained in:
@@ -570,6 +570,85 @@ def test_bidirectional_sync_manifest_is_pruned_after_both_sides_agree(env, tmp_p
|
||||
assert remaining == []
|
||||
|
||||
|
||||
def test_bidirectional_sync_skips_build_artifact_directories(env, tmp_path):
|
||||
workspace = tmp_path / "sync-build-ws"
|
||||
workspace.mkdir()
|
||||
pid = env["project"]["uid"]
|
||||
user = env["user"]
|
||||
(workspace / "build").mkdir()
|
||||
(workspace / "build" / "output.bin").write_text("junk")
|
||||
(workspace / "dist").mkdir()
|
||||
(workspace / "dist" / "bundle.js").write_text("junk")
|
||||
(workspace / "real.txt").write_text("keep me\n")
|
||||
|
||||
counts = project_files.sync_dir_bidirectional(pid, str(workspace), user)
|
||||
|
||||
assert counts["imported"] == 1
|
||||
assert project_files.get_node(pid, "real.txt") is not None
|
||||
assert project_files.get_node(pid, "build/output.bin") is None
|
||||
assert project_files.get_node(pid, "dist/bundle.js") is None
|
||||
|
||||
|
||||
def test_bidirectional_sync_skips_compiled_artifact_extensions(env, tmp_path):
|
||||
workspace = tmp_path / "sync-ext-ws"
|
||||
workspace.mkdir()
|
||||
pid = env["project"]["uid"]
|
||||
user = env["user"]
|
||||
(workspace / "main.o").write_bytes(b"junk")
|
||||
(workspace / "helper.pyc").write_bytes(b"junk")
|
||||
(workspace / "main.c").write_text("int main() { return 0; }\n")
|
||||
|
||||
counts = project_files.sync_dir_bidirectional(pid, str(workspace), user)
|
||||
|
||||
assert counts["imported"] == 1
|
||||
assert project_files.get_node(pid, "main.c") is not None
|
||||
assert project_files.get_node(pid, "main.o") is None
|
||||
assert project_files.get_node(pid, "helper.pyc") is None
|
||||
|
||||
|
||||
def test_sync_dir_bidirectional_locked_skips_concurrent_run_for_same_project(
|
||||
env, tmp_path
|
||||
):
|
||||
workspace = tmp_path / "sync-lock-ws"
|
||||
workspace.mkdir()
|
||||
pid = env["project"]["uid"]
|
||||
user = env["user"]
|
||||
|
||||
lock = api._sync_lock_for(pid)
|
||||
lock.acquire()
|
||||
try:
|
||||
counts = api._sync_dir_bidirectional_locked(pid, str(workspace), user)
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
assert counts == {
|
||||
"exported": 0,
|
||||
"imported": 0,
|
||||
"deleted_in_project": 0,
|
||||
"deleted_in_workspace": 0,
|
||||
}
|
||||
assert project_files.get_node(pid, "real.txt") is None
|
||||
|
||||
|
||||
def test_sync_dir_bidirectional_locked_runs_when_lock_is_free(env, tmp_path):
|
||||
workspace = tmp_path / "sync-lock-free-ws"
|
||||
workspace.mkdir()
|
||||
pid = env["project"]["uid"]
|
||||
user = env["user"]
|
||||
(workspace / "real.txt").write_text("keep me\n")
|
||||
|
||||
counts = api._sync_dir_bidirectional_locked(pid, str(workspace), user)
|
||||
|
||||
assert counts["imported"] == 1
|
||||
assert project_files.get_node(pid, "real.txt") is not None
|
||||
|
||||
|
||||
def test_sync_dir_bidirectional_locked_is_per_project(env, tmp_path):
|
||||
other_pid = "sync-lock-other-project"
|
||||
lock = api._sync_lock_for(other_pid)
|
||||
assert lock is not api._sync_lock_for(env["project"]["uid"])
|
||||
|
||||
|
||||
|
||||
def _proxy_scope(kind: str, headers: dict, scheme: str, query: str = "") -> dict:
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user