Fix container sync races that leaked orphan blobs; add a system-prune CLI command

sync_workspace (user-triggered) and the reconciler's sync_bidirectional_sync
could run concurrently for the same project, and store_upload's read-then-
write on a changed path meant two racing imports each wrote their own blob
while only one ever got referenced - the loser leaked forever. Combined with
no build-artifact exclusion, an actively-compiling workspace hit this
constantly and leaked 5.9M orphan blobs (~96GB) in production before it was
caught.

Closes it at the root: api._sync_dir_bidirectional_locked serializes both
call sites per-project (non-blocking - a project already mid-sync is simply
skipped until the next tick), and IMPORT_SKIP_NAMES/IMPORT_SKIP_EXTENSIONS
keep build output (build/, dist/, *.o, *.pyc, ...) out of the walk entirely.

Recovering what already leaked is a separate concern: a new CLI subcommand
(plus matching make targets) sweeps soft-deleted attachment/project-file
blobs and any blob with zero DB reference at all, plus orphaned container
workspace directories. run_maintenance_cleanup.sh wraps the existing
prune/clear commands for routine disk upkeep.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BWJy6PrMMt5hwWxQwia2rd
This commit is contained in:
2026-09-08 03:43:49 +02:00
co-authored by Claude Sonnet 5
parent 85bd8fad47
commit 7880bf4b31
19 changed files with 1068 additions and 19 deletions
+68
View File
@@ -3,6 +3,7 @@
import asyncio
import ipaddress
import logging
import os
import socket
from datetime import datetime, timezone
from pathlib import Path
@@ -637,6 +638,73 @@ def restore_attachment(uid):
return True
def purge_soft_deleted_attachments(*, dry_run: bool = False) -> tuple[int, int]:
if "attachments" not in db.tables:
return 0, 0
table = get_table("attachments")
rows = list(table.find(table.table.columns.deleted_at.isnot(None)))
removed = 0
freed = 0
for row in rows:
directory = row.get("directory")
stored_name = row.get("stored_name")
if directory and stored_name:
path = ATTACHMENTS_DIR / directory / stored_name
try:
freed += path.stat().st_size
except OSError:
pass
for thumb in (ATTACHMENTS_DIR / directory).glob(
f"{Path(stored_name).stem}_thumb.*"
):
try:
freed += thumb.stat().st_size
except OSError:
pass
if not dry_run:
_unlink_attachment_files(row)
if not dry_run:
table.delete(id=row["id"])
removed += 1
return removed, freed
def sweep_orphan_attachment_blobs(*, dry_run: bool = False) -> tuple[int, int]:
if not ATTACHMENTS_DIR.exists():
return 0, 0
referenced_stems = set()
if "attachments" in db.tables:
for row in get_table("attachments").find():
directory = row.get("directory")
stored_name = row.get("stored_name")
if directory and stored_name:
referenced_stems.add((directory, Path(stored_name).stem))
removed = 0
freed = 0
base = str(ATTACHMENTS_DIR)
for root, _dirs, files in os.walk(base):
directory = os.path.relpath(root, base)
for name in files:
stem = Path(name).stem
if stem.endswith("_thumb"):
stem = stem[: -len("_thumb")]
if (directory, stem) in referenced_stems:
continue
file_path = os.path.join(root, name)
try:
size = os.path.getsize(file_path)
except OSError:
continue
if not dry_run:
try:
os.unlink(file_path)
except OSError:
continue
removed += 1
freed += size
return removed, freed
def soft_delete_target_attachments(target_type, target_uid, deleted_by):
stamp = datetime.now(timezone.utc).isoformat()
for row in get_table("attachments").find(