Fix container sync races that leaked orphan blobs; add a system-prune CLI command
sync_workspace (user-triggered) and the reconciler's sync_bidirectional_sync could run concurrently for the same project, and store_upload's read-then- write on a changed path meant two racing imports each wrote their own blob while only one ever got referenced - the loser leaked forever. Combined with no build-artifact exclusion, an actively-compiling workspace hit this constantly and leaked 5.9M orphan blobs (~96GB) in production before it was caught. Closes it at the root: api._sync_dir_bidirectional_locked serializes both call sites per-project (non-blocking - a project already mid-sync is simply skipped until the next tick), and IMPORT_SKIP_NAMES/IMPORT_SKIP_EXTENSIONS keep build output (build/, dist/, *.o, *.pyc, ...) out of the walk entirely. Recovering what already leaked is a separate concern: a new CLI subcommand (plus matching make targets) sweeps soft-deleted attachment/project-file blobs and any blob with zero DB reference at all, plus orphaned container workspace directories. run_maintenance_cleanup.sh wraps the existing prune/clear commands for routine disk upkeep. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BWJy6PrMMt5hwWxQwia2rd
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
@@ -637,6 +638,73 @@ def restore_attachment(uid):
|
||||
return True
|
||||
|
||||
|
||||
def purge_soft_deleted_attachments(*, dry_run: bool = False) -> tuple[int, int]:
|
||||
if "attachments" not in db.tables:
|
||||
return 0, 0
|
||||
table = get_table("attachments")
|
||||
rows = list(table.find(table.table.columns.deleted_at.isnot(None)))
|
||||
removed = 0
|
||||
freed = 0
|
||||
for row in rows:
|
||||
directory = row.get("directory")
|
||||
stored_name = row.get("stored_name")
|
||||
if directory and stored_name:
|
||||
path = ATTACHMENTS_DIR / directory / stored_name
|
||||
try:
|
||||
freed += path.stat().st_size
|
||||
except OSError:
|
||||
pass
|
||||
for thumb in (ATTACHMENTS_DIR / directory).glob(
|
||||
f"{Path(stored_name).stem}_thumb.*"
|
||||
):
|
||||
try:
|
||||
freed += thumb.stat().st_size
|
||||
except OSError:
|
||||
pass
|
||||
if not dry_run:
|
||||
_unlink_attachment_files(row)
|
||||
if not dry_run:
|
||||
table.delete(id=row["id"])
|
||||
removed += 1
|
||||
return removed, freed
|
||||
|
||||
|
||||
def sweep_orphan_attachment_blobs(*, dry_run: bool = False) -> tuple[int, int]:
|
||||
if not ATTACHMENTS_DIR.exists():
|
||||
return 0, 0
|
||||
referenced_stems = set()
|
||||
if "attachments" in db.tables:
|
||||
for row in get_table("attachments").find():
|
||||
directory = row.get("directory")
|
||||
stored_name = row.get("stored_name")
|
||||
if directory and stored_name:
|
||||
referenced_stems.add((directory, Path(stored_name).stem))
|
||||
removed = 0
|
||||
freed = 0
|
||||
base = str(ATTACHMENTS_DIR)
|
||||
for root, _dirs, files in os.walk(base):
|
||||
directory = os.path.relpath(root, base)
|
||||
for name in files:
|
||||
stem = Path(name).stem
|
||||
if stem.endswith("_thumb"):
|
||||
stem = stem[: -len("_thumb")]
|
||||
if (directory, stem) in referenced_stems:
|
||||
continue
|
||||
file_path = os.path.join(root, name)
|
||||
try:
|
||||
size = os.path.getsize(file_path)
|
||||
except OSError:
|
||||
continue
|
||||
if not dry_run:
|
||||
try:
|
||||
os.unlink(file_path)
|
||||
except OSError:
|
||||
continue
|
||||
removed += 1
|
||||
freed += size
|
||||
return removed, freed
|
||||
|
||||
|
||||
def soft_delete_target_attachments(target_type, target_uid, deleted_by):
|
||||
stamp = datetime.now(timezone.utc).isoformat()
|
||||
for row in get_table("attachments").find(
|
||||
|
||||
Reference in New Issue
Block a user