diff --git a/devplacepy/cli/containers.py b/devplacepy/cli/containers.py index 82d8db5..b076d47 100644 --- a/devplacepy/cli/containers.py +++ b/devplacepy/cli/containers.py @@ -66,13 +66,27 @@ def cmd_containers_prune_builds(args): ) +def _human_bytes(n): + n = float(n) + for unit in ("B", "KB", "MB", "GB", "TB"): + if n < 1024: + return f"{n:.1f}{unit}" + n /= 1024 + return f"{n:.1f}PB" + + def cmd_containers_gc_workspaces(args): from devplacepy.services.containers import store - removed = store.gc_workspaces() - _audit_cli("cli.containers.gc_workspaces", f"CLI removed {removed} unused workspace dirs", metadata={"count": removed}) + removed, freed = store.gc_workspaces() + _audit_cli( + "cli.containers.gc_workspaces", + f"CLI removed {removed} unused workspace dirs, freed {_human_bytes(freed)}", + metadata={"count": removed, "bytes_freed": freed}, + ) print( f"Removed {removed} unused workspace director{'y' if removed == 1 else 'ies'}" + f" ({_human_bytes(freed)})" ) diff --git a/devplacepy/cli/system.py b/devplacepy/cli/system.py index e1e14d8..7dd79dd 100644 --- a/devplacepy/cli/system.py +++ b/devplacepy/cli/system.py @@ -2,8 +2,14 @@ from devplacepy.cli._shared import _audit_cli +RULE_WIDTH = 72 -def _human(n): + +def _rule(char: str = "=") -> str: + return char * RULE_WIDTH + + +def _human(n) -> str: n = float(n) for unit in ("B", "KB", "MB", "GB", "TB"): if n < 1024: @@ -12,65 +18,167 @@ def _human(n): return f"{n:.1f}PB" -def cmd_system_prune(args): +def _dir_stats(path) -> tuple[int, int]: + import os + + count = 0 + total = 0 + for root, _dirs, files in os.walk(str(path)): + for name in files: + count += 1 + try: + total += os.path.getsize(os.path.join(root, name)) + except OSError: + continue + return count, total + + +def _section(title: str) -> None: + print() + print(_rule()) + print(title) + print(_rule()) + + +def _build_areas(): + from pathlib import Path + + from devplacepy import config from devplacepy.attachments import ( + ATTACHMENTS_DIR, purge_soft_deleted_attachments, sweep_orphan_attachment_blobs, ) from devplacepy.project_files import ( + PROJECT_FILES_DIR, purge_soft_deleted_project_files, sweep_orphan_project_file_blobs, ) from devplacepy.services.containers import store as container_store + return [ + { + "label": "Attachments", + "path": ATTACHMENTS_DIR, + "checks": [ + ("soft-deleted attachments", purge_soft_deleted_attachments), + ( + "orphan attachment blobs (zero DB reference)", + sweep_orphan_attachment_blobs, + ), + ], + }, + { + "label": "Project files", + "path": PROJECT_FILES_DIR, + "checks": [ + ("soft-deleted project files", purge_soft_deleted_project_files), + ( + "orphan project-file blobs (zero DB reference)", + sweep_orphan_project_file_blobs, + ), + ], + }, + { + "label": "Container workspaces", + "path": Path(config.CONTAINER_WORKSPACES_DIR), + "checks": [ + ( + "orphaned workspace directories (no live instance)", + container_store.gc_workspaces, + ), + ], + }, + ] + + +def cmd_system_prune(args): + import time + dry_run = bool(args.dry_run) - verb = "Would remove" if dry_run else "Removed" - - totals = {"items": 0, "freed": 0} - - def report(name, count, freed): - totals["items"] += count - totals["freed"] += freed - print(f" {name}: {verb.lower()} {count} item(s), {_human(freed)}") + areas = _build_areas() print("System prune - safe but aggressive disk-space cleanup") print( "Removes only content with zero live reference: soft-deleted attachment/" - "\nproject-file blobs, and blob files with no matching database row at all" - "\n(orphans, e.g. left behind by an interrupted or racing sync). Never" - "\ntouches live content." + "\nproject-file blobs, blob files with no matching database row at all" + "\n(orphans, e.g. left by an interrupted or racing sync), and orphaned" + "\ncontainer workspace directories. Never touches live content." ) if dry_run: - print("DRY RUN: nothing will be deleted.") - print() + print("DRY RUN: this pass only reports; nothing will be deleted.") - count, freed = purge_soft_deleted_attachments(dry_run=dry_run) - report("soft-deleted attachments", count, freed) + started = time.monotonic() - count, freed = purge_soft_deleted_project_files(dry_run=dry_run) - report("soft-deleted project files", count, freed) - - count, freed = sweep_orphan_attachment_blobs(dry_run=dry_run) - report("orphan attachment blobs", count, freed) - - count, freed = sweep_orphan_project_file_blobs(dry_run=dry_run) - report("orphan project-file blobs", count, freed) - - removed = container_store.gc_workspaces(dry_run=dry_run) - totals["items"] += removed - plural = "y" if removed == 1 else "ies" - print(f" orphaned container workspaces: {verb.lower()} {removed} director{plural}") + _section("Where the data is, and what can be reclaimed (estimate)") + estimate_items = 0 + estimate_bytes = 0 + for area in areas: + count, size = _dir_stats(area["path"]) + print(f"\n{area['label']}") + print(f" location: {area['path']}") + print(f" currently on disk: {count} file(s), {_human(size)}") + for check_label, fn in area["checks"]: + c, f = fn(dry_run=True) + estimate_items += c + estimate_bytes += f + print(f" expected to reclaim - {check_label}: {c} item(s), {_human(f)}") print() - suffix = " (estimate)" if dry_run else "" - print(f"{verb} {totals['items']} item(s) total, {_human(totals['freed'])} of disk{suffix}") + print(_rule("-")) + print( + f"Estimated total: {estimate_items} item(s), {_human(estimate_bytes)} reclaimable" + ) + print(_rule("-")) - if not dry_run: - _audit_cli( - "cli.system.prune", - f"CLI system prune removed {totals['items']} item(s), freed {_human(totals['freed'])}", - metadata={"items": totals["items"], "bytes_freed": totals["freed"]}, + if dry_run: + print() + print("DRY RUN complete: nothing was deleted. Re-run without --dry-run to apply.") + return + + _section("Executing") + actual_items = 0 + actual_bytes = 0 + for area in areas: + print(f"\n{area['label']} - {area['path']}") + for check_label, fn in area["checks"]: + c, f = fn(dry_run=False) + actual_items += c + actual_bytes += f + print(f" removed - {check_label}: {c} item(s), {_human(f)}") + + _section("After") + for area in areas: + count, size = _dir_stats(area["path"]) + print(f" {area['label']} ({area['path']}): {count} file(s), {_human(size)}") + + elapsed = time.monotonic() - started + + print() + print(_rule()) + print( + f"Removed {actual_items} item(s) total, freed {_human(actual_bytes)} " + f"in {elapsed:.1f}s" + ) + if estimate_bytes != actual_bytes or estimate_items != actual_items: + print( + f"(Estimate was {estimate_items} item(s), {_human(estimate_bytes)} - " + "state changed between the estimate and execution passes, e.g. the " + "live server wrote or soft-deleted something in between.)" ) + print(_rule()) + + _audit_cli( + "cli.system.prune", + f"CLI system prune removed {actual_items} item(s), freed {_human(actual_bytes)}", + metadata={ + "items": actual_items, + "bytes_freed": actual_bytes, + "estimated_items": estimate_items, + "estimated_bytes": estimate_bytes, + "elapsed_seconds": round(elapsed, 3), + }, + ) def register_system(subparsers): @@ -81,12 +189,14 @@ def register_system(subparsers): help=( "Safe but aggressive disk-space cleanup: purges soft-deleted " "attachment/project-file blobs and any blob with zero database " - "reference at all, plus orphaned container workspace directories" + "reference at all, plus orphaned container workspace directories. " + "Reports location, current size, and estimated reclaim per area " + "before acting, then confirms what was actually freed." ), ) prune.add_argument( "--dry-run", action="store_true", - help="Report what would be removed without deleting anything", + help="Report location, size, and estimated reclaim without deleting anything", ) prune.set_defaults(func=cmd_system_prune) diff --git a/devplacepy/services/containers/store.py b/devplacepy/services/containers/store.py index 425ebea..fddae11 100644 --- a/devplacepy/services/containers/store.py +++ b/devplacepy/services/containers/store.py @@ -92,7 +92,20 @@ def all_instances() -> list: return list(get_table("instances").find(deleted_at=None)) -def gc_workspaces(*, dry_run: bool = False) -> int: +def _dir_size(path) -> int: + import os + + total = 0 + for root, _dirs, files in os.walk(str(path)): + for name in files: + try: + total += os.path.getsize(os.path.join(root, name)) + except OSError: + continue + return total + + +def gc_workspaces(*, dry_run: bool = False) -> tuple[int, int]: import shutil from pathlib import Path @@ -101,14 +114,16 @@ def gc_workspaces(*, dry_run: bool = False) -> int: active = {inst["project_uid"] for inst in all_instances()} base = Path(config.CONTAINER_WORKSPACES_DIR) if not base.is_dir(): - return 0 + return 0, 0 removed = 0 + freed = 0 for child in base.iterdir(): if child.is_dir() and child.name not in active: + freed += _dir_size(child) if not dry_run: shutil.rmtree(child, ignore_errors=True) removed += 1 - return removed + return removed, freed def find_instance_by_ingress(slug: str): diff --git a/pyproject.toml b/pyproject.toml index d6c1158..afe57f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "devplacepy" -version = "1.0.6" +version = "1.0.7" description = "DevPlace - The Developer Social Network" requires-python = ">=3.12" dependencies = [ diff --git a/tests/unit/services/containers/store.py b/tests/unit/services/containers/store.py index 72dcd7c..cc0784e 100644 --- a/tests/unit/services/containers/store.py +++ b/tests/unit/services/containers/store.py @@ -37,11 +37,13 @@ def test_gc_workspaces_removes_directories_with_no_live_instance( orphan_project = "gcws-orphan-project" (tmp_path / active_project).mkdir() (tmp_path / orphan_project).mkdir() + (tmp_path / orphan_project / "leftover.bin").write_bytes(b"x" * 42) _make_instance(active_project) - removed = store.gc_workspaces() + removed, freed = store.gc_workspaces() assert removed == 1 + assert freed == 42 assert (tmp_path / active_project).exists() assert not (tmp_path / orphan_project).exists() @@ -52,9 +54,10 @@ def test_gc_workspaces_ignores_soft_deleted_instances(local_db, tmp_path, monkey (tmp_path / project).mkdir() _make_instance(project, deleted_at="2020-01-01T00:00:00+00:00") - removed = store.gc_workspaces() + removed, freed = store.gc_workspaces() assert removed == 1 + assert freed == 0 assert not (tmp_path / project).exists() @@ -62,10 +65,12 @@ def test_gc_workspaces_dry_run_changes_nothing(local_db, tmp_path, monkeypatch): monkeypatch.setattr("devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path) orphan_project = "gcws-dry-run-project" (tmp_path / orphan_project).mkdir() + (tmp_path / orphan_project / "leftover.bin").write_bytes(b"y" * 7) - removed = store.gc_workspaces(dry_run=True) + removed, freed = store.gc_workspaces(dry_run=True) assert removed == 1 + assert freed == 7 assert (tmp_path / orphan_project).exists() @@ -73,4 +78,4 @@ def test_gc_workspaces_missing_base_dir_is_a_noop(local_db, tmp_path, monkeypatc monkeypatch.setattr( "devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path / "does-not-exist" ) - assert store.gc_workspaces() == 0 + assert store.gc_workspaces() == (0, 0)