Give the system-prune CLI command a two-phase plan/execute report
Restructures the disk-cleanup command to show location, current on-disk size, and an estimated reclaim per area (attachments, project files, container workspaces) before touching anything - --dry-run stops there. A real run re-executes each check, reports actual items/bytes freed, an "After" size per area, and a final total with elapsed time, flagging any drift between the estimate and execution passes (e.g. the live server wrote something in between). store.gc_workspaces() now reports bytes freed alongside the removed count (previously count-only), so the containers gc-workspaces CLI action picks up the same reporting for free. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01W5tFWkm3UbcstcbPKFE5gP
This commit is contained in:
@@ -66,13 +66,27 @@ def cmd_containers_prune_builds(args):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _human_bytes(n):
|
||||||
|
n = float(n)
|
||||||
|
for unit in ("B", "KB", "MB", "GB", "TB"):
|
||||||
|
if n < 1024:
|
||||||
|
return f"{n:.1f}{unit}"
|
||||||
|
n /= 1024
|
||||||
|
return f"{n:.1f}PB"
|
||||||
|
|
||||||
|
|
||||||
def cmd_containers_gc_workspaces(args):
|
def cmd_containers_gc_workspaces(args):
|
||||||
from devplacepy.services.containers import store
|
from devplacepy.services.containers import store
|
||||||
|
|
||||||
removed = store.gc_workspaces()
|
removed, freed = store.gc_workspaces()
|
||||||
_audit_cli("cli.containers.gc_workspaces", f"CLI removed {removed} unused workspace dirs", metadata={"count": removed})
|
_audit_cli(
|
||||||
|
"cli.containers.gc_workspaces",
|
||||||
|
f"CLI removed {removed} unused workspace dirs, freed {_human_bytes(freed)}",
|
||||||
|
metadata={"count": removed, "bytes_freed": freed},
|
||||||
|
)
|
||||||
print(
|
print(
|
||||||
f"Removed {removed} unused workspace director{'y' if removed == 1 else 'ies'}"
|
f"Removed {removed} unused workspace director{'y' if removed == 1 else 'ies'}"
|
||||||
|
f" ({_human_bytes(freed)})"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+149
-39
@@ -2,8 +2,14 @@
|
|||||||
|
|
||||||
from devplacepy.cli._shared import _audit_cli
|
from devplacepy.cli._shared import _audit_cli
|
||||||
|
|
||||||
|
RULE_WIDTH = 72
|
||||||
|
|
||||||
def _human(n):
|
|
||||||
|
def _rule(char: str = "=") -> str:
|
||||||
|
return char * RULE_WIDTH
|
||||||
|
|
||||||
|
|
||||||
|
def _human(n) -> str:
|
||||||
n = float(n)
|
n = float(n)
|
||||||
for unit in ("B", "KB", "MB", "GB", "TB"):
|
for unit in ("B", "KB", "MB", "GB", "TB"):
|
||||||
if n < 1024:
|
if n < 1024:
|
||||||
@@ -12,65 +18,167 @@ def _human(n):
|
|||||||
return f"{n:.1f}PB"
|
return f"{n:.1f}PB"
|
||||||
|
|
||||||
|
|
||||||
def cmd_system_prune(args):
|
def _dir_stats(path) -> tuple[int, int]:
|
||||||
|
import os
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
total = 0
|
||||||
|
for root, _dirs, files in os.walk(str(path)):
|
||||||
|
for name in files:
|
||||||
|
count += 1
|
||||||
|
try:
|
||||||
|
total += os.path.getsize(os.path.join(root, name))
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return count, total
|
||||||
|
|
||||||
|
|
||||||
|
def _section(title: str) -> None:
|
||||||
|
print()
|
||||||
|
print(_rule())
|
||||||
|
print(title)
|
||||||
|
print(_rule())
|
||||||
|
|
||||||
|
|
||||||
|
def _build_areas():
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from devplacepy import config
|
||||||
from devplacepy.attachments import (
|
from devplacepy.attachments import (
|
||||||
|
ATTACHMENTS_DIR,
|
||||||
purge_soft_deleted_attachments,
|
purge_soft_deleted_attachments,
|
||||||
sweep_orphan_attachment_blobs,
|
sweep_orphan_attachment_blobs,
|
||||||
)
|
)
|
||||||
from devplacepy.project_files import (
|
from devplacepy.project_files import (
|
||||||
|
PROJECT_FILES_DIR,
|
||||||
purge_soft_deleted_project_files,
|
purge_soft_deleted_project_files,
|
||||||
sweep_orphan_project_file_blobs,
|
sweep_orphan_project_file_blobs,
|
||||||
)
|
)
|
||||||
from devplacepy.services.containers import store as container_store
|
from devplacepy.services.containers import store as container_store
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"label": "Attachments",
|
||||||
|
"path": ATTACHMENTS_DIR,
|
||||||
|
"checks": [
|
||||||
|
("soft-deleted attachments", purge_soft_deleted_attachments),
|
||||||
|
(
|
||||||
|
"orphan attachment blobs (zero DB reference)",
|
||||||
|
sweep_orphan_attachment_blobs,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Project files",
|
||||||
|
"path": PROJECT_FILES_DIR,
|
||||||
|
"checks": [
|
||||||
|
("soft-deleted project files", purge_soft_deleted_project_files),
|
||||||
|
(
|
||||||
|
"orphan project-file blobs (zero DB reference)",
|
||||||
|
sweep_orphan_project_file_blobs,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Container workspaces",
|
||||||
|
"path": Path(config.CONTAINER_WORKSPACES_DIR),
|
||||||
|
"checks": [
|
||||||
|
(
|
||||||
|
"orphaned workspace directories (no live instance)",
|
||||||
|
container_store.gc_workspaces,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_system_prune(args):
|
||||||
|
import time
|
||||||
|
|
||||||
dry_run = bool(args.dry_run)
|
dry_run = bool(args.dry_run)
|
||||||
verb = "Would remove" if dry_run else "Removed"
|
areas = _build_areas()
|
||||||
|
|
||||||
totals = {"items": 0, "freed": 0}
|
|
||||||
|
|
||||||
def report(name, count, freed):
|
|
||||||
totals["items"] += count
|
|
||||||
totals["freed"] += freed
|
|
||||||
print(f" {name}: {verb.lower()} {count} item(s), {_human(freed)}")
|
|
||||||
|
|
||||||
print("System prune - safe but aggressive disk-space cleanup")
|
print("System prune - safe but aggressive disk-space cleanup")
|
||||||
print(
|
print(
|
||||||
"Removes only content with zero live reference: soft-deleted attachment/"
|
"Removes only content with zero live reference: soft-deleted attachment/"
|
||||||
"\nproject-file blobs, and blob files with no matching database row at all"
|
"\nproject-file blobs, blob files with no matching database row at all"
|
||||||
"\n(orphans, e.g. left behind by an interrupted or racing sync). Never"
|
"\n(orphans, e.g. left by an interrupted or racing sync), and orphaned"
|
||||||
"\ntouches live content."
|
"\ncontainer workspace directories. Never touches live content."
|
||||||
)
|
)
|
||||||
if dry_run:
|
if dry_run:
|
||||||
print("DRY RUN: nothing will be deleted.")
|
print("DRY RUN: this pass only reports; nothing will be deleted.")
|
||||||
print()
|
|
||||||
|
|
||||||
count, freed = purge_soft_deleted_attachments(dry_run=dry_run)
|
started = time.monotonic()
|
||||||
report("soft-deleted attachments", count, freed)
|
|
||||||
|
|
||||||
count, freed = purge_soft_deleted_project_files(dry_run=dry_run)
|
_section("Where the data is, and what can be reclaimed (estimate)")
|
||||||
report("soft-deleted project files", count, freed)
|
estimate_items = 0
|
||||||
|
estimate_bytes = 0
|
||||||
count, freed = sweep_orphan_attachment_blobs(dry_run=dry_run)
|
for area in areas:
|
||||||
report("orphan attachment blobs", count, freed)
|
count, size = _dir_stats(area["path"])
|
||||||
|
print(f"\n{area['label']}")
|
||||||
count, freed = sweep_orphan_project_file_blobs(dry_run=dry_run)
|
print(f" location: {area['path']}")
|
||||||
report("orphan project-file blobs", count, freed)
|
print(f" currently on disk: {count} file(s), {_human(size)}")
|
||||||
|
for check_label, fn in area["checks"]:
|
||||||
removed = container_store.gc_workspaces(dry_run=dry_run)
|
c, f = fn(dry_run=True)
|
||||||
totals["items"] += removed
|
estimate_items += c
|
||||||
plural = "y" if removed == 1 else "ies"
|
estimate_bytes += f
|
||||||
print(f" orphaned container workspaces: {verb.lower()} {removed} director{plural}")
|
print(f" expected to reclaim - {check_label}: {c} item(s), {_human(f)}")
|
||||||
|
|
||||||
print()
|
print()
|
||||||
suffix = " (estimate)" if dry_run else ""
|
print(_rule("-"))
|
||||||
print(f"{verb} {totals['items']} item(s) total, {_human(totals['freed'])} of disk{suffix}")
|
print(
|
||||||
|
f"Estimated total: {estimate_items} item(s), {_human(estimate_bytes)} reclaimable"
|
||||||
|
)
|
||||||
|
print(_rule("-"))
|
||||||
|
|
||||||
if not dry_run:
|
if dry_run:
|
||||||
_audit_cli(
|
print()
|
||||||
"cli.system.prune",
|
print("DRY RUN complete: nothing was deleted. Re-run without --dry-run to apply.")
|
||||||
f"CLI system prune removed {totals['items']} item(s), freed {_human(totals['freed'])}",
|
return
|
||||||
metadata={"items": totals["items"], "bytes_freed": totals["freed"]},
|
|
||||||
|
_section("Executing")
|
||||||
|
actual_items = 0
|
||||||
|
actual_bytes = 0
|
||||||
|
for area in areas:
|
||||||
|
print(f"\n{area['label']} - {area['path']}")
|
||||||
|
for check_label, fn in area["checks"]:
|
||||||
|
c, f = fn(dry_run=False)
|
||||||
|
actual_items += c
|
||||||
|
actual_bytes += f
|
||||||
|
print(f" removed - {check_label}: {c} item(s), {_human(f)}")
|
||||||
|
|
||||||
|
_section("After")
|
||||||
|
for area in areas:
|
||||||
|
count, size = _dir_stats(area["path"])
|
||||||
|
print(f" {area['label']} ({area['path']}): {count} file(s), {_human(size)}")
|
||||||
|
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
|
||||||
|
print()
|
||||||
|
print(_rule())
|
||||||
|
print(
|
||||||
|
f"Removed {actual_items} item(s) total, freed {_human(actual_bytes)} "
|
||||||
|
f"in {elapsed:.1f}s"
|
||||||
|
)
|
||||||
|
if estimate_bytes != actual_bytes or estimate_items != actual_items:
|
||||||
|
print(
|
||||||
|
f"(Estimate was {estimate_items} item(s), {_human(estimate_bytes)} - "
|
||||||
|
"state changed between the estimate and execution passes, e.g. the "
|
||||||
|
"live server wrote or soft-deleted something in between.)"
|
||||||
)
|
)
|
||||||
|
print(_rule())
|
||||||
|
|
||||||
|
_audit_cli(
|
||||||
|
"cli.system.prune",
|
||||||
|
f"CLI system prune removed {actual_items} item(s), freed {_human(actual_bytes)}",
|
||||||
|
metadata={
|
||||||
|
"items": actual_items,
|
||||||
|
"bytes_freed": actual_bytes,
|
||||||
|
"estimated_items": estimate_items,
|
||||||
|
"estimated_bytes": estimate_bytes,
|
||||||
|
"elapsed_seconds": round(elapsed, 3),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def register_system(subparsers):
|
def register_system(subparsers):
|
||||||
@@ -81,12 +189,14 @@ def register_system(subparsers):
|
|||||||
help=(
|
help=(
|
||||||
"Safe but aggressive disk-space cleanup: purges soft-deleted "
|
"Safe but aggressive disk-space cleanup: purges soft-deleted "
|
||||||
"attachment/project-file blobs and any blob with zero database "
|
"attachment/project-file blobs and any blob with zero database "
|
||||||
"reference at all, plus orphaned container workspace directories"
|
"reference at all, plus orphaned container workspace directories. "
|
||||||
|
"Reports location, current size, and estimated reclaim per area "
|
||||||
|
"before acting, then confirms what was actually freed."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
prune.add_argument(
|
prune.add_argument(
|
||||||
"--dry-run",
|
"--dry-run",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
help="Report what would be removed without deleting anything",
|
help="Report location, size, and estimated reclaim without deleting anything",
|
||||||
)
|
)
|
||||||
prune.set_defaults(func=cmd_system_prune)
|
prune.set_defaults(func=cmd_system_prune)
|
||||||
|
|||||||
@@ -92,7 +92,20 @@ def all_instances() -> list:
|
|||||||
return list(get_table("instances").find(deleted_at=None))
|
return list(get_table("instances").find(deleted_at=None))
|
||||||
|
|
||||||
|
|
||||||
def gc_workspaces(*, dry_run: bool = False) -> int:
|
def _dir_size(path) -> int:
|
||||||
|
import os
|
||||||
|
|
||||||
|
total = 0
|
||||||
|
for root, _dirs, files in os.walk(str(path)):
|
||||||
|
for name in files:
|
||||||
|
try:
|
||||||
|
total += os.path.getsize(os.path.join(root, name))
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
def gc_workspaces(*, dry_run: bool = False) -> tuple[int, int]:
|
||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -101,14 +114,16 @@ def gc_workspaces(*, dry_run: bool = False) -> int:
|
|||||||
active = {inst["project_uid"] for inst in all_instances()}
|
active = {inst["project_uid"] for inst in all_instances()}
|
||||||
base = Path(config.CONTAINER_WORKSPACES_DIR)
|
base = Path(config.CONTAINER_WORKSPACES_DIR)
|
||||||
if not base.is_dir():
|
if not base.is_dir():
|
||||||
return 0
|
return 0, 0
|
||||||
removed = 0
|
removed = 0
|
||||||
|
freed = 0
|
||||||
for child in base.iterdir():
|
for child in base.iterdir():
|
||||||
if child.is_dir() and child.name not in active:
|
if child.is_dir() and child.name not in active:
|
||||||
|
freed += _dir_size(child)
|
||||||
if not dry_run:
|
if not dry_run:
|
||||||
shutil.rmtree(child, ignore_errors=True)
|
shutil.rmtree(child, ignore_errors=True)
|
||||||
removed += 1
|
removed += 1
|
||||||
return removed
|
return removed, freed
|
||||||
|
|
||||||
|
|
||||||
def find_instance_by_ingress(slug: str):
|
def find_instance_by_ingress(slug: str):
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "devplacepy"
|
name = "devplacepy"
|
||||||
version = "1.0.6"
|
version = "1.0.7"
|
||||||
description = "DevPlace - The Developer Social Network"
|
description = "DevPlace - The Developer Social Network"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
|||||||
@@ -37,11 +37,13 @@ def test_gc_workspaces_removes_directories_with_no_live_instance(
|
|||||||
orphan_project = "gcws-orphan-project"
|
orphan_project = "gcws-orphan-project"
|
||||||
(tmp_path / active_project).mkdir()
|
(tmp_path / active_project).mkdir()
|
||||||
(tmp_path / orphan_project).mkdir()
|
(tmp_path / orphan_project).mkdir()
|
||||||
|
(tmp_path / orphan_project / "leftover.bin").write_bytes(b"x" * 42)
|
||||||
_make_instance(active_project)
|
_make_instance(active_project)
|
||||||
|
|
||||||
removed = store.gc_workspaces()
|
removed, freed = store.gc_workspaces()
|
||||||
|
|
||||||
assert removed == 1
|
assert removed == 1
|
||||||
|
assert freed == 42
|
||||||
assert (tmp_path / active_project).exists()
|
assert (tmp_path / active_project).exists()
|
||||||
assert not (tmp_path / orphan_project).exists()
|
assert not (tmp_path / orphan_project).exists()
|
||||||
|
|
||||||
@@ -52,9 +54,10 @@ def test_gc_workspaces_ignores_soft_deleted_instances(local_db, tmp_path, monkey
|
|||||||
(tmp_path / project).mkdir()
|
(tmp_path / project).mkdir()
|
||||||
_make_instance(project, deleted_at="2020-01-01T00:00:00+00:00")
|
_make_instance(project, deleted_at="2020-01-01T00:00:00+00:00")
|
||||||
|
|
||||||
removed = store.gc_workspaces()
|
removed, freed = store.gc_workspaces()
|
||||||
|
|
||||||
assert removed == 1
|
assert removed == 1
|
||||||
|
assert freed == 0
|
||||||
assert not (tmp_path / project).exists()
|
assert not (tmp_path / project).exists()
|
||||||
|
|
||||||
|
|
||||||
@@ -62,10 +65,12 @@ def test_gc_workspaces_dry_run_changes_nothing(local_db, tmp_path, monkeypatch):
|
|||||||
monkeypatch.setattr("devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path)
|
monkeypatch.setattr("devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path)
|
||||||
orphan_project = "gcws-dry-run-project"
|
orphan_project = "gcws-dry-run-project"
|
||||||
(tmp_path / orphan_project).mkdir()
|
(tmp_path / orphan_project).mkdir()
|
||||||
|
(tmp_path / orphan_project / "leftover.bin").write_bytes(b"y" * 7)
|
||||||
|
|
||||||
removed = store.gc_workspaces(dry_run=True)
|
removed, freed = store.gc_workspaces(dry_run=True)
|
||||||
|
|
||||||
assert removed == 1
|
assert removed == 1
|
||||||
|
assert freed == 7
|
||||||
assert (tmp_path / orphan_project).exists()
|
assert (tmp_path / orphan_project).exists()
|
||||||
|
|
||||||
|
|
||||||
@@ -73,4 +78,4 @@ def test_gc_workspaces_missing_base_dir_is_a_noop(local_db, tmp_path, monkeypatc
|
|||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path / "does-not-exist"
|
"devplacepy.config.CONTAINER_WORKSPACES_DIR", tmp_path / "does-not-exist"
|
||||||
)
|
)
|
||||||
assert store.gc_workspaces() == 0
|
assert store.gc_workspaces() == (0, 0)
|
||||||
|
|||||||
Reference in New Issue
Block a user