diff --git a/readme.md b/readme.md index c1853c3..99219a8 100644 --- a/readme.md +++ b/readme.md @@ -328,7 +328,9 @@ When the remote is unreachable, the service keeps working from the spool and cat - **Retention** (`POST /api/v1/admin/retention/run`, dry run by default): keeps every version younger than `retention.keep_days` (default **5**), thins older ones to one per file per UTC day, and always keeps each file's latest version and all - pins. Follow with GC to reclaim the freed blobs. + pins. Every execute ends with a hard reclaim: `VACUUM` (freelist → 0) plus WAL + checkpoint/truncate, and reports `index_reclaimed` bytes. Follow with GC to + reclaim the freed blobs. - **Garbage collection** (`POST /api/v1/admin/gc`, dry run by default): deletes blobs no version references anymore, from the local spool and (unless `remote: false`) from WebDAV. Only unreferenced data is ever touched, so a crash can leak garbage diff --git a/skills/versiond-admin/SKILL.md b/skills/versiond-admin/SKILL.md index 2fec19f..7bc1f6d 100644 --- a/skills/versiond-admin/SKILL.md +++ b/skills/versiond-admin/SKILL.md @@ -43,6 +43,10 @@ server is the trust model); there are no keys to manage. thinned to 1/file/day; per-file latest + pins always kept. - `POST /admin/retention/run` (dry_run default true) then `POST /admin/gc` (dry_run default true, `remote: true` also wipes WebDAV). +- Local maintenance rules: every retention/forget execute ends with + `VACUUM` + WAL checkpoint/truncate (`index_reclaimed` bytes reported), so + deletes really shrink `index.sqlite`; GC removes orphan spool blobs. + Without this, deletes would only pile onto freelist pages forever. - Built-in loop: retention daily, GC weekly (`[scheduler]`). No cron needed. - Capacity backstop `scheduler.remote_max_used_percent` (default 70): over-limit runs a pass but NEVER breaks the keep-days floor; health shows diff --git a/src/versiond/api.py b/src/versiond/api.py index 2afa97e..9f04a37 100644 --- a/src/versiond/api.py +++ b/src/versiond/api.py @@ -774,9 +774,11 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None, blobs += chunk await asyncio.sleep(0) projects = s.repo.drop_empty_projects() + reclaimed = recovery.vacuum_index(s.db) s.db.audit("forget", path=path, files=files, blobs=blobs) return {"path": path, "dry_run": False, "files_removed": files, - "versions_removed": counts["versions"], "blobs_removed": blobs, "projects_removed": projects} + "versions_removed": counts["versions"], "blobs_removed": blobs, "projects_removed": projects, + "index_reclaimed": reclaimed} # remote purge (remote data only; never touches the local index or spool) @@ -913,9 +915,12 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None, return {"dry_run": True, **{k: v for k, v in plan.items() if k != "delete_ids"}, "versions_to_delete": len(plan["delete_ids"])} deleted = await recovery.delete_versions(s.db, plan["delete_ids"]) - s.db.audit("retention.run", keep_days=keep_days, deleted=deleted) + reclaimed = recovery.vacuum_index(s.db) + s.db.audit("retention.run", keep_days=keep_days, deleted=deleted, + reclaimed_bytes=reclaimed["bytes_reclaimed"]) return {"dry_run": False, "keep_days": keep_days, "candidates": plan["candidates"], - "kept_daily": plan["kept_daily"], "versions_deleted": deleted} + "kept_daily": plan["kept_daily"], "versions_deleted": deleted, + "index_reclaimed": reclaimed} @api.post("/admin/gc", tags=["admin"]) async def run_gc(body: GcIn, s: Services = Depends(services)) -> dict[str, Any]: diff --git a/src/versiond/maintenance.py b/src/versiond/maintenance.py index d7cd9ae..ee23ace 100644 --- a/src/versiond/maintenance.py +++ b/src/versiond/maintenance.py @@ -94,12 +94,13 @@ async def run_scheduled_pass(s: Any, reason: str) -> dict[str, Any]: await dav.close() s.last_maintenance["gc"] = now s.last_maintenance["retention"] = now + reclaimed = recovery.vacuum_index(s.db) s.db.audit("maintenance.pass", reason=reason, deleted=deleted, - gc=gc_result["blobs_removed"]) + gc=gc_result["blobs_removed"], reclaimed_bytes=reclaimed["bytes_reclaimed"]) await refresh_usage(s) return {"reason": reason, "keep_days": keep_days, "versions_deleted": deleted, "kept_daily": plan["kept_daily"], - "gc": gc_result, "pressure": over_limit(s)} + "gc": gc_result, "index_reclaimed": reclaimed, "pressure": over_limit(s)} async def maintenance_loop(s: Any) -> None: diff --git a/src/versiond/recovery.py b/src/versiond/recovery.py index a1bc181..db999a6 100644 --- a/src/versiond/recovery.py +++ b/src/versiond/recovery.py @@ -13,6 +13,7 @@ from __future__ import annotations import asyncio import json +import os import time from datetime import datetime, timezone from typing import Any @@ -164,12 +165,30 @@ async def delete_versions(db: Database, ids: list[int], chunk: int = 500) -> int return deleted +def vacuum_index(db: Database) -> dict[str, Any]: + """Hard reclaim: VACUUM the freelist away, checkpoint+truncate the WAL. + + Deleted rows otherwise sit on freelist pages forever and the file never + shrinks. Returns byte counts so callers can report what was reclaimed. + """ + main = db.one("PRAGMA database_list")[0] if False else None + row = db.conn.execute("PRAGMA database_list").fetchone() + path = row[2] if row else "" + freelist_before = db.one("PRAGMA freelist_count")["freelist_count"] + size_before = os.path.getsize(path) if path and os.path.exists(path) else None + db.conn.execute("VACUUM") + db.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + freelist_after = db.one("PRAGMA freelist_count")["freelist_count"] + size_after = os.path.getsize(path) if path and os.path.exists(path) else None + reclaimed = (size_before - size_after) if size_before is not None and size_after is not None else None + return {"freelist_before": freelist_before, "freelist_after": freelist_after, + "bytes_before": size_before, "bytes_after": size_after, "bytes_reclaimed": reclaimed} + def orphan_blobs(db: Database) -> list[dict[str, Any]]: return db.all( "SELECT sha256, stored_size FROM blobs b WHERE NOT EXISTS " "(SELECT 1 FROM versions v WHERE v.blob_sha256 = b.sha256)") - async def collect_orphans( dav: WebDAV | None, db: Database, blobs: BlobStore, directory: str, orphans: list[dict[str, Any]], task: dict[str, Any] | None = None, diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index beaf187..f0dce9d 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -100,4 +100,5 @@ def test_scheduled_pass_cleans_old_and_reports(maint_env): conn.close() result = asyncio.run(maintenance.run_scheduled_pass(svc, "test-scheduled")) assert result["versions_deleted"] == 1 + assert result["index_reclaimed"]["freelist_after"] == 0 assert len(history(client, project / "main.py")) == 2 diff --git a/tests/test_recovery.py b/tests/test_recovery.py index 23614b0..77adf04 100644 --- a/tests/test_recovery.py +++ b/tests/test_recovery.py @@ -151,6 +151,7 @@ def test_retention_thins_old_keeps_five_days(rec_env): assert dry["dry_run"] and dry["versions_to_delete"] == 1 # 3 old minus pinned minus daily-kept done = client.post("/api/v1/admin/retention/run", json={"dry_run": False}).json() assert done["versions_deleted"] == 1 + assert done["index_reclaimed"]["freelist_after"] == 0 # hard delete, not freelist bloat remaining = history(client, project / "main.py") assert len(remaining) == 4 # latest + pinned + one-per-day + fresh middle assert remaining[0]["pinned"] == 0 # newest still the newest