Hard local maintenance: VACUUM + WAL checkpoint on every retention/forget execute

readme.md

@@ -328,7 +328,9 @@ When the remote is unreachable, the service keeps working from the spool and cat
 - **Retention** (`POST /api/v1/admin/retention/run`, dry run by default): keeps every
   version younger than `retention.keep_days` (default **5**), thins older ones to
   one per file per UTC day, and always keeps each file's latest version and all
-  pins. Follow with GC to reclaim the freed blobs.
+  pins. Every execute ends with a hard reclaim: `VACUUM` (freelist → 0) plus WAL
+  checkpoint/truncate, and reports `index_reclaimed` bytes. Follow with GC to
+  reclaim the freed blobs.
 - **Garbage collection** (`POST /api/v1/admin/gc`, dry run by default): deletes blobs
   no version references anymore, from the local spool and (unless `remote: false`)
   from WebDAV. Only unreferenced data is ever touched, so a crash can leak garbage

skills/versiond-admin/SKILL.md

@@ -43,6 +43,10 @@ server is the trust model); there are no keys to manage.
   thinned to 1/file/day; per-file latest + pins always kept.
 - `POST /admin/retention/run` (dry_run default true) then
   `POST /admin/gc` (dry_run default true, `remote: true` also wipes WebDAV).
+- Local maintenance rules: every retention/forget execute ends with
+  `VACUUM` + WAL checkpoint/truncate (`index_reclaimed` bytes reported), so
+  deletes really shrink `index.sqlite`; GC removes orphan spool blobs.
+  Without this, deletes would only pile onto freelist pages forever.
 - Built-in loop: retention daily, GC weekly (`[scheduler]`). No cron needed.
 - Capacity backstop `scheduler.remote_max_used_percent` (default 70):
   over-limit runs a pass but NEVER breaks the keep-days floor; health shows

src/versiond/api.py

@@ -774,9 +774,11 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
             blobs += chunk
             await asyncio.sleep(0)
         projects = s.repo.drop_empty_projects()
+        reclaimed = recovery.vacuum_index(s.db)
         s.db.audit("forget", path=path, files=files, blobs=blobs)
         return {"path": path, "dry_run": False, "files_removed": files,
-                "versions_removed": counts["versions"], "blobs_removed": blobs, "projects_removed": projects}
+                "versions_removed": counts["versions"], "blobs_removed": blobs, "projects_removed": projects,
+                "index_reclaimed": reclaimed}
 
     # remote purge (remote data only; never touches the local index or spool)
 
@@ -913,9 +915,12 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
             return {"dry_run": True, **{k: v for k, v in plan.items() if k != "delete_ids"},
                     "versions_to_delete": len(plan["delete_ids"])}
         deleted = await recovery.delete_versions(s.db, plan["delete_ids"])
-        s.db.audit("retention.run", keep_days=keep_days, deleted=deleted)
+        reclaimed = recovery.vacuum_index(s.db)
+        s.db.audit("retention.run", keep_days=keep_days, deleted=deleted,
+                   reclaimed_bytes=reclaimed["bytes_reclaimed"])
         return {"dry_run": False, "keep_days": keep_days, "candidates": plan["candidates"],
-                "kept_daily": plan["kept_daily"], "versions_deleted": deleted}
+                "kept_daily": plan["kept_daily"], "versions_deleted": deleted,
+                "index_reclaimed": reclaimed}
 
     @api.post("/admin/gc", tags=["admin"])
     async def run_gc(body: GcIn, s: Services = Depends(services)) -> dict[str, Any]:

src/versiond/maintenance.py

@@ -94,12 +94,13 @@ async def run_scheduled_pass(s: Any, reason: str) -> dict[str, Any]:
                     await dav.close()
             s.last_maintenance["gc"] = now
         s.last_maintenance["retention"] = now
+        reclaimed = recovery.vacuum_index(s.db)
         s.db.audit("maintenance.pass", reason=reason, deleted=deleted,
-                   gc=gc_result["blobs_removed"])
+                   gc=gc_result["blobs_removed"], reclaimed_bytes=reclaimed["bytes_reclaimed"])
         await refresh_usage(s)
         return {"reason": reason, "keep_days": keep_days,
                 "versions_deleted": deleted, "kept_daily": plan["kept_daily"],
-                "gc": gc_result, "pressure": over_limit(s)}
+                "gc": gc_result, "index_reclaimed": reclaimed, "pressure": over_limit(s)}
 
 
 async def maintenance_loop(s: Any) -> None:

src/versiond/recovery.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+import os
 import time
 from datetime import datetime, timezone
 from typing import Any
@@ -164,12 +165,30 @@ async def delete_versions(db: Database, ids: list[int], chunk: int = 500) -> int
     return deleted
 
 
+def vacuum_index(db: Database) -> dict[str, Any]:
+    """Hard reclaim: VACUUM the freelist away, checkpoint+truncate the WAL.
+
+    Deleted rows otherwise sit on freelist pages forever and the file never
+    shrinks. Returns byte counts so callers can report what was reclaimed.
+    """
+    main = db.one("PRAGMA database_list")[0] if False else None
+    row = db.conn.execute("PRAGMA database_list").fetchone()
+    path = row[2] if row else ""
+    freelist_before = db.one("PRAGMA freelist_count")["freelist_count"]
+    size_before = os.path.getsize(path) if path and os.path.exists(path) else None
+    db.conn.execute("VACUUM")
+    db.conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+    freelist_after = db.one("PRAGMA freelist_count")["freelist_count"]
+    size_after = os.path.getsize(path) if path and os.path.exists(path) else None
+    reclaimed = (size_before - size_after) if size_before is not None and size_after is not None else None
+    return {"freelist_before": freelist_before, "freelist_after": freelist_after,
+            "bytes_before": size_before, "bytes_after": size_after, "bytes_reclaimed": reclaimed}
+
 def orphan_blobs(db: Database) -> list[dict[str, Any]]:
     return db.all(
         "SELECT sha256, stored_size FROM blobs b WHERE NOT EXISTS "
         "(SELECT 1 FROM versions v WHERE v.blob_sha256 = b.sha256)")
 
-
 async def collect_orphans(
     dav: WebDAV | None, db: Database, blobs: BlobStore, directory: str, orphans: list[dict[str, Any]],
     task: dict[str, Any] | None = None,

tests/test_maintenance.py

@@ -100,4 +100,5 @@ def test_scheduled_pass_cleans_old_and_reports(maint_env):
     conn.close()
     result = asyncio.run(maintenance.run_scheduled_pass(svc, "test-scheduled"))
     assert result["versions_deleted"] == 1
+    assert result["index_reclaimed"]["freelist_after"] == 0
     assert len(history(client, project / "main.py")) == 2

tests/test_recovery.py

@@ -151,6 +151,7 @@ def test_retention_thins_old_keeps_five_days(rec_env):
     assert dry["dry_run"] and dry["versions_to_delete"] == 1  # 3 old minus pinned minus daily-kept
     done = client.post("/api/v1/admin/retention/run", json={"dry_run": False}).json()
     assert done["versions_deleted"] == 1
+    assert done["index_reclaimed"]["freelist_after"] == 0  # hard delete, not freelist bloat
     remaining = history(client, project / "main.py")
     assert len(remaining) == 4  # latest + pinned + one-per-day + fresh middle
     assert remaining[0]["pinned"] == 0  # newest still the newest