retoor/versioning
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove encryption; user-data filters; SQLite safety; recovery, retention, scheduler

Browse Source 
- No client-side encryption: plaintext content-addressed remote (SHA-256
  names), no backup key, no cryptography dependency (server disk encryption
  is the trust model).
- Filters back user data: images, archives, databases, PDFs accepted; 10 MiB
  cap; binary sniffing removed; temp names hardened (~$, #..#, .temp).
- SQLite zero-error policy: backup-API snapshots + integrity_check, journal
  folding, locked/corrupt loud skips, verified restores.
- Recovery: reindex from manifests, remote adopt, on-demand blob fetch,
  5-day retention + thinning, GC, date-guarded remote purge, metrics.
- Scheduler with WebDAV quota signal and 70% pressure backstop (floor kept).
- 37 tests incl. live-monitor capture safety and DB safety.
This commit is contained in:
retoor
2026-10-10 03:41:36 +02:00
parent 7e05e3d26c
commit 3499f6cda0
35 changed files with 3148 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/versiond/api.py
+358 -12
View File
@@ -23,17 +23,19 @@ from pydantic import BaseModel, Field
from importlib import resources
from . import __version__
from . import dbsafe
from . import diff as difflib_
from . import maintenance
from . import recovery
from . import stats as stats_
from .config import Config
from .crypto import KeyRing
from .db import Database
from .filters import Filters
from .ingest import Coalescer, Rejected, Repository, TokenBucket, tree_range
from .monitor import Monitor, RootError
from .remote import RemoteError, RemoteSettings
from .remote import RemoteError, RemoteSettings, WebDAV, join as remote_join, list_tree
from .restore import Criteria, Restorer, RestoreError
from .store import BlobStore
from .store import BlobStore, decompress
from .uploader import Uploader
log = logging.getLogger(__name__)
@@ -49,9 +51,9 @@ class Problem(Exception):
REJECT_STATUS = {"file-too-large": 413}
ROOT_STATUS = {"not-found": 404, "not-a-directory": 400, "root-overlap": 409, "watch-limit": 409}
REMOTE_STATUS = {"remote-auth": 400, "remote-unreachable": 502, "remote-http": 502,
"remote-full": 507, "key-mismatch": 409, "claim-failed": 409}
"remote-full": 507, "claim-failed": 409}
RESTORE_STATUS = {"not-found": 404, "unsafe-path": 400, "criteria-too-broad": 400,
"plan-expired": 410, "plan-used": 409, "conflict": 409}
"plan-expired": 410, "plan-used": 409, "conflict": 409, "unrestorable": 422}
def _problem(status: int, code: str, detail: str) -> JSONResponse:
@@ -92,6 +94,63 @@ def sd_notify(message: str) -> None:
log.debug("sd_notify failed", exc_info=True)
async def _run_purge(s: "Services", task_id: str, paths: list[str]) -> None:
"""Background half of purge-remote: WebDAV DELETEs only, no local writes."""
task = s.purge_tasks.get(task_id)
if task is None:
return
dav = WebDAV(s.uploader.settings(), transport=s.uploader.transport)
try:
for path in paths:
try:
await dav.delete(path)
task["files_deleted"] += 1
except Exception as exc: # noqa: BLE001 - recorded, never raised
task["errors"].append(f"{path}: {exc}")
if len(task["errors"]) > 50:
task["errors"].append("... truncated")
break
await asyncio.sleep(0)
# Remove now-empty subdirectories, deepest first. Best-effort: ignore failures.
parents = sorted({p.rsplit("/", 1)[0] for p in paths if "/" in p[1:]},
key=lambda p: p.count("/"), reverse=True)
top = task["directory"]
for parent in parents:
if parent == top or not parent.startswith(top + "/"):
continue
try:
await dav.delete(parent)
except Exception: # noqa: BLE001 - empty-dir cleanup is best-effort
pass
task["status"] = "failed" if task["errors"] else "done"
except Exception as exc: # noqa: BLE001 - task record is the error channel
task["status"] = "failed"
task["errors"].append(str(exc))
finally:
await dav.close()
async def _run_reindex(s: "Services", task_id: str) -> None:
"""Background half of reindex: downloads manifests, rebuilds the index."""
task = s.reindex_tasks.get(task_id)
if task is None:
return
dav = WebDAV(s.uploader.settings(), transport=s.uploader.transport)
try:
versions, renames, batches = await recovery.collect_manifest_records(dav, s.uploader.directory)
task["manifests"] = len(batches)
task["versions"] = len(versions)
counts = recovery.rebuild_index(s.db, versions, renames)
s.repo.reload_roots()
task.update(counts)
task["status"] = "done"
except Exception as exc: # noqa: BLE001 - task record is the error channel
task["status"] = "failed"
task["errors"].append(str(exc))
finally:
await dav.close()
# request models
class SnapshotIn(BaseModel):
@@ -134,6 +193,28 @@ class ForgetIn(BaseModel):
dry_run: bool = True
class PurgeRemoteIn(BaseModel):
today: str = Field(..., description="today's date as dd-mm-yyyy (safety confirmation)")
class AdoptIn(BaseModel):
url: str
username: str
password: str | None = None
base_path: str = "/versioned/"
directory: str = Field(..., description="existing remote directory to take over")
class RetentionIn(BaseModel):
dry_run: bool = True
keep_days: float | None = Field(None, description="override config retention.keep_days")
class GcIn(BaseModel):
dry_run: bool = True
remote: bool = Field(True, description="also delete orphan blobs from WebDAV")
class RestoreIn(BaseModel):
as_of: str | float | None = None
paths: list[str] = []
@@ -171,13 +252,50 @@ class Services:
max_watch_fraction=float(cfg.get("monitor", "max_watch_fraction")),
)
self.restorer = Restorer(self.db, self.repo, self.blobs)
self.keys = KeyRing.load_or_create(cfg.key_file)
self.uploader = Uploader(cfg, self.db, self.blobs, self.keys, transport=remote_transport)
self.uploader = Uploader(cfg, self.db, self.blobs, transport=remote_transport)
self.repo.on_commit = self.uploader.wake
rate = float(cfg.get("limits", "ingest_requests_per_second"))
self.ingest_limit = TokenBucket(rate, 1.0, time.monotonic)
self.token = cfg.api_token()
self.started_at = time.time()
self.purge_tasks: dict[str, dict[str, Any]] = {}
self.reindex_tasks: dict[str, dict[str, Any]] = {}
self.maintenance_lock = asyncio.Lock()
self.last_maintenance: dict[str, float] = {}
self.restorer.fetch_blob = self.blob_bytes
async def blob_bytes(self, sha256: str) -> bytes:
"""Blob content, fetching from WebDAV on demand after index loss.
Fast path is the local spool; the remote fetch only triggers when the
index says `uploaded` but no local copy exists (fresh reindex).
"""
try:
return self.blobs.get(sha256)
except FileNotFoundError:
pass
row = self.db.one("SELECT remote_state FROM blobs WHERE sha256 = ?", (sha256,))
if row is None or row["remote_state"] != "uploaded":
raise FileNotFoundError(sha256)
settings = self.uploader.settings()
if not settings.configured or not settings.directory:
raise FileNotFoundError(sha256)
dav = WebDAV(settings, transport=self.uploader.transport)
try:
data = await dav.get(remote_join(settings.directory, "blobs", sha256[:2], sha256))
finally:
await dav.close()
if data is None:
raise FileNotFoundError(sha256)
target = self.blobs.path_for(sha256)
target.parent.mkdir(parents=True, exist_ok=True)
tmp = target.with_name(f".{target.name}.{os.getpid()}.tmp")
with open(tmp, "wb") as fh:
fh.write(data)
fh.flush()
os.fsync(fh.fileno())
os.replace(tmp, target)
return decompress(data)
def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
@@ -191,12 +309,15 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
app.state.services = services
await services.monitor.start()
await services.uploader.start()
maint = asyncio.get_running_loop().create_task(maintenance.maintenance_loop(services))
sd_notify("READY=1")
log.info("versiond %s ready on %s:%s", __version__, cfg.host, cfg.port)
try:
yield
finally:
sd_notify("STOPPING=1")
maint.cancel()
await asyncio.gather(maint, return_exceptions=True)
await services.monitor.stop()
await services.uploader.stop()
flushed = services.coalescer.flush()
@@ -252,6 +373,10 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
async def health(s: Services = Depends(services)) -> dict[str, Any]:
roots = s.db.roots()
degraded = [r["path"] for r in roots if r["mode"] in ("degraded", "missing")]
usage = dict(getattr(s.uploader, "remote_usage", {}) or {})
pressure = maintenance.over_limit(s)
if pressure:
degraded = degraded + ["remote-over-capacity-limit"]
return {
"status": "degraded" if degraded else "ok",
"version": __version__,
@@ -261,6 +386,8 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
"pending_paths": sum(1 for p in s.coalescer.pending.values() if p.content is not None),
"watches": s.monitor.watch_total,
"remote": s.uploader.state,
"remote_usage": usage,
"remote_pressure": pressure,
}
@app.get("/dashboard", include_in_schema=False)
@@ -538,7 +665,10 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
@api.get("/versions/{version_id}/content", tags=["versions"])
async def get_content(version_id: int, s: Services = Depends(services)) -> Response:
version = _version(s, version_id)
content = s.blobs.get(version["blob_sha256"])
try:
content = await s.blob_bytes(version["blob_sha256"])
except (FileNotFoundError, ValueError):
raise Problem(404, "not-found", "blob is gone locally and remotely")
try:
return PlainTextResponse(content.decode("utf-8"))
except UnicodeDecodeError:
@@ -568,7 +698,16 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
if body.conflict == "rename":
stem, ext = os.path.splitext(destination)
destination = f"{stem}.restored-{time.strftime('%Y%m%d-%H%M%S')}{ext}"
content = s.blobs.get(version["blob_sha256"])
content: bytes
try:
content = await s.blob_bytes(version["blob_sha256"])
except (FileNotFoundError, ValueError):
raise RestoreError("unrestorable", f"blob {version['blob_sha256']} is gone or corrupt")
if dbsafe.is_sqlite_image(content):
try:
dbsafe.verify_sqlite_bytes(content)
except dbsafe.DatabaseUnsafe as exc:
raise RestoreError("unrestorable", f"stored version fails integrity check: {exc.reason}")
result = s.restorer.write(destination, content, version_id)
s.db.audit("restore.version", version_id=version_id, destination=destination)
return {"result": "restored", "destination": destination, **result}
@@ -586,7 +725,10 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
s: Services = Depends(services),
):
a_version = _version(s, from_version)
a = s.blobs.get(a_version["blob_sha256"])
try:
a = await s.blob_bytes(a_version["blob_sha256"])
except (FileNotFoundError, ValueError):
raise Problem(404, "not-found", "blob is gone locally and remotely")
from_label = f"{a_version['path']}@{from_version}"
if to == "disk":
try:
@@ -599,7 +741,10 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
if not to.isdigit():
raise Problem(400, "invalid-target", "to must be a version id or 'disk'")
b_version = _version(s, int(to))
b = s.blobs.get(b_version["blob_sha256"])
try:
b = await s.blob_bytes(b_version["blob_sha256"])
except (FileNotFoundError, ValueError):
raise Problem(404, "not-found", "blob is gone locally and remotely")
to_label = f"{b_version['path']}@{to}"
if format == "html":
return HTMLResponse(difflib_.side_by_side_html(a, b, from_label, to_label, context))
@@ -633,6 +778,207 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
return {"path": path, "dry_run": False, "files_removed": files,
"versions_removed": counts["versions"], "blobs_removed": blobs, "projects_removed": projects}
# remote purge (remote data only; never touches the local index or spool)
@api.post("/admin/purge-remote", tags=["admin"], status_code=202)
async def purge_remote(body: PurgeRemoteIn, s: Services = Depends(services)) -> dict[str, Any]:
"""Delete everything this installation uploaded (blobs/ + manifests/).
Safety: `today` must be today's date as dd-mm-yyyy. Only the claimed
remote directory is ever deleted, never the base path or anything else.
The listing happens synchronously so the response can state how many
files/bytes will go; the actual deletes run in the background.
Nothing local (index, spool, config) is read or modified here.
"""
try:
given = datetime.strptime(body.today.strip(), "%d-%m-%Y").date()
except ValueError:
raise Problem(400, "invalid-date", "today must be today's date as dd-mm-yyyy (e.g. "
f"{datetime.now().strftime('%d-%m-%Y')})")
if given != datetime.now().date():
raise Problem(400, "invalid-date", "today must be today's date as dd-mm-yyyy; "
f"today is {datetime.now().strftime('%d-%m-%Y')}")
settings = s.uploader.settings()
directory = settings.directory.rstrip("/") or ""
base = remote_join(settings.base_path)
if not settings.configured or not directory:
raise Problem(409, "remote-unconfigured", "no WebDAV remote is configured")
if directory == base or directory == "/" or not directory.startswith(base + "/"):
raise Problem(409, "refusing-purge", f"refusing to purge {directory!r}: outside the claimed directory")
dav = WebDAV(settings, transport=s.uploader.transport)
try:
files: list = []
for subtree in ("blobs", "manifests"):
top = remote_join(directory, subtree)
if await dav.exists(top):
sub_files, _ = await list_tree(dav, top)
files.extend(sub_files)
finally:
await dav.close()
total_bytes: int | None = sum(f.size for f in files if f.size is not None)
if any(f.size is None for f in files):
total_bytes = None # server omitted some sizes; file count is exact
task_id = secrets.token_hex(8)
task: dict[str, Any] = {
"task_id": task_id, "directory": directory, "status": "running",
"files": len(files), "bytes": total_bytes,
"files_deleted": 0, "errors": [],
}
s.purge_tasks[task_id] = task
asyncio.get_running_loop().create_task(_run_purge(s, task_id, [f.path for f in files]))
return {"task_id": task_id, "directory": directory, "status": "running",
"files": len(files), "bytes": total_bytes,
"note": "deletion runs in the background; poll GET /api/v1/admin/purge-remote/{task_id}"}
@api.get("/admin/purge-remote/{task_id}", tags=["admin"])
async def purge_remote_status(task_id: str, s: Services = Depends(services)) -> dict[str, Any]:
task = s.purge_tasks.get(task_id)
if task is None:
raise Problem(404, "not-found", f"purge task {task_id} does not exist")
return task
# disaster recovery: rebuild the index from remote manifests
@api.post("/admin/reindex", tags=["admin"], status_code=202)
async def start_reindex(s: Services = Depends(services)) -> dict[str, Any]:
"""Rebuild the local index from remote manifests (fresh machine / lost index).
Lists manifests, replays version+rename records, marks everything
durable. Responds immediately; poll the status endpoint. Local files
are only re-registered as metadata; nothing is downloaded except
manifests (blobs stay remote until restored).
"""
settings = s.uploader.settings()
if not settings.configured or not settings.directory:
raise Problem(409, "remote-unconfigured", "no WebDAV remote is configured")
task_id = secrets.token_hex(8)
task: dict[str, Any] = {"task_id": task_id, "status": "running",
"manifests": 0, "versions": 0, "files": 0, "errors": []}
s.reindex_tasks[task_id] = task
asyncio.get_running_loop().create_task(_run_reindex(s, task_id))
return {**task, "note": "poll GET /api/v1/admin/reindex/{task_id}"}
@api.get("/admin/reindex/{task_id}", tags=["admin"])
async def reindex_status(task_id: str, s: Services = Depends(services)) -> dict[str, Any]:
task = s.reindex_tasks.get(task_id)
if task is None:
raise Problem(404, "not-found", f"reindex task {task_id} does not exist")
return task
@api.post("/config/remote/adopt", tags=["remote"])
async def adopt_remote(body: AdoptIn, s: Services = Depends(services)) -> dict[str, Any]:
"""Take over an existing remote directory (new machine after total loss)."""
password = body.password if body.password is not None else s.cfg.credential("remote_password")
if not password:
raise Problem(400, "missing-password", "password is required")
settings = RemoteSettings(url=body.url, username=body.username, password=password,
base_path=body.base_path, verify_tls=True, timeout_seconds=30.0,
directory=body.directory.rstrip("/") or "")
if not settings.directory or settings.directory == remote_join(settings.base_path):
raise Problem(400, "invalid-directory", "directory must be an existing claimed directory")
dav = WebDAV(settings, transport=s.uploader.transport)
try:
owner_raw = await dav.get(remote_join(settings.directory, "meta", "owner.json"))
if owner_raw is None:
raise Problem(404, "not-found",
f"{settings.directory} has no owner.json; nothing to adopt")
try:
owner = json.loads(owner_raw)
except ValueError:
owner = {}
finally:
await dav.close()
await s.uploader.stop()
s.cfg.update("remote", {
"url": settings.url, "username": settings.username, "base_path": settings.base_path,
"verify_tls": True, "timeout_seconds": 30.0, "directory": settings.directory,
})
s.cfg.set_credential("remote_password", password)
s.db.audit("remote.adopt", url=settings.url, directory=settings.directory, owner=owner)
await s.uploader.start()
return {**s.uploader.public_settings(), "adopted": True, "previous_owner": owner}
# retention + garbage collection
@api.post("/admin/retention/run", tags=["admin"])
async def run_retention(body: RetentionIn, s: Services = Depends(services)) -> dict[str, Any]:
"""Thin old history: keep everything younger than keep_days (default 5),
one version per file per older day, plus every per-file latest and all
pins. Dry run by default; follow with POST /admin/gc to reclaim blobs."""
keep_days = body.keep_days if body.keep_days is not None else float(s.cfg.get("retention", "keep_days"))
if keep_days < 0:
raise Problem(400, "invalid-keep-days", "keep_days must be >= 0")
plan = recovery.retention_plan(s.db, keep_days)
if body.dry_run:
return {"dry_run": True, **{k: v for k, v in plan.items() if k != "delete_ids"},
"versions_to_delete": len(plan["delete_ids"])}
deleted = await recovery.delete_versions(s.db, plan["delete_ids"])
s.db.audit("retention.run", keep_days=keep_days, deleted=deleted)
return {"dry_run": False, "keep_days": keep_days, "candidates": plan["candidates"],
"kept_daily": plan["kept_daily"], "versions_deleted": deleted}
@api.post("/admin/gc", tags=["admin"])
async def run_gc(body: GcIn, s: Services = Depends(services)) -> dict[str, Any]:
"""Delete blobs no version references (local spool + optionally remote)."""
orphans = recovery.orphan_blobs(s.db)
total_bytes = sum(o["stored_size"] or 0 for o in orphans)
if body.dry_run:
return {"dry_run": True, "blobs": len(orphans), "bytes": total_bytes}
dav = None
directory = ""
if body.remote:
settings = s.uploader.settings()
if settings.configured and settings.directory:
dav = WebDAV(settings, transport=s.uploader.transport)
directory = settings.directory
try:
result = await recovery.collect_orphans(dav, s.db, s.blobs, directory, orphans)
finally:
if dav is not None:
await dav.close()
s.db.audit("gc", **{k: v for k, v in result.items() if k != "errors"})
return {"dry_run": False, "blobs": len(orphans), "bytes": total_bytes, **result}
# metrics
@api.get("/metrics", tags=["status"])
async def get_metrics(s: Services = Depends(services)):
"""Prometheus text exposition (behind the same bearer auth)."""
files = s.db.one("SELECT count(*) AS n FROM files")["n"]
versions = s.db.one("SELECT count(*) AS n FROM versions")["n"]
local = s.db.one("SELECT count(*) AS n FROM versions WHERE durability = 'local'")["n"]
pending = s.db.one("SELECT count(*) AS n FROM blobs WHERE remote_state IN ('pending','failed')")["n"]
failed = s.db.one("SELECT count(*) AS n FROM blobs WHERE remote_state = 'failed'")["n"]
events = s.monitor.counters.get("events", 0)
uptime = round(time.time() - s.started_at)
body = (
"# HELP versiond_files_tracked Files in the index\n"
"# TYPE versiond_files_tracked gauge\n"
f"versiond_files_tracked {files}\n"
"# HELP versiond_versions_total Stored versions\n"
"# TYPE versiond_versions_total gauge\n"
f"versiond_versions_total {versions}\n"
"# HELP versiond_versions_local Versions not yet durable remotely\n"
"# TYPE versiond_versions_local gauge\n"
f"versiond_versions_local {local}\n"
"# HELP versiond_blobs_pending Blobs awaiting (re)upload\n"
"# TYPE versiond_blobs_pending gauge\n"
f"versiond_blobs_pending {pending}\n"
"# HELP versiond_blobs_failed Blobs in failed/backoff state\n"
"# TYPE versiond_blobs_failed gauge\n"
f"versiond_blobs_failed {failed}\n"
"# HELP versiond_monitor_events_total Inotify events seen\n"
"# TYPE versiond_monitor_events_total counter\n"
f"versiond_monitor_events_total {events}\n"
"# HELP versiond_remote_state Remote state (1 when online)\n"
"# TYPE versiond_remote_state gauge\n"
f"versiond_remote_state {1 if s.uploader.state == 'online' else 0}\n"
"# HELP versiond_uptime_seconds Process uptime\n"
"# TYPE versiond_uptime_seconds counter\n"
f"versiond_uptime_seconds {uptime}\n"
)
return Response(content=body, media_type="text/plain; version=0.0.4")
# bulk restore
@api.post("/restores", tags=["restore"], status_code=201)
@@ -650,7 +996,7 @@ def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None,
@api.post("/restores/{plan_id}/execute", tags=["restore"])
async def execute_restore(plan_id: str, s: Services = Depends(services)) -> dict[str, Any]:
return s.restorer.execute(plan_id)
return await s.restorer.execute(plan_id)
app.include_router(api)
return app
src/versiond/cli.py
+2 -16
View File
@@ -93,7 +93,7 @@ def cmd_install(args: argparse.Namespace) -> int:
print(f"wrote {unit_path}")
Config.load().api_token()
user = os.environ.get("USER") or os.getlogin()
user = os.environ.get("USER") or os.environ.get("LOGNAME") or getpass.getuser()
linger = _run("loginctl", "show-user", user, "-p", "Linger").stdout.strip()
if linger != "Linger=yes":
result = _run("loginctl", "enable-linger", user)
@@ -306,8 +306,7 @@ def cmd_remote(args: argparse.Namespace) -> int:
result = c.request("PUT", "/api/v1/config/remote", body)
verb = "adopted existing" if result.get("adopted") else "claimed new"
print(f"remote configured; {verb} directory {result['directory']} on {result['url']}")
print(f"encryption: {result['encryption']}, key id {result['key_id']}")
print("IMPORTANT: back up your key with `versiond key export`; without it the remote copy cannot be read.")
print("remote storage is unencrypted; it relies on the server's disk encryption.")
return 0
@@ -323,15 +322,6 @@ def cmd_forget(args: argparse.Namespace) -> int:
return 0
def cmd_key(args: argparse.Namespace) -> int:
from .crypto import KeyRing
keys = KeyRing.load_or_create(Config.load().key_file)
print(keys.export())
print(f"# key id {keys.key_id} - store this somewhere safe and off this machine", file=sys.stderr)
return 0
def cmd_stats(args: argparse.Namespace) -> int:
s = Client().request("GET", "/api/v1/stats")
if args.json:
@@ -468,10 +458,6 @@ def build_parser() -> argparse.ArgumentParser:
p.add_argument("--execute", action="store_true")
p.set_defaults(func=cmd_forget)
p = sub.add_parser("key", help="backup encryption key")
p.add_argument("action", choices=["export"])
p.set_defaults(func=cmd_key)
p = sub.add_parser("stats", help="statistics: files, versions, storage, activity")
p.add_argument("--json", action="store_true")
p.set_defaults(func=cmd_stats)
src/versiond/config.py
+11 -5
View File
@@ -17,7 +17,7 @@ DEFAULT_PORT = 9922
DEFAULTS: dict[str, Any] = {
"server": {"host": DEFAULT_HOST, "port": DEFAULT_PORT},
"limits": {
"max_file_bytes": 204_800,
"max_file_bytes": 10_485_760,
"ingest_requests_per_second": 50,
},
"coalesce": {
@@ -50,6 +50,16 @@ DEFAULTS: dict[str, Any] = {
"reconcile_interval_seconds": 86_400.0,
"max_watch_fraction": 0.5,
},
"retention": {
"keep_days": 5,
},
"scheduler": {
"enabled": True,
"retention_interval_seconds": 86_400.0,
"gc_interval_seconds": 604_800.0,
"usage_check_seconds": 600.0,
"remote_max_used_percent": 70.0,
},
}
@@ -196,7 +206,3 @@ class Config:
token = secrets.token_urlsafe(32)
self.set_credential("api_token", token)
return token
@property
def key_file(self):
return self.paths.config_dir / "backup.key"
src/versiond/crypto.py
-57
View File
@@ -1,57 +0,0 @@
"""Client-side encryption for everything sent to the remote."""
from __future__ import annotations
import base64
import hashlib
import hmac
import os
from pathlib import Path
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
MAGIC = b"VD1"
NONCE_BYTES = 12
AAD = b"versiond"
class KeyRing:
"""A 64-byte master key: 32 bytes for AES-256-GCM, 32 bytes for keyed naming."""
def __init__(self, key: bytes):
if len(key) != 64:
raise ValueError("backup key must be 64 bytes")
self._aead = AESGCM(key[:32])
self._mac_key = key[32:]
self.key = key
@classmethod
def load_or_create(cls, path: Path) -> "KeyRing":
if path.exists():
return cls(base64.b64decode(path.read_text().strip()))
key = os.urandom(64)
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, "w") as fh:
fh.write(base64.b64encode(key).decode() + "\n")
return cls(key)
def export(self) -> str:
return base64.b64encode(self.key).decode()
@property
def key_id(self) -> str:
return hmac.new(self._mac_key, b"versiond-key-id", hashlib.sha256).hexdigest()[:16]
def blob_name(self, digest: str) -> str:
"""Remote name for a blob; keyed so the server cannot confirm guessed content."""
return hmac.new(self._mac_key, digest.encode(), hashlib.sha256).hexdigest()
def encrypt(self, data: bytes) -> bytes:
nonce = os.urandom(NONCE_BYTES)
return MAGIC + nonce + self._aead.encrypt(nonce, data, AAD)
def decrypt(self, data: bytes) -> bytes:
if not data.startswith(MAGIC):
raise ValueError("not a versiond encrypted object")
nonce = data[len(MAGIC): len(MAGIC) + NONCE_BYTES]
return self._aead.decrypt(nonce, data[len(MAGIC) + NONCE_BYTES:], AAD)
src/versiond/dbsafe.py
+142
View File
@@ -0,0 +1,142 @@
"""Enterprise-safe snapshots of live SQLite databases.
A raw byte copy of a live (especially WAL-mode) SQLite file can be torn or
silently stale: the main file, -wal and -shm must be captured in one instant,
which a plain read cannot do. So any file with the SQLite magic is snapshotted
through the SQLite Online Backup API instead of read directly:
1. Open the source read-only (never triggers WAL recovery or checkpointing,
never takes a write lock, never mutates the live DB).
2. Copy page-by-page with ``Connection.backup()`` (restarts safely if the
source is being written to; retries while the source is locked).
3. Run ``PRAGMA integrity_check`` on the snapshot; only ``ok`` is stored.
4. Anything else (locked past the deadline, unreadable, corrupt) raises
``DatabaseUnsafe`` and nothing is versioned -- a loud skip beats a silent
corrupt version. Monitor/API surfaces it as ``database-locked`` /
``database-corrupt`` instead of storing garbage.
Journal files (``*-wal``, ``*-shm``, ``*-journal``) are never versioned on
their own (see filters: ``database-journal``); they are folded into the main
file's snapshot. For non-SQLite engines (postgres/mysql data files) only a
crash-consistent raw copy is possible -- dump-then-backup remains required.
"""
from __future__ import annotations
import os
import queue
import sqlite3
import tempfile
import threading
from pathlib import Path
SQLITE_MAGIC = b"SQLite format 3\x00"
SNAPSHOT_TIMEOUT_SECONDS = 5.0
class DatabaseUnsafe(Exception):
"""Raised when no consistent snapshot can be produced. Never store partial data."""
def __init__(self, reason: str, detail: str = ""):
super().__init__(detail or reason)
self.reason = reason
self.detail = detail
def is_sqlite_image(data: bytes) -> bool:
return data[: len(SQLITE_MAGIC)] == SQLITE_MAGIC
def _integrity_ok(conn: sqlite3.Connection) -> bool:
row = conn.execute("PRAGMA integrity_check").fetchone()
return row is not None and row[0] == "ok"
def verify_sqlite_bytes(data: bytes) -> None:
"""Raise DatabaseUnsafe unless data is a fully consistent SQLite image."""
if not is_sqlite_image(data):
raise DatabaseUnsafe("not-a-database", "missing SQLite magic")
fd, tmp = tempfile.mkstemp(prefix="versiond-verify-", suffix=".sqlite")
try:
with os.fdopen(fd, "wb") as fh:
fh.write(data)
conn = sqlite3.connect(f"file:{tmp}?mode=ro", uri=True)
try:
if not _integrity_ok(conn):
raise DatabaseUnsafe("database-corrupt", "integrity_check failed on pushed content")
finally:
conn.close()
except DatabaseUnsafe:
raise
except Exception as exc:
raise DatabaseUnsafe("database-unreadable", str(exc)) from exc
finally:
try:
os.unlink(tmp)
except OSError:
pass
def snapshot_sqlite(path: str, max_bytes: int, timeout: float = SNAPSHOT_TIMEOUT_SECONDS) -> bytes:
"""Consistent, verified snapshot of a live SQLite DB file. Read-only on source.
The backup runs on a daemon worker thread bounded by ``timeout``: a source
locked past the deadline fails loudly (``database-locked``) instead of
blocking the monitor loop forever. SQLite connections are thread-local to
the worker.
"""
out: queue.Queue = queue.Queue(maxsize=1)
def _work() -> None:
try:
out.put((True, _snapshot_once(path, max_bytes)))
except Exception as exc: # noqa: BLE001 - ferried back to the caller
out.put((False, exc))
worker = threading.Thread(target=_work, daemon=True)
worker.start()
try:
ok, payload = out.get(timeout=timeout)
except queue.Empty as exc:
raise DatabaseUnsafe("database-locked", f"{path}: still locked after {timeout}s") from exc
if ok:
return payload
if isinstance(payload, DatabaseUnsafe):
raise payload
raise DatabaseUnsafe("database-unreadable", f"{path}: {payload}") from payload
def _snapshot_once(path: str, max_bytes: int) -> bytes:
fd, tmp = tempfile.mkstemp(prefix="versiond-db-", suffix=".sqlite")
os.close(fd)
try:
try:
src = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=1.0)
except Exception as exc:
raise DatabaseUnsafe("database-unreadable", f"{path}: {exc}") from exc
try:
dst = sqlite3.connect(tmp)
try:
src.backup(dst)
except sqlite3.OperationalError as exc:
raise DatabaseUnsafe("database-locked", f"{path}: {exc}") from exc
finally:
dst.close()
finally:
src.close()
conn = sqlite3.connect(f"file:{tmp}?mode=ro", uri=True)
try:
if not _integrity_ok(conn):
raise DatabaseUnsafe("database-corrupt", f"integrity_check failed for {path}")
finally:
conn.close()
data = Path(tmp).read_bytes()
if len(data) > max_bytes:
raise DatabaseUnsafe("file-too-large", f"{len(data)} bytes")
return data
finally:
try:
os.unlink(tmp)
except OSError:
pass
src/versiond/filters.py
+21 -20
View File
@@ -8,31 +8,30 @@ from pathlib import PurePath
IGNORED_DIRS = frozenset({
"node_modules", "bower_components", "jspm_packages", "vendor", "__pycache__",
"venv", "env", "site-packages", "build", "dist", "target", "out", "bin", "obj",
"Pods", "Carthage", "DerivedData", "coverage", "_build", "deps", "elm-stuff",
"venv", "env", "site-packages", ".tox", "build", "dist", "target", "out", "bin", "obj",
".gradle", "Pods", "Carthage", "DerivedData", ".next", ".nuxt", ".svelte-kit",
"coverage", ".terraform", "_build", "deps", "elm-stuff",
"zig-cache", "zig-out", "__pypackages__", "htmlcov",
})
IGNORED_DIR_SUFFIXES = (".egg-info", ".dist-info", ".xcodeproj", ".xcworkspace")
IGNORED_SUFFIXES = (
# compiled / object code
# compiled / object code (reproducible from source; lockfiles/manifests are kept)
".pyc", ".pyo", ".pyd", ".class", ".o", ".obj", ".so", ".dylib", ".dll", ".exe",
".a", ".lib", ".wasm", ".jar", ".war", ".ear", ".whl", ".egg", ".beam", ".elc",
# archives
".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".zst", ".7z", ".rar",
# media
".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".bmp", ".tiff", ".psd",
".mp3", ".mp4", ".wav", ".ogg", ".flac", ".mov", ".avi", ".mkv", ".webm",
".pdf", ".ttf", ".otf", ".woff", ".woff2", ".eot",
# databases
".sqlite", ".sqlite3", ".db", ".db-wal", ".db-shm", ".sqlite-wal", ".sqlite-shm",
# generated
# generated bundles (rebuilt by the toolchain)
".min.js", ".min.css", ".map",
# editor temp files
".swp", ".swo", ".swx", ".tmp", ".part", ".crdownload", ".orig", ".rej",
# editor / downloader temp files (unfinished work: never versioned)
".swp", ".swo", ".swx", ".kate-swp", ".tmp", ".temp", ".part", ".crdownload",
".orig", ".rej",
)
# NOTE: images, audio/video, archives, databases (incl. -wal/-shm journals),
# documents (.pdf) and fonts are deliberately ACCEPTED: they are user data that
# backup guides list as must-include. Dependency/build/cache directories above
# stay excluded; only the per-file content types changed.
IGNORED_NAMES = frozenset({"4913"}) # vim's write-permission probe file
ALLOWED_DOTFILES = frozenset({
@@ -47,7 +46,7 @@ ALLOWED_DOTFILES = frozenset({
@dataclass
class Filters:
max_file_bytes: int = 204_800
max_file_bytes: int = 10_485_760 # 10 MiB: covers images, archives, small DBs
extra_dirs: frozenset[str] = field(default_factory=frozenset)
patterns: tuple[str, ...] = ()
extra_dotfiles: frozenset[str] = field(default_factory=frozenset)
@@ -97,10 +96,16 @@ class Filters:
if self.dir_path_ignored("/".join(parts[:depth])):
return "ignored-pattern"
name = parts[-1]
if name.endswith(("-wal", "-shm", "-journal")):
# SQLite journals are folded into the main file's verified snapshot
# (see dbsafe); they are never independently restorable versions.
return "database-journal"
if name.startswith("."):
if not self._dotfile_allowed(name):
return "hidden-file"
elif name.endswith("~") or name in IGNORED_NAMES:
elif (name.endswith("~") or name in IGNORED_NAMES or name.startswith("~$")
or (name.startswith("#") and name.endswith("#"))):
# trailing-~ backups, vim probe, Office locks, Emacs autosaves
return "temporary-file"
lower = name.lower()
if lower.endswith(IGNORED_SUFFIXES):
@@ -113,7 +118,3 @@ class Filters:
def size_reason(self, size: int) -> str | None:
return "file-too-large" if size > self.max_file_bytes else None
@staticmethod
def content_reason(content: bytes) -> str | None:
return "binary-content" if b"\x00" in content[:8192] else None
src/versiond/ingest.py
+35 -3
View File
@@ -12,6 +12,7 @@ from pathlib import Path, PurePath
from typing import Any, Callable
from .db import Database
from . import dbsafe
from .filters import Filters
from .store import BlobStore, sha256
@@ -83,12 +84,22 @@ class Repository:
raise Rejected(reason, path)
def check_content(self, content: bytes) -> None:
reason = self.filters.size_reason(len(content)) or self.filters.content_reason(content)
reason = self.filters.size_reason(len(content))
if reason:
raise Rejected(reason, f"{len(content)} bytes")
if dbsafe.is_sqlite_image(content):
# Pushed bytes claiming to be a database: verify before storing.
try:
dbsafe.verify_sqlite_bytes(content)
except dbsafe.DatabaseUnsafe as exc:
raise Rejected(exc.reason, exc.detail) from exc
def read_file(self, path: str) -> tuple[bytes, os.stat_result]:
"""Read a regular file without following a final symlink."""
"""Read a regular file without following a final symlink.
SQLite images never go through the raw path: they are snapshotted via
the Online Backup API + integrity_check (see dbsafe).
"""
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
try:
st = os.fstat(fd)
@@ -97,12 +108,29 @@ class Repository:
reason = self.filters.size_reason(st.st_size)
if reason:
raise Rejected(reason, f"{st.st_size} bytes")
magic = os.pread(fd, len(dbsafe.SQLITE_MAGIC), 0)
if dbsafe.is_sqlite_image(magic):
os.close(fd)
return self._read_live_db(path)
with os.fdopen(fd, "rb", closefd=False) as fh:
content = fh.read(self.filters.max_file_bytes + 1)
finally:
os.close(fd)
try:
os.close(fd)
except OSError:
pass
return content, st
def _read_live_db(self, path: str) -> tuple[bytes, os.stat_result]:
try:
data = dbsafe.snapshot_sqlite(path, self.filters.max_file_bytes)
except dbsafe.DatabaseUnsafe as exc:
raise Rejected(exc.reason, exc.detail or path) from exc
st = os.stat(path)
fields = list(st)
fields[6] = len(data) # st_size tracks the verified snapshot, not the live file
return data, os.stat_result(fields)
# projects
def _detect_project(self, path: str, root: dict[str, Any] | None) -> tuple[str, str]:
@@ -365,6 +393,10 @@ class Coalescer:
def _bucket(self, path: str) -> TokenBucket:
bucket = self.buckets.get(path)
if bucket is None:
# Bound memory: the bucket table would otherwise grow with every
# distinct path ever seen. Evict the oldest entry first.
if len(self.buckets) >= 10_000:
self.buckets.pop(next(iter(self.buckets)))
bucket = self.buckets[path] = TokenBucket(self.per_hour, 3600, self.clock)
return bucket
src/versiond/maintenance.py
+124
View File
@@ -0,0 +1,124 @@
"""Built-in maintenance scheduler.
Cadence (all configurable under ``[scheduler]``):
- retention pass every ``retention_interval_seconds`` (default daily),
- GC folded into the pass every ``gc_interval_seconds`` (default weekly),
- remote usage refreshed every ``usage_check_seconds`` (default 10 min).
Capacity is a backstop, never the driver: when remote usage reaches
``remote_max_used_percent`` (default 70) an out-of-schedule pass runs and the
service reports pressure (see /health) so a human analyzes — the 5-day
retention floor is never violated automatically. This mirrors the surveyed
consensus: time policy first, pressure relief second, static ceilings rather
than autotuning.
"""
from __future__ import annotations
import asyncio
import logging
import time
from typing import Any
from . import recovery
from .remote import WebDAV, join as remote_join, quota as remote_quota
log = logging.getLogger(__name__)
def _scheduler_cfg(s: Any, key: str) -> float:
return float(s.cfg.get("scheduler", key))
async def refresh_usage(s: Any) -> dict[str, Any]:
"""Snapshot remote usage; cached on the uploader for /health and /progress."""
usage: dict[str, Any] = {"percent": None, "used": None, "total": None,
"source": "unconfigured", "at": time.time()}
settings = s.uploader.settings()
if settings.configured and settings.directory:
dav = WebDAV(settings, transport=s.uploader.transport)
try:
try:
used, available = await remote_quota(dav, settings.directory)
except Exception as exc:
usage = {**usage, "source": "error", "error": str(exc)}
else:
if used is not None:
total = used + available if available is not None else None
usage = {
"percent": round(100 * used / total, 1) if total else None,
"used": used, "total": total, "source": "webdav-quota",
"at": time.time(),
}
else:
row = s.db.one(
"SELECT coalesce(sum(b.stored_size), 0) AS n FROM versions v "
"JOIN blobs b ON b.sha256 = v.blob_sha256 WHERE v.durability = 'durable'")
usage = {"percent": None, "used": row["n"], "total": None,
"source": "index-estimate", "at": time.time()}
finally:
await dav.close()
s.uploader.remote_usage = usage
return usage
def over_limit(s: Any) -> bool:
usage = getattr(s.uploader, "remote_usage", None) or {}
percent = usage.get("percent")
if percent is None:
return False
return percent >= _scheduler_cfg(s, "remote_max_used_percent")
async def run_scheduled_pass(s: Any, reason: str) -> dict[str, Any]:
"""One maintenance pass: refresh usage, retention, GC if due. Serialized."""
async with s.maintenance_lock:
await refresh_usage(s)
keep_days = float(s.cfg.get("retention", "keep_days"))
plan = recovery.retention_plan(s.db, keep_days)
deleted = await recovery.delete_versions(s.db, plan["delete_ids"])
now = time.time()
gc_due = now - s.last_maintenance.get("gc", 0) >= _scheduler_cfg(s, "gc_interval_seconds")
gc_result: dict[str, Any] = {"blobs_removed": 0, "remote_removed": 0, "errors": []}
if gc_due:
orphans = recovery.orphan_blobs(s.db)
dav = None
settings = s.uploader.settings()
if settings.configured and settings.directory:
dav = WebDAV(settings, transport=s.uploader.transport)
try:
gc_result = await recovery.collect_orphans(
dav, s.db, s.blobs, settings.directory if dav else "", orphans)
finally:
if dav is not None:
await dav.close()
s.last_maintenance["gc"] = now
s.last_maintenance["retention"] = now
s.db.audit("maintenance.pass", reason=reason, deleted=deleted,
gc=gc_result["blobs_removed"])
await refresh_usage(s)
return {"reason": reason, "keep_days": keep_days,
"versions_deleted": deleted, "kept_daily": plan["kept_daily"],
"gc": gc_result, "pressure": over_limit(s)}
async def maintenance_loop(s: Any) -> None:
"""Daemon loop; cancelled on shutdown. Never raises out."""
if not s.cfg.data.get("scheduler", {}).get("enabled", True):
return
await refresh_usage(s)
while True:
try:
await asyncio.sleep(_scheduler_cfg(s, "usage_check_seconds"))
await refresh_usage(s)
now = time.time()
if over_limit(s):
log.warning("remote usage over limit; running pressure maintenance")
await run_scheduled_pass(s, "capacity-pressure")
elif now - s.last_maintenance.get("retention", 0) >= _scheduler_cfg(
s, "retention_interval_seconds"):
await run_scheduled_pass(s, "scheduled")
except asyncio.CancelledError:
raise
except Exception:
log.exception("maintenance pass failed; will retry next interval")
src/versiond/recovery.py
+200
View File
@@ -0,0 +1,200 @@
"""Disaster recovery (reindex from remote), retention and garbage collection.
Reindex rebuilds the local SQLite index purely from remote manifests, so a
total local loss (disk dead, fresh machine after adopt) is recoverable.
Retention keeps every version younger than ``keep_days`` (default 5), thins
older ones to one-per-day plus the per-file latest, and never touches pinned
versions. GC deletes blobs no version references anymore, locally and (on
request) remotely. All three are additive-safe: they can only leak remote
garbage, never delete referenced data.
"""
from __future__ import annotations
import asyncio
import json
import time
from datetime import datetime, timezone
from typing import Any
from .db import Database
from .remote import WebDAV, join as remote_join, list_tree
from .store import BlobStore, decompress
MANIFEST_SUFFIX = ".jsonl.zst"
async def collect_manifest_records(
dav: WebDAV, directory: str
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[str]]:
"""Download and parse every manifest. Returns (versions, renames, batches)."""
top = remote_join(directory, "manifests")
if not await dav.exists(top):
return [], [], []
files, _ = await list_tree(dav, top)
versions: list[dict[str, Any]] = []
renames: list[dict[str, Any]] = []
batches: list[str] = []
for entry in sorted(files, key=lambda e: e.path):
if not entry.path.endswith(MANIFEST_SUFFIX):
continue
data = await dav.get(entry.path)
if data is None:
continue
batch = entry.path.rsplit("/", 1)[-1][: -len(MANIFEST_SUFFIX)]
batches.append(batch)
for line in decompress(data).decode().splitlines():
if not line.strip():
continue
try:
rec = json.loads(line)
except ValueError:
continue
rec["manifest_batch"] = batch
(versions if rec.get("type") == "version" else renames).append(rec)
return versions, renames, batches
def rebuild_index(
db: Database, versions: list[dict[str, Any]], renames: list[dict[str, Any]]
) -> dict[str, int]:
"""Replace index contents from manifest records. Returns counts."""
with db.transaction():
db.execute("DELETE FROM versions")
db.execute("DELETE FROM renames")
db.execute("DELETE FROM files")
db.execute("DELETE FROM projects")
db.execute("DELETE FROM blobs")
now = time.time()
n_blobs = 0
for rec in versions:
sha = rec.get("sha256", "")
if not sha:
continue
cur = db.execute(
"INSERT OR IGNORE INTO blobs(sha256, size, stored_size, created_at, remote_state, uploaded_at)"
" VALUES (?, ?, ?, ?, 'uploaded', ?)",
(sha, rec.get("size", 0), rec.get("stored_size", 0) or 0,
rec.get("captured_at", now), now),
)
n_blobs += cur.rowcount
file_ids: dict[str, int] = {}
n_versions = 0
for rec in sorted(versions, key=lambda r: (r.get("captured_at", 0), r.get("id", 0))):
path, sha = rec.get("path", ""), rec.get("sha256", "")
if not path or not sha:
continue
file_id = file_ids.get(path)
if file_id is None:
db.execute(
"INSERT OR IGNORE INTO files(path, exists_on_disk, size, first_seen_at) "
"VALUES (?, 0, ?, ?)",
(path, rec.get("size", 0), rec.get("captured_at", now)),
)
row = db.one("SELECT id FROM files WHERE path = ?", (path,))
file_id = file_ids[path] = row["id"]
db.execute(
"INSERT INTO versions(file_id, blob_sha256, captured_at, source, reason, pinned,"
" durability, manifest_batch) VALUES (?, ?, ?, ?, ?, ?, 'durable', ?)",
(file_id, sha, rec.get("captured_at", now), rec.get("source", "?"),
rec.get("reason", "?"), 1 if rec.get("pinned") else 0, rec.get("manifest_batch")),
)
n_versions += 1
for file_id, path in ((v, k) for k, v in file_ids.items()):
row = db.one(
"SELECT count(*) AS n, min(captured_at) AS first, max(captured_at) AS last "
"FROM versions WHERE file_id = ?",
(file_id,),
)
db.execute(
"UPDATE files SET first_seen_at = ?, last_version_at = ? WHERE id = ?",
(row["first"], row["last"], file_id),
)
n_renames = 0
for rec in renames:
row = db.one("SELECT id FROM files WHERE path = ?", (rec.get("new_path", ""),))
if row is None:
continue
db.execute(
"INSERT INTO renames(file_id, old_path, new_path, at, manifest_batch) "
"VALUES (?, ?, ?, ?, ?)",
(row["id"], rec.get("old_path", ""), rec.get("new_path", ""),
rec.get("at", now), rec.get("manifest_batch")),
)
n_renames += 1
db.audit("reindex", files=len(file_ids), versions=n_versions, renames=n_renames, blobs=n_blobs)
return {"files": len(file_ids), "versions": n_versions, "renames": n_renames, "blobs": n_blobs}
def retention_plan(db: Database, keep_days: float) -> dict[str, Any]:
"""Versions safe to delete: older than keep_days, not pinned, not per-file latest.
Of those, one per file per UTC day is kept (thinning); the rest go.
"""
cutoff = time.time() - keep_days * 86_400
latest = {r["file_id"]: r["latest"] for r in db.all(
"SELECT file_id, max(id) AS latest FROM versions GROUP BY file_id")}
cands = db.all(
"SELECT id, file_id, captured_at FROM versions "
"WHERE captured_at < ? AND pinned = 0 ORDER BY file_id, captured_at",
(cutoff,),
)
cands = [c for c in cands if c["id"] != latest.get(c["file_id"])]
keep: set[int] = set()
by_day: dict[tuple[int, str], dict[str, Any]] = {}
for cand in cands:
day = datetime.fromtimestamp(cand["captured_at"], timezone.utc).strftime("%Y-%m-%d")
key = (cand["file_id"], day)
if key not in by_day or cand["id"] > by_day[key]["id"]:
by_day[key] = cand
keep = {c["id"] for c in by_day.values()}
delete = [c["id"] for c in cands if c["id"] not in keep]
return {"cutoff": cutoff, "keep_days": keep_days, "candidates": len(cands),
"kept_daily": len(keep), "delete_ids": delete}
async def delete_versions(db: Database, ids: list[int], chunk: int = 500) -> int:
deleted = 0
for i in range(0, len(ids), chunk):
part = ids[i: i + chunk]
with db.transaction():
db.execute(f"DELETE FROM versions WHERE id IN ({','.join('?' * len(part))})", part)
deleted += len(part)
await asyncio.sleep(0)
return deleted
def orphan_blobs(db: Database) -> list[dict[str, Any]]:
return db.all(
"SELECT sha256, stored_size FROM blobs b WHERE NOT EXISTS "
"(SELECT 1 FROM versions v WHERE v.blob_sha256 = b.sha256)")
async def collect_orphans(
dav: WebDAV | None, db: Database, blobs: BlobStore, directory: str, orphans: list[dict[str, Any]],
task: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Delete orphan blobs locally and remotely. Only unreferenced data is touched."""
removed = remote_removed = 0
errors: list[str] = []
for orphan in orphans:
sha = orphan["sha256"]
for cand in blobs._candidates(sha):
try:
cand.unlink()
except FileNotFoundError:
pass
except OSError as exc:
errors.append(f"{sha}: {exc}")
db.execute("DELETE FROM blobs WHERE sha256 = ?", (sha,))
removed += 1
if dav is not None:
try:
await dav.delete(remote_join(directory, "blobs", sha[:2], sha))
remote_removed += 1
except Exception as exc: # noqa: BLE001 - recorded per blob
errors.append(f"remote {sha}: {exc}")
if task is not None:
task["blobs_removed"] = removed
await asyncio.sleep(0)
return {"blobs_removed": removed, "remote_removed": remote_removed, "errors": errors[:50]}
src/versiond/remote.py
+135 -13
View File
@@ -10,13 +10,15 @@ import re
import secrets
import socket
import time
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from typing import Any
from urllib.parse import unquote, urlparse
import httpx
INSTALL_APP_ID = b"versiond:4c1b9e0f7a2d4e8b"
FORMAT_VERSION = 1
FORMAT_VERSION = 2
class RemoteError(Exception):
@@ -53,8 +55,12 @@ def installation_id() -> str:
machine_id = ""
if not machine_id:
return secrets.token_hex(8)
try:
key = bytes.fromhex(machine_id)
except ValueError:
return secrets.token_hex(8)
msg = INSTALL_APP_ID + b":" + str(os.getuid()).encode()
return hmac.new(bytes.fromhex(machine_id), msg, hashlib.sha256).hexdigest()[:16]
return hmac.new(key, msg, hashlib.sha256).hexdigest()[:16]
def hostname_slug() -> str:
@@ -155,11 +161,13 @@ async def check_connection(dav: WebDAV) -> None:
await dav.delete(probe)
async def claim_directory(dav: WebDAV, key_id: str) -> dict[str, Any]:
async def claim_directory(dav: WebDAV) -> dict[str, Any]:
"""Pick and claim this installation's unique directory below base_path.
Returns {"directory": ..., "adopted": bool}. Never takes over a directory that
belongs to a different installation id.
belongs to a different installation id. Remote data is stored unencrypted
(relying on server disk encryption); blobs are plain content-addressed
objects named by their SHA-256.
"""
base = join(dav.settings.base_path)
await dav.makedirs(base)
@@ -177,12 +185,11 @@ async def claim_directory(dav: WebDAV, key_id: str) -> dict[str, Any]:
"hostname": socket.gethostname(),
"user": os.environ.get("USER", ""),
"created_at": time.time(),
"key_id": key_id,
}
if not await dav.put(owner_path, json.dumps(owner, indent=2).encode(), if_none_match=True):
continue # someone created it between our GET and PUT; re-read
fmt = {"format": FORMAT_VERSION, "encryption": "aes-256-gcm", "compression": "zstd",
"blob_layout": "blobs/<hmac[:2]>/<hmac>", "key_id": key_id}
fmt = {"format": FORMAT_VERSION, "encryption": "none", "compression": "zstd-or-zlib",
"blob_layout": "blobs/<sha256[:2]>/<sha256>"}
await dav.put(join(directory, "meta", "format.json"), json.dumps(fmt, indent=2).encode())
return {"directory": directory, "adopted": False}
try:
@@ -190,12 +197,127 @@ async def claim_directory(dav: WebDAV, key_id: str) -> dict[str, Any]:
except ValueError:
owner = {}
if owner.get("installation_id") == install_id:
if owner.get("key_id") not in (None, key_id):
raise RemoteError(
"key-mismatch",
f"{directory} was written with a different backup key; restore that key "
"(~/.config/versiond/backup.key) or remove the directory",
)
return {"directory": directory, "adopted": True}
name = f"{hostname_slug()}-{install_id[:8]}-{secrets.token_hex(2)}"
raise RemoteError("claim-failed", "could not claim a unique remote directory")
# remote listing (read-only PROPFIND) for the purge endpoint
DAV_NS = "DAV:"
_PROPFIND_BODY = (
'<?xml version="1.0" encoding="utf-8"?>'
'<d:propfind xmlns:d="DAV:"><d:prop>'
"<d:resourcetype/><d:getcontentlength/>"
"</d:prop></d:propfind>"
).encode()
_QUOTA_BODY = (
'<?xml version="1.0" encoding="utf-8"?>'
'<d:propfind xmlns:d="DAV:"><d:prop>'
"<d:quota-used-bytes/><d:quota-available-bytes/>"
"</d:prop></d:propfind>"
).encode()
@dataclass
class RemoteEntry:
path: str
is_dir: bool
size: int | None = None
def _href_to_path(href: str) -> str:
path = unquote(urlparse(href).path or href)
return path.rstrip("/") or "/"
async def list_dir(dav: "WebDAV", path: str) -> list[RemoteEntry]:
"""Immediate children of a collection via PROPFIND Depth:1 (read-only)."""
resp = await dav.request(
"PROPFIND", path,
headers={"Depth": "1", "Content-Type": "application/xml"},
content=_PROPFIND_BODY,
)
if resp.status_code == 404:
return []
WebDAV._check(resp, 207)
try:
root = ET.fromstring(resp.content)
except ET.ParseError as exc:
raise RemoteError("remote-http", f"PROPFIND {path} returned unparseable XML") from exc
entries: list[RemoteEntry] = []
for response in root.findall(f"{{{DAV_NS}}}response"):
href = response.findtext(f"{{{DAV_NS}}}href")
if not href:
continue
child = _href_to_path(href)
if child == path.rstrip("/") or child == path:
continue # the collection itself, not a child
is_dir = False
size: int | None = None
for propstat in response.findall(f"{{{DAV_NS}}}propstat"):
prop = propstat.find(f"{{{DAV_NS}}}prop")
if prop is None:
continue
restype = prop.find(f"{{{DAV_NS}}}resourcetype")
if restype is not None and restype.find(f"{{{DAV_NS}}}collection") is not None:
is_dir = True
length = prop.findtext(f"{{{DAV_NS}}}getcontentlength")
if length is not None:
try:
size = int(length)
except ValueError:
size = None
entries.append(RemoteEntry(child, is_dir, size))
return entries
async def list_tree(dav: "WebDAV", top: str) -> tuple[list[RemoteEntry], list[RemoteEntry]]:
"""All files and directories strictly below top. Returns (files, dirs)."""
files: list[RemoteEntry] = []
dirs: list[RemoteEntry] = []
stack = [top]
while stack:
current = stack.pop()
for entry in await list_dir(dav, current):
if entry.is_dir:
dirs.append(entry)
stack.append(entry.path)
else:
files.append(entry)
return files, dirs
async def quota(dav: "WebDAV", path: str) -> tuple[int | None, int | None]:
"""RFC 4331 quota properties. Returns (used_bytes, available_bytes or None).
(None, None) when the server exposes no quota information.
"""
resp = await dav.request(
"PROPFIND", path,
headers={"Depth": "0", "Content-Type": "application/xml"},
content=_QUOTA_BODY,
)
if resp.status_code == 404:
return None, None
WebDAV._check(resp, 207)
try:
root = ET.fromstring(resp.content)
except ET.ParseError:
return None, None
used = available = None
for prop in root.findall(f".//{{{DAV_NS}}}prop"):
used_text = prop.findtext(f"{{{DAV_NS}}}quota-used-bytes")
avail_text = prop.findtext(f"{{{DAV_NS}}}quota-available-bytes")
if used_text is not None:
try:
used = int(used_text)
except ValueError:
used = None
if avail_text is not None:
try:
available = int(avail_text)
except ValueError:
available = None
return used, available
src/versiond/restore.py
+35 -5
View File
@@ -12,6 +12,7 @@ from pathlib import Path
from typing import Any, Literal
from .db import Database
from . import dbsafe
from .ingest import Repository, is_within
from .store import BlobStore, sha256
@@ -48,12 +49,24 @@ class Restorer:
self.db = db
self.repo = repo
self.blobs = blobs
# Async (sha256) -> bytes; set by the service layer for on-demand
# remote fetch after reindex. Defaults to local spool only.
self.fetch_blob = None
# path safety
def allowed_bases(self) -> list[str]:
return [str(Path.home())] + [r["path"] for r in self.db.roots()]
@staticmethod
def _protected_dirs() -> list[str]:
try:
from .config import Paths
p = Paths.resolve()
return [str(p.config_dir), str(p.data_dir), str(p.cache_dir)]
except Exception:
return []
def safe_target(self, target: str) -> str:
"""Resolve a write target; refuse traversal, symlink escapes and symlink targets."""
if not os.path.isabs(target):
@@ -61,6 +74,9 @@ class Restorer:
normalized = os.path.normpath(target)
if normalized != target.rstrip("/") or ".." in Path(target).parts:
raise RestoreError("unsafe-path", f"{target} is not a normalized path")
for protected in self._protected_dirs():
if is_within(normalized, protected):
raise RestoreError("unsafe-path", f"{target} is inside versiond's own data directory {protected}")
parent = Path(normalized).parent
existing = parent
while not existing.exists() and existing != existing.parent:
@@ -172,7 +188,7 @@ class Restorer:
# execution
def execute(self, plan_id: str) -> dict[str, Any]:
async def execute(self, plan_id: str) -> dict[str, Any]:
row = self.db.one("SELECT * FROM restore_plans WHERE id = ?", (plan_id,))
if row is None:
raise RestoreError("not-found", f"restore plan {plan_id} does not exist")
@@ -187,14 +203,19 @@ class Restorer:
results = []
for action in plan["actions"]:
try:
results.append(self._apply(action))
results.append(await self._apply(action))
except (OSError, RestoreError) as exc:
results.append({**action, "result": "error", "error": str(exc)})
self.db.execute("UPDATE restore_plans SET status = 'executed' WHERE id = ?", (plan_id,))
self.db.audit("restore.execute", plan_id=plan_id, count=len(results))
return {"plan_id": plan_id, "results": results}
def _apply(self, action: dict[str, Any]) -> dict[str, Any]:
async def _blob(self, sha256: str) -> bytes:
if self.fetch_blob is not None:
return await self.fetch_blob(sha256)
return self.blobs.get(sha256)
async def _apply(self, action: dict[str, Any]) -> dict[str, Any]:
kind = action["action"]
if kind in ("unchanged", "skip-conflict", "skip-not-existing-at-as-of"):
return {**action, "result": "skipped"}
@@ -205,7 +226,16 @@ class Restorer:
self.repo.mark_deleted(destination)
return {**action, "result": "removed"}
version = self.db.version(action["version_id"])
content = self.blobs.get(version["blob_sha256"])
try:
content = await self._blob(version["blob_sha256"])
except (FileNotFoundError, ValueError) as exc:
raise RestoreError("unrestorable", f"blob {version['blob_sha256']} is gone or corrupt") from exc
if dbsafe.is_sqlite_image(content):
# Never write an unverified database back to disk.
try:
dbsafe.verify_sqlite_bytes(content)
except dbsafe.DatabaseUnsafe as exc:
raise RestoreError("unrestorable", f"stored version fails integrity check: {exc.reason}")
if kind == "write-renamed":
stem, ext = os.path.splitext(destination)
destination = f"{stem}.restored-{time.strftime('%Y%m%d-%H%M%S')}{ext}"
@@ -217,7 +247,7 @@ class Restorer:
content, st = self.repo.read_file(path)
except (OSError, Exception):
return None
if self.repo.filters.size_reason(len(content)) or self.repo.filters.content_reason(content):
if self.repo.filters.size_reason(len(content)):
return None
return self.repo.commit(path, content, "restore", "pre-restore", st)["version_id"]
src/versiond/store.py
+53 -7
View File
@@ -7,12 +7,37 @@ import os
from pathlib import Path
try:
from compression import zstd as _codec # Python 3.14+
from compression import zstd as _preferred # Python 3.14+
_EXT = ".zst"
except ImportError: # pragma: no cover - older Pythons
import zlib as _codec
_preferred = None
_EXT = ".z"
import zlib as _fallback
# Back-compat: external code (and old tests) import _codec.
_codec = _preferred if _preferred is not None else _fallback
_EXTS = (".zst", ".z")
def compress(content: bytes) -> bytes:
return _codec.compress(content)
def decompress(data: bytes) -> bytes:
"""Decompress spool/manifest bytes; raises ValueError uniformly on garbage."""
codecs = [ _codec ]
if _preferred is not None and _fallback is not _codec:
codecs.append(_fallback)
last: Exception | None = None
for codec in codecs:
try:
return codec.decompress(data)
except Exception as exc: # noqa: BLE001 - normalized below
last = exc
raise ValueError(f"cannot decompress blob: {last}")
def sha256(content: bytes) -> str:
return hashlib.sha256(content).hexdigest()
@@ -26,17 +51,28 @@ class BlobStore:
def path_for(self, digest: str) -> Path:
return self.root / digest[:2] / digest[2:4] / f"{digest}{_EXT}"
def _candidates(self, digest: str) -> list[Path]:
base = self.root / digest[:2] / digest[2:4]
return [base / f"{digest}{ext}" for ext in _EXTS]
def _existing(self, digest: str) -> Path | None:
for cand in self._candidates(digest):
if cand.exists():
return cand
return None
def has(self, digest: str) -> bool:
return self.path_for(digest).exists()
return self._existing(digest) is not None
def put(self, content: bytes, digest: str | None = None) -> tuple[str, int]:
"""Store content; returns (digest, stored_size). Idempotent."""
digest = digest or sha256(content)
found = self._existing(digest)
if found is not None:
return digest, found.stat().st_size
target = self.path_for(digest)
if target.exists():
return digest, target.stat().st_size
target.parent.mkdir(parents=True, exist_ok=True)
data = _codec.compress(content)
data = compress(content)
tmp = target.with_name(f".{target.name}.{os.getpid()}.tmp")
with open(tmp, "wb") as fh:
fh.write(data)
@@ -46,4 +82,14 @@ class BlobStore:
return digest, len(data)
def get(self, digest: str) -> bytes:
return _codec.decompress(self.path_for(digest).read_bytes())
found = self._existing(digest)
if found is None:
raise FileNotFoundError(self.path_for(digest))
return decompress(found.read_bytes())
def read_stored(self, digest: str) -> bytes:
"""Raw (compressed) bytes as stored, regardless of codec extension."""
found = self._existing(digest)
if found is None:
raise FileNotFoundError(self.path_for(digest))
return found.read_bytes()
src/versiond/uploader.py
+22 -17
View File
@@ -14,10 +14,9 @@ from typing import Any
import httpx
from .config import Config
from .crypto import KeyRing
from .db import Database
from .remote import RemoteError, RemoteSettings, WebDAV, check_connection, claim_directory, join
from .store import BlobStore, _codec
from .store import BlobStore, compress as _compress
log = logging.getLogger(__name__)
@@ -48,12 +47,11 @@ class Throttle:
class Uploader:
def __init__(self, cfg: Config, db: Database, blobs: BlobStore, keys: KeyRing,
def __init__(self, cfg: Config, db: Database, blobs: BlobStore,
transport: httpx.AsyncBaseTransport | None = None):
self.cfg = cfg
self.db = db
self.blobs = blobs
self.keys = keys
self.transport = transport
self.dav: WebDAV | None = None
self.task: asyncio.Task | None = None
@@ -71,6 +69,8 @@ class Uploader:
self._last_manifest = 0.0
self._wake = asyncio.Event()
self.offline_sleep = OFFLINE_SLEEP
self.remote_usage: dict[str, Any] = {"percent": None, "used": None, "total": None,
"source": "unknown", "at": None}
# settings
@@ -91,7 +91,7 @@ class Uploader:
"configured": s.configured, "url": s.url, "username": s.username,
"password_set": bool(s.password), "base_path": s.base_path, "directory": s.directory,
"verify_tls": s.verify_tls, "timeout_seconds": s.timeout_seconds,
"encryption": "aes-256-gcm", "key_id": self.keys.key_id, "state": self.state,
"encryption": "none", "state": self.state,
}
async def test(self, settings: RemoteSettings) -> None:
@@ -106,7 +106,7 @@ class Uploader:
dav = WebDAV(settings, transport=self.transport)
try:
await check_connection(dav)
claim = await claim_directory(dav, self.keys.key_id)
claim = await claim_directory(dav)
finally:
await dav.close()
await self.stop()
@@ -156,7 +156,7 @@ class Uploader:
async def _ensure_directory(self) -> None:
assert self.dav is not None
if not self.directory:
claim = await claim_directory(self.dav, self.keys.key_id)
claim = await claim_directory(self.dav)
self.directory = claim["directory"]
self.cfg.update("remote", {"directory": self.directory})
await self.dav.makedirs(join(self.directory, "blobs"))
@@ -182,7 +182,7 @@ class Uploader:
raise
except RemoteError as exc:
self._failed(exc)
self.state = "error" if exc.code in ("remote-auth", "key-mismatch", "remote-full") else "offline"
self.state = "error" if exc.code in ("remote-auth", "remote-full") else "offline"
log.warning("remote %s: %s; retrying in %ss", self.state, exc.detail, self.offline_sleep)
await asyncio.sleep(self.offline_sleep)
except Exception as exc:
@@ -208,8 +208,11 @@ class Uploader:
fatal: list[RemoteError] = []
async def worker() -> None:
while not queue.empty() and not fatal:
row = queue.get_nowait()
while not fatal:
try:
row = queue.get_nowait()
except asyncio.QueueEmpty:
return
await requests.take()
try:
await self._upload_blob(row, bandwidth)
@@ -226,14 +229,14 @@ class Uploader:
assert self.dav is not None
digest = row["sha256"]
try:
stored = self.blobs.path_for(digest).read_bytes()
stored = self.blobs.read_stored(digest)
except FileNotFoundError:
self.db.execute("UPDATE blobs SET remote_state = 'missing-local', last_error = ? WHERE sha256 = ?",
("local blob file is missing", digest))
return
name = self.keys.blob_name(digest)
name = digest
prefix = join(self.directory, "blobs", name[:2])
payload = self.keys.encrypt(stored)
payload = stored
await bandwidth.take(len(payload))
try:
await self.dav.mkcol(prefix)
@@ -266,7 +269,8 @@ class Uploader:
assert self.dav is not None
self._last_manifest = time.monotonic()
versions = self.db.all(
"SELECT v.id, f.path, v.blob_sha256 AS sha256, b.size, v.captured_at, v.source, v.reason, v.pinned "
"SELECT v.id, f.path, v.blob_sha256 AS sha256, b.size, b.stored_size, v.captured_at, "
"v.source, v.reason, v.pinned "
"FROM versions v JOIN files f ON f.id = v.file_id JOIN blobs b ON b.sha256 = v.blob_sha256 "
"WHERE v.durability = 'local' AND b.remote_state = 'uploaded' ORDER BY v.id LIMIT ?",
(MANIFEST_MAX_RECORDS,),
@@ -280,12 +284,12 @@ class Uploader:
return False
stamp = time.gmtime()
batch_id = time.strftime("%Y%m%dT%H%M%SZ", stamp) + "-" + secrets.token_hex(4)
lines = [json.dumps({"type": "version", **v, "blob": self.keys.blob_name(v["sha256"])}) for v in versions]
lines = [json.dumps({"type": "version", **v, "blob": v["sha256"]}) for v in versions]
lines += [json.dumps({"type": "rename", **r}) for r in renames]
payload = self.keys.encrypt(_codec.compress(("\n".join(lines) + "\n").encode()))
payload = _compress(("\n".join(lines) + "\n").encode())
folder = join(self.directory, "manifests", time.strftime("%Y/%m/%d", stamp))
await self.dav.makedirs(folder)
await self.dav.put(join(folder, f"{batch_id}.jsonl.zst.enc"), payload)
await self.dav.put(join(folder, f"{batch_id}.jsonl.zst"), payload)
with self.db.transaction():
for chunk in _chunks([v["id"] for v in versions], 500):
self.db.execute(
@@ -340,6 +344,7 @@ class Uploader:
"last_error": self.last_error,
"last_error_at": self.last_error_at,
"consecutive_failures": self.consecutive_failures,
"usage": dict(self.remote_usage),
"session": dict(self.counters),
}