Implement versiond M1: inotify monitor, versioning, diff, restore, systemd install
FastAPI service on 127.0.0.1:9922 that monitors user-chosen directories with pruned inotify watches, stores versions in a local content-addressed spool indexed in SQLite, coalesces bursts (first + last), and offers history, diff and dry-run-first bulk restore. Includes a CLI with a systemd user unit installer. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
retoor
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
3455499cea
commit
ddf09bea88
@@ -0,0 +1,3 @@
|
||||
"""versiond - local file versioning and backup service."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,5 @@
|
||||
import sys
|
||||
|
||||
from .cli import main
|
||||
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,515 @@
|
||||
"""HTTP API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import fnmatch
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import socket
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Query, Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, Response
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from . import __version__
|
||||
from . import diff as difflib_
|
||||
from .config import Config
|
||||
from .db import Database
|
||||
from .filters import Filters
|
||||
from .ingest import Coalescer, Rejected, Repository, TokenBucket
|
||||
from .monitor import Monitor, RootError
|
||||
from .restore import Criteria, Restorer, RestoreError
|
||||
from .store import BlobStore
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# errors
|
||||
|
||||
class Problem(Exception):
|
||||
def __init__(self, status: int, code: str, detail: str):
|
||||
self.status, self.code, self.detail = status, code, detail
|
||||
|
||||
|
||||
REJECT_STATUS = {"file-too-large": 413}
|
||||
ROOT_STATUS = {"not-found": 404, "not-a-directory": 400, "root-overlap": 409, "watch-limit": 409}
|
||||
RESTORE_STATUS = {"not-found": 404, "unsafe-path": 400, "criteria-too-broad": 400,
|
||||
"plan-expired": 410, "plan-used": 409, "conflict": 409}
|
||||
|
||||
|
||||
def _problem(status: int, code: str, detail: str) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
{"type": f"urn:versiond:{code}", "title": code.replace("-", " "), "status": status, "detail": detail},
|
||||
status_code=status,
|
||||
media_type="application/problem+json",
|
||||
)
|
||||
|
||||
|
||||
def parse_time(value: str | float | None) -> float | None:
|
||||
"""Accept epoch seconds or ISO 8601 (local time if no offset)."""
|
||||
if value is None or value == "":
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return datetime.fromisoformat(value).timestamp()
|
||||
except ValueError:
|
||||
raise Problem(400, "invalid-time", f"cannot parse time {value!r}; use ISO 8601 or epoch seconds")
|
||||
|
||||
|
||||
def sd_notify(message: str) -> None:
|
||||
address = os.environ.get("NOTIFY_SOCKET")
|
||||
if not address:
|
||||
return
|
||||
if address.startswith("@"):
|
||||
address = "\0" + address[1:]
|
||||
try:
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
|
||||
sock.connect(address)
|
||||
sock.sendall(message.encode())
|
||||
except OSError:
|
||||
log.debug("sd_notify failed", exc_info=True)
|
||||
|
||||
|
||||
# request models
|
||||
|
||||
class SnapshotIn(BaseModel):
|
||||
path: str
|
||||
content_b64: str | None = None
|
||||
read_from_disk: bool = False
|
||||
source: str = Field("api", max_length=64)
|
||||
reason: str = Field("push", max_length=200)
|
||||
|
||||
|
||||
class SnapshotBatchIn(BaseModel):
|
||||
snapshots: list[SnapshotIn] = Field(max_length=100)
|
||||
|
||||
|
||||
class RootIn(BaseModel):
|
||||
path: str
|
||||
force: bool = False
|
||||
|
||||
|
||||
class FilterTestIn(BaseModel):
|
||||
paths: list[str] = Field(max_length=1000)
|
||||
|
||||
|
||||
class VersionRestoreIn(BaseModel):
|
||||
target_path: str | None = None
|
||||
conflict: Literal["overwrite", "skip", "rename"] = "overwrite"
|
||||
|
||||
|
||||
class RestoreIn(BaseModel):
|
||||
as_of: str | float | None = None
|
||||
paths: list[str] = []
|
||||
globs: list[str] = []
|
||||
exclude_globs: list[str] = []
|
||||
extensions: list[str] = []
|
||||
root_id: int | None = None
|
||||
project_id: int | None = None
|
||||
changed_since: str | float | None = None
|
||||
changed_by_source: str | None = None
|
||||
target_dir: str | None = None
|
||||
conflict: Literal["overwrite", "skip", "rename", "fail"] = "overwrite"
|
||||
remove_newer_files: bool = False
|
||||
|
||||
|
||||
# services
|
||||
|
||||
class Services:
|
||||
def __init__(self, cfg: Config):
|
||||
self.cfg = cfg
|
||||
self.filters = Filters.from_config(cfg)
|
||||
self.db = Database(cfg.paths.index_file)
|
||||
self.blobs = BlobStore(cfg.paths.spool_dir)
|
||||
self.repo = Repository(self.db, self.blobs, self.filters)
|
||||
self.coalescer = Coalescer(
|
||||
self.repo,
|
||||
window=float(cfg.get("coalesce", "window_seconds")),
|
||||
max_hold=float(cfg.get("coalesce", "max_hold_seconds")),
|
||||
per_hour=int(cfg.get("coalesce", "versions_per_hour_per_path")),
|
||||
)
|
||||
self.monitor = Monitor(
|
||||
self.db, self.repo, self.coalescer, self.filters,
|
||||
poll_interval=float(cfg.get("monitor", "poll_interval_seconds")),
|
||||
reconcile_interval=float(cfg.get("monitor", "reconcile_interval_seconds")),
|
||||
max_watch_fraction=float(cfg.get("monitor", "max_watch_fraction")),
|
||||
)
|
||||
self.restorer = Restorer(self.db, self.repo, self.blobs)
|
||||
rate = float(cfg.get("limits", "ingest_requests_per_second"))
|
||||
self.ingest_limit = TokenBucket(rate, 1.0, time.monotonic)
|
||||
self.token = cfg.api_token()
|
||||
self.started_at = time.time()
|
||||
|
||||
|
||||
def create_app(cfg: Config | None = None, allowed_hosts: set[str] | None = None) -> FastAPI:
|
||||
cfg = cfg or Config.load()
|
||||
hosts = allowed_hosts or {f"127.0.0.1:{cfg.port}", f"localhost:{cfg.port}"}
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
services = Services(cfg)
|
||||
app.state.services = services
|
||||
await services.monitor.start()
|
||||
sd_notify("READY=1")
|
||||
log.info("versiond %s ready on %s:%s", __version__, cfg.host, cfg.port)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
sd_notify("STOPPING=1")
|
||||
await services.monitor.stop()
|
||||
flushed = services.coalescer.flush()
|
||||
log.info("shutdown: flushed %d pending versions", flushed)
|
||||
services.db.close()
|
||||
|
||||
app = FastAPI(
|
||||
title="versiond",
|
||||
version=__version__,
|
||||
description="Local file versioning service: monitors directories, keeps every "
|
||||
"version of small project files, and offers history, diff and restore.",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
@app.middleware("http")
|
||||
async def check_host(request: Request, call_next):
|
||||
if request.headers.get("host") not in hosts:
|
||||
return _problem(421, "invalid-host", "requests must be addressed to the loopback host")
|
||||
return await call_next(request)
|
||||
|
||||
@app.exception_handler(Problem)
|
||||
async def on_problem(_: Request, exc: Problem):
|
||||
return _problem(exc.status, exc.code, exc.detail)
|
||||
|
||||
@app.exception_handler(Rejected)
|
||||
async def on_rejected(_: Request, exc: Rejected):
|
||||
return _problem(REJECT_STATUS.get(exc.reason, 422), exc.reason, exc.detail or exc.reason)
|
||||
|
||||
@app.exception_handler(RootError)
|
||||
async def on_root_error(_: Request, exc: RootError):
|
||||
return _problem(ROOT_STATUS.get(exc.code, 400), exc.code, exc.detail)
|
||||
|
||||
@app.exception_handler(RestoreError)
|
||||
async def on_restore_error(_: Request, exc: RestoreError):
|
||||
return _problem(RESTORE_STATUS.get(exc.code, 400), exc.code, exc.detail)
|
||||
|
||||
async def services(request: Request) -> Services:
|
||||
return request.app.state.services
|
||||
|
||||
async def authenticate(request: Request) -> None:
|
||||
token = request.app.state.services.token
|
||||
header = request.headers.get("authorization", "")
|
||||
scheme, _, given = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not secrets.compare_digest(given.strip(), token):
|
||||
raise Problem(401, "unauthorized", "missing or invalid bearer token "
|
||||
"(see ~/.config/versiond/credentials or run `versiond token`)")
|
||||
|
||||
@app.get("/health", tags=["status"])
|
||||
async def health(s: Services = Depends(services)) -> dict[str, Any]:
|
||||
roots = s.db.roots()
|
||||
degraded = [r["path"] for r in roots if r["mode"] in ("degraded", "missing")]
|
||||
return {
|
||||
"status": "degraded" if degraded else "ok",
|
||||
"version": __version__,
|
||||
"uptime_seconds": round(time.time() - s.started_at),
|
||||
"roots": len(roots),
|
||||
"degraded_roots": degraded,
|
||||
"pending_paths": sum(1 for p in s.coalescer.pending.values() if p.content is not None),
|
||||
"watches": len(s.monitor.watches),
|
||||
}
|
||||
|
||||
api = APIRouter(prefix="/api/v1", dependencies=[Depends(authenticate)])
|
||||
|
||||
# config & filters
|
||||
|
||||
@api.get("/config", tags=["config"])
|
||||
async def get_config(s: Services = Depends(services)) -> dict[str, Any]:
|
||||
return {"config_file": str(s.cfg.paths.config_file), **s.cfg.data}
|
||||
|
||||
@api.post("/filters/test", tags=["config"])
|
||||
async def test_filters(body: FilterTestIn, s: Services = Depends(services)) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
for raw in body.paths:
|
||||
path = os.path.abspath(os.path.expanduser(raw))
|
||||
reason = s.filters.path_reason(s.repo.relative(path))
|
||||
if reason is None and os.path.isfile(path):
|
||||
reason = s.filters.size_reason(os.path.getsize(path))
|
||||
out.append({"path": path, "accepted": reason is None, "reason": reason})
|
||||
return out
|
||||
|
||||
# snapshots
|
||||
|
||||
def _snapshot(s: Services, body: SnapshotIn) -> dict[str, Any]:
|
||||
path = os.path.abspath(os.path.expanduser(body.path))
|
||||
s.repo.check_path(path)
|
||||
st = None
|
||||
if body.read_from_disk:
|
||||
try:
|
||||
content, st = s.repo.read_file(path)
|
||||
except FileNotFoundError:
|
||||
raise Problem(404, "file-not-found", path)
|
||||
elif body.content_b64 is not None:
|
||||
try:
|
||||
content = base64.b64decode(body.content_b64, validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
raise Problem(400, "invalid-base64", "content_b64 is not valid base64")
|
||||
else:
|
||||
raise Problem(400, "missing-content", "give content_b64 or read_from_disk=true")
|
||||
s.repo.check_content(content)
|
||||
return s.coalescer.submit(path, content, body.source, body.reason, st)
|
||||
|
||||
@api.post("/snapshots", tags=["snapshots"])
|
||||
async def snapshot(body: SnapshotIn, s: Services = Depends(services)):
|
||||
if not s.ingest_limit.take():
|
||||
return JSONResponse(
|
||||
{"type": "urn:versiond:rate-limited", "title": "rate limited", "status": 429,
|
||||
"detail": "too many snapshot requests"},
|
||||
status_code=429, media_type="application/problem+json",
|
||||
headers={"Retry-After": "1"},
|
||||
)
|
||||
return _snapshot(s, body)
|
||||
|
||||
@api.post("/snapshots/batch", tags=["snapshots"])
|
||||
async def snapshot_batch(body: SnapshotBatchIn, s: Services = Depends(services)) -> list[dict[str, Any]]:
|
||||
if not s.ingest_limit.take():
|
||||
raise Problem(429, "rate-limited", "too many snapshot requests")
|
||||
results = []
|
||||
for item in body.snapshots:
|
||||
try:
|
||||
results.append(_snapshot(s, item))
|
||||
except Rejected as exc:
|
||||
results.append({"path": item.path, "status": "rejected", "reason": exc.reason})
|
||||
except Problem as exc:
|
||||
results.append({"path": item.path, "status": "error", "reason": exc.code})
|
||||
return results
|
||||
|
||||
# roots
|
||||
|
||||
@api.get("/roots", tags=["roots"])
|
||||
async def list_roots(s: Services = Depends(services)) -> list[dict[str, Any]]:
|
||||
return s.db.roots()
|
||||
|
||||
@api.post("/roots", tags=["roots"], status_code=201)
|
||||
async def add_root(body: RootIn, s: Services = Depends(services)) -> dict[str, Any]:
|
||||
return await s.monitor.add_root(body.path, force=body.force)
|
||||
|
||||
@api.get("/roots/{root_id}", tags=["roots"])
|
||||
async def get_root(root_id: int, s: Services = Depends(services)) -> dict[str, Any]:
|
||||
root = s.db.root(root_id)
|
||||
if root is None:
|
||||
raise Problem(404, "not-found", f"root {root_id} does not exist")
|
||||
stats = s.db.one(
|
||||
"SELECT count(*) AS files, coalesce(sum(exists_on_disk), 0) AS existing FROM files WHERE root_id = ?",
|
||||
(root_id,),
|
||||
)
|
||||
versions = s.db.one(
|
||||
"SELECT count(*) AS n FROM versions v JOIN files f ON f.id = v.file_id WHERE f.root_id = ?",
|
||||
(root_id,),
|
||||
)
|
||||
return {**root, **stats, "versions": versions["n"],
|
||||
"polled_paths": sorted(s.monitor.polled.get(root_id, ()))}
|
||||
|
||||
@api.delete("/roots/{root_id}", tags=["roots"], status_code=204)
|
||||
async def remove_root(root_id: int, s: Services = Depends(services)) -> Response:
|
||||
await s.monitor.remove_root(root_id)
|
||||
return Response(status_code=204)
|
||||
|
||||
@api.post("/roots/{root_id}/rescan", tags=["roots"])
|
||||
async def rescan_root(root_id: int, s: Services = Depends(services)) -> dict[str, Any]:
|
||||
if s.db.root(root_id) is None:
|
||||
raise Problem(404, "not-found", f"root {root_id} does not exist")
|
||||
committed = await s.monitor.reconcile(root_id, "rescan")
|
||||
return {"root_id": root_id, "versions_stored": committed}
|
||||
|
||||
# projects & files
|
||||
|
||||
@api.get("/projects", tags=["browse"])
|
||||
async def list_projects(root_id: int | None = None, s: Services = Depends(services)) -> list[dict[str, Any]]:
|
||||
sql = ("SELECT p.*, count(f.id) AS files FROM projects p LEFT JOIN files f ON f.project_id = p.id")
|
||||
params: list[Any] = []
|
||||
if root_id is not None:
|
||||
sql += " WHERE p.root_id = ?"
|
||||
params.append(root_id)
|
||||
return s.db.all(sql + " GROUP BY p.id ORDER BY p.last_activity_at DESC", params)
|
||||
|
||||
@api.get("/projects/{project_id}/tree", tags=["browse"])
|
||||
async def project_tree(
|
||||
project_id: int, as_of: str | None = None, s: Services = Depends(services)
|
||||
) -> dict[str, Any]:
|
||||
project = s.db.one("SELECT * FROM projects WHERE id = ?", (project_id,))
|
||||
if project is None:
|
||||
raise Problem(404, "not-found", f"project {project_id} does not exist")
|
||||
when = parse_time(as_of) or time.time()
|
||||
rows = s.db.all(
|
||||
"SELECT f.path, v.id AS version_id, v.captured_at, v.blob_sha256, b.size "
|
||||
"FROM files f JOIN versions v ON v.id = ("
|
||||
" SELECT id FROM versions WHERE file_id = f.id AND captured_at <= ? "
|
||||
" ORDER BY captured_at DESC, id DESC LIMIT 1) "
|
||||
"JOIN blobs b ON b.sha256 = v.blob_sha256 "
|
||||
"WHERE f.project_id = ? ORDER BY f.path",
|
||||
(when, project_id),
|
||||
)
|
||||
return {"project": project, "as_of": when, "files": rows}
|
||||
|
||||
@api.get("/files", tags=["browse"])
|
||||
async def list_files(
|
||||
root_id: int | None = None,
|
||||
project_id: int | None = None,
|
||||
glob: str | None = None,
|
||||
ext: str | None = None,
|
||||
exists: bool | None = None,
|
||||
modified_since: str | None = None,
|
||||
limit: int = Query(200, le=5000),
|
||||
offset: int = 0,
|
||||
s: Services = Depends(services),
|
||||
) -> list[dict[str, Any]]:
|
||||
sql = ("SELECT f.*, (SELECT count(*) FROM versions v WHERE v.file_id = f.id) AS versions "
|
||||
"FROM files f WHERE 1=1")
|
||||
params: list[Any] = []
|
||||
for column, value in (("root_id", root_id), ("project_id", project_id)):
|
||||
if value is not None:
|
||||
sql += f" AND f.{column} = ?"
|
||||
params.append(value)
|
||||
if exists is not None:
|
||||
sql += " AND f.exists_on_disk = ?"
|
||||
params.append(int(exists))
|
||||
since = parse_time(modified_since)
|
||||
if since is not None:
|
||||
sql += " AND f.last_version_at >= ?"
|
||||
params.append(since)
|
||||
rows = s.db.all(sql + " ORDER BY f.last_version_at DESC", params)
|
||||
if glob:
|
||||
rows = [r for r in rows if fnmatch.fnmatch(r["path"], glob)]
|
||||
if ext:
|
||||
suffix = ext if ext.startswith(".") else "." + ext
|
||||
rows = [r for r in rows if r["path"].endswith(suffix)]
|
||||
return rows[offset: offset + limit]
|
||||
|
||||
def _file_by_path(s: Services, path: str) -> dict[str, Any]:
|
||||
file = s.db.file_by_path(os.path.abspath(os.path.expanduser(path)))
|
||||
if file is None:
|
||||
raise Problem(404, "not-found", f"no history for {path}")
|
||||
return file
|
||||
|
||||
@api.get("/files/history", tags=["browse"])
|
||||
async def file_history(path: str, s: Services = Depends(services)) -> dict[str, Any]:
|
||||
file = _file_by_path(s, path)
|
||||
renames = s.db.all("SELECT old_path, new_path, at FROM renames WHERE file_id = ? ORDER BY at", (file["id"],))
|
||||
return {"file": file, "renames": renames, "versions": s.db.history(file["id"])}
|
||||
|
||||
# versions
|
||||
|
||||
def _version(s: Services, version_id: int) -> dict[str, Any]:
|
||||
version = s.db.version(version_id)
|
||||
if version is None:
|
||||
raise Problem(404, "not-found", f"version {version_id} does not exist")
|
||||
return version
|
||||
|
||||
@api.get("/versions/{version_id}", tags=["versions"])
|
||||
async def get_version(version_id: int, s: Services = Depends(services)) -> dict[str, Any]:
|
||||
return _version(s, version_id)
|
||||
|
||||
@api.get("/versions/{version_id}/content", tags=["versions"])
|
||||
async def get_content(version_id: int, s: Services = Depends(services)) -> Response:
|
||||
version = _version(s, version_id)
|
||||
content = s.blobs.get(version["blob_sha256"])
|
||||
try:
|
||||
return PlainTextResponse(content.decode("utf-8"))
|
||||
except UnicodeDecodeError:
|
||||
return Response(content, media_type="application/octet-stream")
|
||||
|
||||
@api.post("/versions/{version_id}/pin", tags=["versions"])
|
||||
async def pin(version_id: int, s: Services = Depends(services)) -> dict[str, Any]:
|
||||
_version(s, version_id)
|
||||
s.db.execute("UPDATE versions SET pinned = 1 WHERE id = ?", (version_id,))
|
||||
return _version(s, version_id)
|
||||
|
||||
@api.delete("/versions/{version_id}/pin", tags=["versions"])
|
||||
async def unpin(version_id: int, s: Services = Depends(services)) -> dict[str, Any]:
|
||||
_version(s, version_id)
|
||||
s.db.execute("UPDATE versions SET pinned = 0 WHERE id = ?", (version_id,))
|
||||
return _version(s, version_id)
|
||||
|
||||
@api.post("/versions/{version_id}/restore", tags=["restore"])
|
||||
async def restore_version(
|
||||
version_id: int, body: VersionRestoreIn, s: Services = Depends(services)
|
||||
) -> dict[str, Any]:
|
||||
version = _version(s, version_id)
|
||||
destination = os.path.abspath(os.path.expanduser(body.target_path or version["path"]))
|
||||
if os.path.exists(destination):
|
||||
if body.conflict == "skip":
|
||||
return {"result": "skipped", "destination": destination}
|
||||
if body.conflict == "rename":
|
||||
stem, ext = os.path.splitext(destination)
|
||||
destination = f"{stem}.restored-{time.strftime('%Y%m%d-%H%M%S')}{ext}"
|
||||
content = s.blobs.get(version["blob_sha256"])
|
||||
result = s.restorer.write(destination, content, version_id)
|
||||
s.db.audit("restore.version", version_id=version_id, destination=destination)
|
||||
return {"result": "restored", "destination": destination, **result}
|
||||
|
||||
# diff
|
||||
|
||||
@api.get("/diff", tags=["versions"])
|
||||
async def get_diff(
|
||||
from_version: int = Query(..., alias="from"),
|
||||
to: str = Query("disk", description="a version id, or 'disk' for the working copy"),
|
||||
format: Literal["unified", "json", "html"] = "unified",
|
||||
context: int = Query(3, ge=0, le=100),
|
||||
ignore_whitespace: bool = False,
|
||||
ignore_eol: bool = False,
|
||||
s: Services = Depends(services),
|
||||
):
|
||||
a_version = _version(s, from_version)
|
||||
a = s.blobs.get(a_version["blob_sha256"])
|
||||
from_label = f"{a_version['path']}@{from_version}"
|
||||
if to == "disk":
|
||||
try:
|
||||
with open(a_version["path"], "rb") as fh:
|
||||
b = fh.read(s.filters.max_file_bytes + 1)
|
||||
except FileNotFoundError:
|
||||
b = b""
|
||||
to_label = f"{a_version['path']} (disk)"
|
||||
else:
|
||||
if not to.isdigit():
|
||||
raise Problem(400, "invalid-target", "to must be a version id or 'disk'")
|
||||
b_version = _version(s, int(to))
|
||||
b = s.blobs.get(b_version["blob_sha256"])
|
||||
to_label = f"{b_version['path']}@{to}"
|
||||
if format == "html":
|
||||
return HTMLResponse(difflib_.side_by_side_html(a, b, from_label, to_label, context))
|
||||
hunk_list = difflib_.hunks(a, b, context, ignore_whitespace, ignore_eol)
|
||||
if format == "json":
|
||||
return {"from": from_label, "to": to_label, "identical": not hunk_list, "hunks": hunk_list}
|
||||
return PlainTextResponse(difflib_.unified(hunk_list, from_label, to_label))
|
||||
|
||||
# bulk restore
|
||||
|
||||
@api.post("/restores", tags=["restore"], status_code=201)
|
||||
async def create_restore(body: RestoreIn, s: Services = Depends(services)) -> dict[str, Any]:
|
||||
criteria = Criteria(**{**body.model_dump(),
|
||||
"as_of": parse_time(body.as_of),
|
||||
"changed_since": parse_time(body.changed_since)})
|
||||
if criteria.target_dir:
|
||||
criteria.target_dir = os.path.abspath(os.path.expanduser(criteria.target_dir))
|
||||
return s.restorer.plan(criteria)
|
||||
|
||||
@api.get("/restores/{plan_id}", tags=["restore"])
|
||||
async def get_restore(plan_id: str, s: Services = Depends(services)) -> dict[str, Any]:
|
||||
return s.restorer.get_plan(plan_id)
|
||||
|
||||
@api.post("/restores/{plan_id}/execute", tags=["restore"])
|
||||
async def execute_restore(plan_id: str, s: Services = Depends(services)) -> dict[str, Any]:
|
||||
return s.restorer.execute(plan_id)
|
||||
|
||||
app.include_router(api)
|
||||
return app
|
||||
@@ -0,0 +1,329 @@
|
||||
"""Command line interface: server, installer and a thin API client."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import __version__
|
||||
from .config import Config
|
||||
|
||||
UNIT_NAME = "versiond.service"
|
||||
|
||||
UNIT_TEMPLATE = """[Unit]
|
||||
Description=versiond - local file versioning and backup service
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
ExecStart={exec_start} serve
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
NoNewPrivileges=true
|
||||
MemoryMax=512M
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
"""
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
entry = {"level": record.levelname.lower(), "logger": record.name, "msg": record.getMessage()}
|
||||
if record.exc_info:
|
||||
entry["exc"] = self.formatException(record.exc_info)
|
||||
return json.dumps(entry)
|
||||
|
||||
|
||||
# server
|
||||
|
||||
def cmd_serve(args: argparse.Namespace) -> int:
|
||||
import uvicorn
|
||||
|
||||
from .api import create_app
|
||||
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO, handlers=[handler])
|
||||
cfg = Config.load()
|
||||
if cfg.host not in ("127.0.0.1", "::1", "localhost"):
|
||||
logging.error("refusing to bind to non-loopback address %s", cfg.host)
|
||||
return 2
|
||||
uvicorn.run(create_app(cfg), host=cfg.host, port=cfg.port, log_config=None,
|
||||
access_log=args.debug)
|
||||
return 0
|
||||
|
||||
|
||||
# installer
|
||||
|
||||
def _unit_dir() -> Path:
|
||||
base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config")
|
||||
return Path(base) / "systemd" / "user"
|
||||
|
||||
|
||||
def _executable() -> str:
|
||||
found = shutil.which("versiond")
|
||||
if found:
|
||||
return os.path.realpath(found)
|
||||
return f"{sys.executable} -m versiond"
|
||||
|
||||
|
||||
def _run(*cmd: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(cmd, check=False, capture_output=True, text=True)
|
||||
|
||||
|
||||
def cmd_install(args: argparse.Namespace) -> int:
|
||||
unit_path = _unit_dir() / UNIT_NAME
|
||||
unit_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
unit_path.write_text(UNIT_TEMPLATE.format(exec_start=_executable()))
|
||||
print(f"wrote {unit_path}")
|
||||
Config.load().api_token()
|
||||
|
||||
user = os.environ.get("USER") or os.getlogin()
|
||||
linger = _run("loginctl", "show-user", user, "-p", "Linger").stdout.strip()
|
||||
if linger != "Linger=yes":
|
||||
result = _run("loginctl", "enable-linger", user)
|
||||
print("enabled lingering" if result.returncode == 0 else f"could not enable lingering: {result.stderr.strip()}")
|
||||
else:
|
||||
print("lingering already enabled")
|
||||
|
||||
for cmd in (("daemon-reload",), ("enable", UNIT_NAME)):
|
||||
result = _run("systemctl", "--user", *cmd)
|
||||
if result.returncode != 0:
|
||||
print(f"systemctl --user {' '.join(cmd)} failed: {result.stderr.strip()}", file=sys.stderr)
|
||||
return 1
|
||||
if not args.no_start:
|
||||
result = _run("systemctl", "--user", "restart", UNIT_NAME)
|
||||
if result.returncode != 0:
|
||||
print(f"start failed: {result.stderr.strip()}", file=sys.stderr)
|
||||
return 1
|
||||
print("service started; check with `versiond status`")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_uninstall(args: argparse.Namespace) -> int:
|
||||
_run("systemctl", "--user", "disable", "--now", UNIT_NAME)
|
||||
unit_path = _unit_dir() / UNIT_NAME
|
||||
if unit_path.exists():
|
||||
unit_path.unlink()
|
||||
print(f"removed {unit_path}")
|
||||
_run("systemctl", "--user", "daemon-reload")
|
||||
print("service removed; data in ~/.local/share/versiond was kept")
|
||||
return 0
|
||||
|
||||
|
||||
# client
|
||||
|
||||
class Client:
|
||||
def __init__(self) -> None:
|
||||
self.cfg = Config.load()
|
||||
self.base = f"http://{self.cfg.host}:{self.cfg.port}"
|
||||
self.token = self.cfg.api_token()
|
||||
|
||||
def request(self, method: str, path: str, body: Any = None, query: dict | None = None,
|
||||
raw: bool = False) -> Any:
|
||||
if query:
|
||||
path += "?" + urllib.parse.urlencode({k: v for k, v in query.items() if v is not None})
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
req = urllib.request.Request(self.base + path, data=data, method=method)
|
||||
req.add_header("Authorization", f"Bearer {self.token}")
|
||||
if data is not None:
|
||||
req.add_header("Content-Type", "application/json")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
payload = resp.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode(errors="replace")
|
||||
try:
|
||||
detail = json.loads(detail).get("detail", detail)
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
raise SystemExit(f"error {exc.code}: {detail}")
|
||||
except urllib.error.URLError as exc:
|
||||
raise SystemExit(f"cannot reach versiond at {self.base}: {exc.reason} "
|
||||
"(is it running? `systemctl --user status versiond`)")
|
||||
if raw:
|
||||
return payload.decode(errors="replace")
|
||||
return json.loads(payload) if payload else None
|
||||
|
||||
|
||||
def _ts(epoch: float | None) -> str:
|
||||
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(epoch)) if epoch else "-"
|
||||
|
||||
|
||||
def _abs(path: str) -> str:
|
||||
return os.path.abspath(os.path.expanduser(path))
|
||||
|
||||
|
||||
def cmd_status(args: argparse.Namespace) -> int:
|
||||
c = Client()
|
||||
health = c.request("GET", "/health")
|
||||
print(f"versiond {health['version']} status={health['status']} uptime={health['uptime_seconds']}s "
|
||||
f"watches={health['watches']} pending={health['pending_paths']}")
|
||||
for root in c.request("GET", "/api/v1/roots"):
|
||||
print(f" [{root['id']}] {root['path']} mode={root['mode']} watches={root['watch_count']} "
|
||||
f"baseline={root['baseline_state']} last_scan={_ts(root['last_scan_at'])}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_token(args: argparse.Namespace) -> int:
|
||||
print(Config.load().api_token())
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_add(args: argparse.Namespace) -> int:
|
||||
root = Client().request("POST", "/api/v1/roots", {"path": _abs(args.path), "force": args.force})
|
||||
print(f"monitoring [{root['id']}] {root['path']} mode={root['mode']} watches={root['watch_count']}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_roots(args: argparse.Namespace) -> int:
|
||||
for root in Client().request("GET", "/api/v1/roots"):
|
||||
print(f"[{root['id']}] {root['path']} mode={root['mode']} watches={root['watch_count']}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_remove(args: argparse.Namespace) -> int:
|
||||
Client().request("DELETE", f"/api/v1/roots/{args.root_id}")
|
||||
print(f"stopped monitoring root {args.root_id} (history kept)")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_snapshot(args: argparse.Namespace) -> int:
|
||||
c = Client()
|
||||
for path in args.paths:
|
||||
result = c.request("POST", "/api/v1/snapshots",
|
||||
{"path": _abs(path), "read_from_disk": True, "source": "cli", "reason": args.reason})
|
||||
print(f"{result['status']:<10} {result['path']} version={result.get('version_id')}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_history(args: argparse.Namespace) -> int:
|
||||
data = Client().request("GET", "/api/v1/files/history", query={"path": _abs(args.path)})
|
||||
print(data["file"]["path"] + ("" if data["file"]["exists_on_disk"] else " (deleted)"))
|
||||
for v in data["versions"]:
|
||||
pin = " pinned" if v["pinned"] else ""
|
||||
print(f" {v['id']:>6} {_ts(v['captured_at'])} {v['size']:>7}B {v['source']}/{v['reason']}{pin}")
|
||||
for r in data["renames"]:
|
||||
print(f" renamed {_ts(r['at'])}: {r['old_path']} -> {r['new_path']}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_diff(args: argparse.Namespace) -> int:
|
||||
c = Client()
|
||||
from_version = args.from_version
|
||||
if from_version is None:
|
||||
versions = c.request("GET", "/api/v1/files/history", query={"path": _abs(args.path)})["versions"]
|
||||
if not versions:
|
||||
raise SystemExit("no versions")
|
||||
# Default: the version before the newest one, so the output shows the last change.
|
||||
from_version = versions[1 if len(versions) > 1 else 0]["id"]
|
||||
out = c.request("GET", "/api/v1/diff", query={"from": from_version, "to": args.to}, raw=True)
|
||||
print(out or "(identical)", end="")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_show(args: argparse.Namespace) -> int:
|
||||
print(Client().request("GET", f"/api/v1/versions/{args.version_id}/content", raw=True), end="")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_restore(args: argparse.Namespace) -> int:
|
||||
c = Client()
|
||||
globs = [_abs(g) if not g.startswith("*") else g for g in args.globs]
|
||||
plan = c.request("POST", "/api/v1/restores", {
|
||||
"globs": globs, "exclude_globs": args.exclude, "as_of": args.as_of,
|
||||
"target_dir": _abs(args.target_dir) if args.target_dir else None,
|
||||
"conflict": args.conflict, "changed_by_source": args.source,
|
||||
})
|
||||
for a in plan["actions"]:
|
||||
print(f" {a['action']:<28} {a['destination']} (version {a['version_id']})")
|
||||
print(f"plan {plan['plan_id']}: {plan['summary']}")
|
||||
if not args.execute:
|
||||
print("dry run; add --execute to apply")
|
||||
return 0
|
||||
result = c.request("POST", f"/api/v1/restores/{plan['plan_id']}/execute")
|
||||
for r in result["results"]:
|
||||
print(f" {r['result']:<10} {r.get('written_to', r['destination'])}" + (f" {r['error']}" if r.get("error") else ""))
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="versiond", description="Local file versioning service")
|
||||
parser.add_argument("--version", action="version", version=f"versiond {__version__}")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
p = sub.add_parser("serve", help="run the server (used by systemd)")
|
||||
p.add_argument("--debug", action="store_true")
|
||||
p.set_defaults(func=cmd_serve)
|
||||
|
||||
p = sub.add_parser("install", help="install and start the systemd user service")
|
||||
p.add_argument("--no-start", action="store_true")
|
||||
p.set_defaults(func=cmd_install)
|
||||
|
||||
sub.add_parser("uninstall", help="stop and remove the systemd user service").set_defaults(func=cmd_uninstall)
|
||||
sub.add_parser("status", help="show service and root status").set_defaults(func=cmd_status)
|
||||
sub.add_parser("token", help="print the local API token").set_defaults(func=cmd_token)
|
||||
|
||||
p = sub.add_parser("add", help="start monitoring a directory")
|
||||
p.add_argument("path")
|
||||
p.add_argument("--force", action="store_true", help="skip the inotify watch budget check")
|
||||
p.set_defaults(func=cmd_add)
|
||||
|
||||
sub.add_parser("roots", help="list monitored directories").set_defaults(func=cmd_roots)
|
||||
|
||||
p = sub.add_parser("remove", help="stop monitoring a directory (keeps history)")
|
||||
p.add_argument("root_id", type=int)
|
||||
p.set_defaults(func=cmd_remove)
|
||||
|
||||
p = sub.add_parser("snapshot", help="store the current state of files now")
|
||||
p.add_argument("paths", nargs="+")
|
||||
p.add_argument("--reason", default="manual")
|
||||
p.set_defaults(func=cmd_snapshot)
|
||||
|
||||
p = sub.add_parser("history", help="list versions of a file")
|
||||
p.add_argument("path")
|
||||
p.set_defaults(func=cmd_history)
|
||||
|
||||
p = sub.add_parser("diff", help="diff a version against disk or another version")
|
||||
p.add_argument("path")
|
||||
p.add_argument("--from", dest="from_version", type=int)
|
||||
p.add_argument("--to", default="disk")
|
||||
p.set_defaults(func=cmd_diff)
|
||||
|
||||
p = sub.add_parser("show", help="print the content of a version")
|
||||
p.add_argument("version_id", type=int)
|
||||
p.set_defaults(func=cmd_show)
|
||||
|
||||
p = sub.add_parser("restore", help="restore files matching globs (dry run unless --execute)")
|
||||
p.add_argument("globs", nargs="+", help="absolute path globs, e.g. '~/proj/src/*.py'")
|
||||
p.add_argument("--as-of", help="ISO time or epoch seconds (default: latest version)")
|
||||
p.add_argument("--exclude", action="append", default=[])
|
||||
p.add_argument("--target-dir")
|
||||
p.add_argument("--source", help="only files changed by this source (e.g. claude-agent)")
|
||||
p.add_argument("--conflict", choices=["overwrite", "skip", "rename", "fail"], default="overwrite")
|
||||
p.add_argument("--execute", action="store_true")
|
||||
p.set_defaults(func=cmd_restore)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
return args.func(args) or 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Configuration, XDG paths and credentials."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
import secrets
|
||||
import tomllib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
APP_NAME = "versiond"
|
||||
DEFAULT_HOST = "127.0.0.1"
|
||||
DEFAULT_PORT = 9922
|
||||
|
||||
DEFAULTS: dict[str, Any] = {
|
||||
"server": {"host": DEFAULT_HOST, "port": DEFAULT_PORT},
|
||||
"limits": {
|
||||
"max_file_bytes": 204_800,
|
||||
"ingest_requests_per_second": 50,
|
||||
},
|
||||
"coalesce": {
|
||||
"window_seconds": 10.0,
|
||||
"max_hold_seconds": 60.0,
|
||||
"versions_per_hour_per_path": 30,
|
||||
},
|
||||
"ignore": {
|
||||
"extra_dirs": [],
|
||||
"patterns": [],
|
||||
"extra_dotfiles": [],
|
||||
},
|
||||
"monitor": {
|
||||
"poll_interval_seconds": 60.0,
|
||||
"reconcile_interval_seconds": 86_400.0,
|
||||
"max_watch_fraction": 0.5,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _xdg(var: str, fallback: str) -> Path:
|
||||
value = os.environ.get(var)
|
||||
return Path(value) if value else Path.home() / fallback
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Paths:
|
||||
config_dir: Path
|
||||
data_dir: Path
|
||||
cache_dir: Path
|
||||
|
||||
@classmethod
|
||||
def resolve(cls) -> "Paths":
|
||||
home = os.environ.get("VERSIOND_HOME")
|
||||
if home:
|
||||
base = Path(home)
|
||||
return cls(base / "config", base / "data", base / "cache")
|
||||
return cls(
|
||||
_xdg("XDG_CONFIG_HOME", ".config") / APP_NAME,
|
||||
_xdg("XDG_DATA_HOME", ".local/share") / APP_NAME,
|
||||
_xdg("XDG_CACHE_HOME", ".cache") / APP_NAME,
|
||||
)
|
||||
|
||||
@property
|
||||
def config_file(self) -> Path:
|
||||
return self.config_dir / "config.toml"
|
||||
|
||||
@property
|
||||
def credentials_file(self) -> Path:
|
||||
return self.config_dir / "credentials"
|
||||
|
||||
@property
|
||||
def index_file(self) -> Path:
|
||||
return self.data_dir / "index.sqlite"
|
||||
|
||||
@property
|
||||
def spool_dir(self) -> Path:
|
||||
return self.data_dir / "spool"
|
||||
|
||||
def ensure(self) -> None:
|
||||
for d in (self.config_dir, self.data_dir, self.cache_dir, self.spool_dir):
|
||||
d.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
|
||||
|
||||
def _merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
|
||||
out = copy.deepcopy(base)
|
||||
for key, value in override.items():
|
||||
if isinstance(value, dict) and isinstance(out.get(key), dict):
|
||||
out[key] = _merge(out[key], value)
|
||||
else:
|
||||
out[key] = value
|
||||
return out
|
||||
|
||||
|
||||
def _toml_value(value: Any) -> str:
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, (int, float)):
|
||||
return repr(value)
|
||||
if isinstance(value, str):
|
||||
escaped = value.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return f'"{escaped}"'
|
||||
if isinstance(value, list):
|
||||
return "[" + ", ".join(_toml_value(v) for v in value) + "]"
|
||||
raise TypeError(f"unsupported TOML value: {value!r}")
|
||||
|
||||
|
||||
def dump_toml(data: dict[str, Any]) -> str:
|
||||
"""Serialize a two-level dict (tables of scalars/lists) to TOML."""
|
||||
lines: list[str] = []
|
||||
for key, value in data.items():
|
||||
if not isinstance(value, dict):
|
||||
lines.append(f"{key} = {_toml_value(value)}")
|
||||
for table, values in data.items():
|
||||
if isinstance(values, dict):
|
||||
lines.append(f"\n[{table}]")
|
||||
for key, value in values.items():
|
||||
lines.append(f"{key} = {_toml_value(value)}")
|
||||
return "\n".join(lines).lstrip() + "\n"
|
||||
|
||||
|
||||
def _write_private(path: Path, text: str) -> None:
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w") as fh:
|
||||
fh.write(text)
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self, paths: Paths, data: dict[str, Any]):
|
||||
self.paths = paths
|
||||
self.data = data
|
||||
|
||||
@classmethod
|
||||
def load(cls, paths: Paths | None = None) -> "Config":
|
||||
paths = paths or Paths.resolve()
|
||||
paths.ensure()
|
||||
user: dict[str, Any] = {}
|
||||
if paths.config_file.exists():
|
||||
user = tomllib.loads(paths.config_file.read_text())
|
||||
else:
|
||||
_write_private(paths.config_file, dump_toml(DEFAULTS))
|
||||
return cls(paths, _merge(DEFAULTS, user))
|
||||
|
||||
def get(self, section: str, key: str) -> Any:
|
||||
return self.data[section][key]
|
||||
|
||||
@property
|
||||
def host(self) -> str:
|
||||
return self.data["server"]["host"]
|
||||
|
||||
@property
|
||||
def port(self) -> int:
|
||||
return int(self.data["server"]["port"])
|
||||
|
||||
def api_token(self) -> str:
|
||||
"""Return the local API token, creating it on first use."""
|
||||
creds: dict[str, Any] = {}
|
||||
if self.paths.credentials_file.exists():
|
||||
creds = tomllib.loads(self.paths.credentials_file.read_text())
|
||||
token = creds.get("api_token")
|
||||
if not token:
|
||||
token = secrets.token_urlsafe(32)
|
||||
creds["api_token"] = token
|
||||
_write_private(self.paths.credentials_file, dump_toml(creds))
|
||||
return token
|
||||
@@ -0,0 +1,186 @@
|
||||
"""SQLite metadata index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS roots (
|
||||
id INTEGER PRIMARY KEY,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
mode TEXT NOT NULL DEFAULT 'pending',
|
||||
watch_count INTEGER NOT NULL DEFAULT 0,
|
||||
polled_dirs INTEGER NOT NULL DEFAULT 0,
|
||||
baseline_state TEXT NOT NULL DEFAULT 'pending',
|
||||
baseline_files INTEGER NOT NULL DEFAULT 0,
|
||||
added_at REAL NOT NULL,
|
||||
last_scan_at REAL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS projects (
|
||||
id INTEGER PRIMARY KEY,
|
||||
root_id INTEGER REFERENCES roots(id) ON DELETE SET NULL,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
detected_by TEXT NOT NULL,
|
||||
created_at REAL NOT NULL,
|
||||
last_activity_at REAL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
id INTEGER PRIMARY KEY,
|
||||
root_id INTEGER REFERENCES roots(id) ON DELETE SET NULL,
|
||||
project_id INTEGER REFERENCES projects(id) ON DELETE SET NULL,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
exists_on_disk INTEGER NOT NULL DEFAULT 1,
|
||||
inode INTEGER,
|
||||
mtime_ns INTEGER,
|
||||
size INTEGER,
|
||||
first_seen_at REAL NOT NULL,
|
||||
last_version_at REAL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS blobs (
|
||||
sha256 TEXT PRIMARY KEY,
|
||||
size INTEGER NOT NULL,
|
||||
stored_size INTEGER NOT NULL,
|
||||
created_at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS versions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
|
||||
blob_sha256 TEXT NOT NULL REFERENCES blobs(sha256),
|
||||
captured_at REAL NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
pinned INTEGER NOT NULL DEFAULT 0,
|
||||
durability TEXT NOT NULL DEFAULT 'local'
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS renames (
|
||||
id INTEGER PRIMARY KEY,
|
||||
file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
|
||||
old_path TEXT NOT NULL,
|
||||
new_path TEXT NOT NULL,
|
||||
at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS restore_plans (
|
||||
id TEXT PRIMARY KEY,
|
||||
created_at REAL NOT NULL,
|
||||
expires_at REAL NOT NULL,
|
||||
criteria_json TEXT NOT NULL,
|
||||
plan_json TEXT NOT NULL,
|
||||
status TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS audit_log (
|
||||
id INTEGER PRIMARY KEY,
|
||||
at REAL NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
details_json TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS versions_file_time ON versions(file_id, captured_at);
|
||||
CREATE INDEX IF NOT EXISTS versions_blob ON versions(blob_sha256);
|
||||
CREATE INDEX IF NOT EXISTS files_root ON files(root_id);
|
||||
CREATE INDEX IF NOT EXISTS files_project ON files(project_id);
|
||||
"""
|
||||
|
||||
|
||||
def row_dict(row: sqlite3.Row | None) -> dict[str, Any] | None:
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, path: Path):
|
||||
self.conn = sqlite3.connect(path, check_same_thread=False, isolation_level=None)
|
||||
self.conn.row_factory = sqlite3.Row
|
||||
self.conn.execute("PRAGMA journal_mode=WAL")
|
||||
self.conn.execute("PRAGMA synchronous=NORMAL")
|
||||
self.conn.execute("PRAGMA foreign_keys=ON")
|
||||
self.conn.executescript(SCHEMA)
|
||||
current = self.conn.execute("PRAGMA user_version").fetchone()[0]
|
||||
if current < SCHEMA_VERSION:
|
||||
self.conn.execute(f"PRAGMA user_version={SCHEMA_VERSION}")
|
||||
|
||||
def close(self) -> None:
|
||||
self.conn.close()
|
||||
|
||||
def execute(self, sql: str, params: Iterable[Any] = ()) -> sqlite3.Cursor:
|
||||
return self.conn.execute(sql, tuple(params))
|
||||
|
||||
def one(self, sql: str, params: Iterable[Any] = ()) -> dict[str, Any] | None:
|
||||
return row_dict(self.conn.execute(sql, tuple(params)).fetchone())
|
||||
|
||||
def all(self, sql: str, params: Iterable[Any] = ()) -> list[dict[str, Any]]:
|
||||
return [dict(r) for r in self.conn.execute(sql, tuple(params)).fetchall()]
|
||||
|
||||
def transaction(self):
|
||||
return _Transaction(self.conn)
|
||||
|
||||
def audit(self, action: str, **details: Any) -> None:
|
||||
self.execute(
|
||||
"INSERT INTO audit_log(at, action, details_json) VALUES (?, ?, ?)",
|
||||
(time.time(), action, json.dumps(details, default=str)),
|
||||
)
|
||||
|
||||
# roots
|
||||
|
||||
def roots(self) -> list[dict[str, Any]]:
|
||||
return self.all("SELECT * FROM roots ORDER BY id")
|
||||
|
||||
def root(self, root_id: int) -> dict[str, Any] | None:
|
||||
return self.one("SELECT * FROM roots WHERE id = ?", (root_id,))
|
||||
|
||||
def update_root(self, root_id: int, **fields: Any) -> None:
|
||||
cols = ", ".join(f"{k} = ?" for k in fields)
|
||||
self.execute(f"UPDATE roots SET {cols} WHERE id = ?", (*fields.values(), root_id))
|
||||
|
||||
# files and versions
|
||||
|
||||
def file_by_path(self, path: str) -> dict[str, Any] | None:
|
||||
return self.one("SELECT * FROM files WHERE path = ?", (path,))
|
||||
|
||||
def file(self, file_id: int) -> dict[str, Any] | None:
|
||||
return self.one("SELECT * FROM files WHERE id = ?", (file_id,))
|
||||
|
||||
def latest_version(self, file_id: int) -> dict[str, Any] | None:
|
||||
return self.one(
|
||||
"SELECT * FROM versions WHERE file_id = ? ORDER BY captured_at DESC, id DESC LIMIT 1",
|
||||
(file_id,),
|
||||
)
|
||||
|
||||
def version(self, version_id: int) -> dict[str, Any] | None:
|
||||
return self.one(
|
||||
"SELECT v.*, f.path, b.size FROM versions v "
|
||||
"JOIN files f ON f.id = v.file_id JOIN blobs b ON b.sha256 = v.blob_sha256 "
|
||||
"WHERE v.id = ?",
|
||||
(version_id,),
|
||||
)
|
||||
|
||||
def version_at(self, file_id: int, as_of: float) -> dict[str, Any] | None:
|
||||
return self.one(
|
||||
"SELECT * FROM versions WHERE file_id = ? AND captured_at <= ? "
|
||||
"ORDER BY captured_at DESC, id DESC LIMIT 1",
|
||||
(file_id, as_of),
|
||||
)
|
||||
|
||||
def history(self, file_id: int) -> list[dict[str, Any]]:
|
||||
return self.all(
|
||||
"SELECT v.id, v.captured_at, v.source, v.reason, v.pinned, v.durability, "
|
||||
"v.blob_sha256, b.size FROM versions v JOIN blobs b ON b.sha256 = v.blob_sha256 "
|
||||
"WHERE v.file_id = ? ORDER BY v.captured_at DESC, v.id DESC",
|
||||
(file_id,),
|
||||
)
|
||||
|
||||
|
||||
class _Transaction:
|
||||
def __init__(self, conn: sqlite3.Connection):
|
||||
self.conn = conn
|
||||
|
||||
def __enter__(self):
|
||||
self.conn.execute("BEGIN IMMEDIATE")
|
||||
return self.conn
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
self.conn.execute("ROLLBACK" if exc_type else "COMMIT")
|
||||
return False
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Text diffs between stored versions or a version and the working copy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
_WS = re.compile(r"\s+")
|
||||
|
||||
|
||||
def _lines(content: bytes) -> list[str]:
|
||||
return content.decode("utf-8", errors="replace").splitlines(keepends=True)
|
||||
|
||||
|
||||
def _normalize(line: str, ignore_whitespace: bool, ignore_eol: bool) -> str:
|
||||
if ignore_whitespace:
|
||||
return _WS.sub(" ", line).strip()
|
||||
if ignore_eol:
|
||||
return line.rstrip("\r\n")
|
||||
return line
|
||||
|
||||
|
||||
def hunks(
|
||||
a: bytes,
|
||||
b: bytes,
|
||||
context: int = 3,
|
||||
ignore_whitespace: bool = False,
|
||||
ignore_eol: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
a_lines, b_lines = _lines(a), _lines(b)
|
||||
a_norm = [_normalize(x, ignore_whitespace, ignore_eol) for x in a_lines]
|
||||
b_norm = [_normalize(x, ignore_whitespace, ignore_eol) for x in b_lines]
|
||||
matcher = difflib.SequenceMatcher(None, a_norm, b_norm, autojunk=False)
|
||||
result = []
|
||||
for group in matcher.get_grouped_opcodes(context):
|
||||
first, last = group[0], group[-1]
|
||||
lines = []
|
||||
for tag, i1, i2, j1, j2 in group:
|
||||
if tag == "equal":
|
||||
lines += [{"op": " ", "text": x} for x in a_lines[i1:i2]]
|
||||
continue
|
||||
if tag in ("replace", "delete"):
|
||||
lines += [{"op": "-", "text": x} for x in a_lines[i1:i2]]
|
||||
if tag in ("replace", "insert"):
|
||||
lines += [{"op": "+", "text": x} for x in b_lines[j1:j2]]
|
||||
result.append({
|
||||
"old_start": first[1] + 1,
|
||||
"old_lines": last[2] - first[1],
|
||||
"new_start": first[3] + 1,
|
||||
"new_lines": last[4] - first[3],
|
||||
"lines": lines,
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
def unified(hunk_list: list[dict[str, Any]], from_label: str, to_label: str) -> str:
|
||||
if not hunk_list:
|
||||
return ""
|
||||
out = [f"--- {from_label}\n", f"+++ {to_label}\n"]
|
||||
for h in hunk_list:
|
||||
out.append(f"@@ -{h['old_start']},{h['old_lines']} +{h['new_start']},{h['new_lines']} @@\n")
|
||||
for line in h["lines"]:
|
||||
text = line["text"]
|
||||
out.append(line["op"] + text + ("" if text.endswith("\n") else "\n\\ No newline at end of file\n"))
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def side_by_side_html(a: bytes, b: bytes, from_label: str, to_label: str, context: int = 3) -> str:
|
||||
return difflib.HtmlDiff(wrapcolumn=100).make_file(
|
||||
_lines(a), _lines(b), from_label, to_label, context=True, numlines=context
|
||||
)
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Ignore rules deciding which directories are watched and which files are versioned."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import PurePath
|
||||
|
||||
IGNORED_DIRS = frozenset({
|
||||
"node_modules", "bower_components", "jspm_packages", "vendor", "__pycache__",
|
||||
"venv", "env", "site-packages", "build", "dist", "target", "out", "bin", "obj",
|
||||
"Pods", "Carthage", "DerivedData", "coverage", "_build", "deps", "elm-stuff",
|
||||
"zig-cache", "zig-out", "__pypackages__", "htmlcov",
|
||||
})
|
||||
|
||||
IGNORED_DIR_SUFFIXES = (".egg-info", ".dist-info", ".xcodeproj", ".xcworkspace")
|
||||
|
||||
IGNORED_SUFFIXES = (
|
||||
# compiled / object code
|
||||
".pyc", ".pyo", ".pyd", ".class", ".o", ".obj", ".so", ".dylib", ".dll", ".exe",
|
||||
".a", ".lib", ".wasm", ".jar", ".war", ".ear", ".whl", ".egg", ".beam", ".elc",
|
||||
# archives
|
||||
".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".zst", ".7z", ".rar",
|
||||
# media
|
||||
".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".bmp", ".tiff", ".psd",
|
||||
".mp3", ".mp4", ".wav", ".ogg", ".flac", ".mov", ".avi", ".mkv", ".webm",
|
||||
".pdf", ".ttf", ".otf", ".woff", ".woff2", ".eot",
|
||||
# databases
|
||||
".sqlite", ".sqlite3", ".db", ".db-wal", ".db-shm", ".sqlite-wal", ".sqlite-shm",
|
||||
# generated
|
||||
".min.js", ".min.css", ".map",
|
||||
# editor temp files
|
||||
".swp", ".swo", ".swx", ".tmp", ".part", ".crdownload", ".orig", ".rej",
|
||||
)
|
||||
|
||||
IGNORED_NAMES = frozenset({"4913"}) # vim's write-permission probe file
|
||||
|
||||
ALLOWED_DOTFILES = frozenset({
|
||||
".env", ".envrc", ".gitignore", ".gitattributes", ".gitmodules", ".editorconfig",
|
||||
".dockerignore", ".npmrc", ".nvmrc", ".yarnrc", ".prettierrc", ".prettierignore",
|
||||
".eslintrc", ".eslintrc.json", ".eslintrc.js", ".eslintignore", ".babelrc",
|
||||
".flake8", ".pylintrc", ".python-version", ".tool-versions", ".htaccess",
|
||||
".clang-format", ".clang-tidy", ".rubocop.yml", ".stylelintrc", ".browserslistrc",
|
||||
".pre-commit-config.yaml", ".golangci.yml", ".terraform-version",
|
||||
})
|
||||
|
||||
|
||||
@dataclass
|
||||
class Filters:
|
||||
max_file_bytes: int = 204_800
|
||||
extra_dirs: frozenset[str] = field(default_factory=frozenset)
|
||||
patterns: tuple[str, ...] = ()
|
||||
extra_dotfiles: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, cfg) -> "Filters":
|
||||
return cls(
|
||||
max_file_bytes=int(cfg.get("limits", "max_file_bytes")),
|
||||
extra_dirs=frozenset(cfg.get("ignore", "extra_dirs")),
|
||||
patterns=tuple(cfg.get("ignore", "patterns")),
|
||||
extra_dotfiles=frozenset(cfg.get("ignore", "extra_dotfiles")),
|
||||
)
|
||||
|
||||
def dir_ignored(self, name: str) -> bool:
|
||||
"""True if a directory with this name must never be watched or scanned."""
|
||||
return (
|
||||
name.startswith(".")
|
||||
or name in IGNORED_DIRS
|
||||
or name in self.extra_dirs
|
||||
or name.endswith(IGNORED_DIR_SUFFIXES)
|
||||
)
|
||||
|
||||
def _dotfile_allowed(self, name: str) -> bool:
|
||||
return (
|
||||
name in ALLOWED_DOTFILES
|
||||
or name in self.extra_dotfiles
|
||||
or name.startswith(".env.")
|
||||
)
|
||||
|
||||
def path_reason(self, rel: PurePath) -> str | None:
|
||||
"""Reason a path (relative to its root) is ignored, or None if accepted."""
|
||||
parts = rel.parts
|
||||
if not parts:
|
||||
return "empty-path"
|
||||
for part in parts[:-1]:
|
||||
if self.dir_ignored(part):
|
||||
return "ignored-directory"
|
||||
name = parts[-1]
|
||||
if name.startswith("."):
|
||||
if not self._dotfile_allowed(name):
|
||||
return "hidden-file"
|
||||
elif name.endswith("~") or name in IGNORED_NAMES:
|
||||
return "temporary-file"
|
||||
lower = name.lower()
|
||||
if lower.endswith(IGNORED_SUFFIXES):
|
||||
return "ignored-extension"
|
||||
rel_posix = rel.as_posix()
|
||||
for pattern in self.patterns:
|
||||
if fnmatch.fnmatch(rel_posix, pattern) or fnmatch.fnmatch(name, pattern):
|
||||
return "ignored-pattern"
|
||||
return None
|
||||
|
||||
def size_reason(self, size: int) -> str | None:
|
||||
return "file-too-large" if size > self.max_file_bytes else None
|
||||
|
||||
@staticmethod
|
||||
def content_reason(content: bytes) -> str | None:
|
||||
return "binary-content" if b"\x00" in content[:8192] else None
|
||||
@@ -0,0 +1,392 @@
|
||||
"""Version commits, project detection, coalescing and rate limiting."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePath
|
||||
from typing import Any, Callable
|
||||
|
||||
from .db import Database
|
||||
from .filters import Filters
|
||||
from .store import BlobStore, sha256
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PROJECT_MARKERS = (
|
||||
(".git", "git"),
|
||||
("pyproject.toml", "python"),
|
||||
("setup.py", "python"),
|
||||
("package.json", "node"),
|
||||
("go.mod", "go"),
|
||||
("Cargo.toml", "rust"),
|
||||
("pom.xml", "maven"),
|
||||
("build.gradle", "gradle"),
|
||||
("composer.json", "php"),
|
||||
("Gemfile", "ruby"),
|
||||
("mix.exs", "elixir"),
|
||||
("CMakeLists.txt", "cmake"),
|
||||
("Makefile", "make"),
|
||||
)
|
||||
|
||||
|
||||
class Rejected(Exception):
|
||||
def __init__(self, reason: str, detail: str = ""):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
self.detail = detail
|
||||
|
||||
|
||||
def is_within(path: str, parent: str) -> bool:
|
||||
return path == parent or path.startswith(parent.rstrip("/") + "/")
|
||||
|
||||
|
||||
class Repository:
|
||||
"""Owns the rules for turning file content into stored versions."""
|
||||
|
||||
def __init__(self, db: Database, blobs: BlobStore, filters: Filters):
|
||||
self.db = db
|
||||
self.blobs = blobs
|
||||
self.filters = filters
|
||||
self._roots: list[dict[str, Any]] = []
|
||||
self._project_cache: dict[str, tuple[str, str]] = {}
|
||||
self.reload_roots()
|
||||
|
||||
def reload_roots(self) -> None:
|
||||
self._roots = sorted(self.db.roots(), key=lambda r: len(r["path"]), reverse=True)
|
||||
self._project_cache.clear()
|
||||
|
||||
def root_for(self, path: str) -> dict[str, Any] | None:
|
||||
for root in self._roots:
|
||||
if is_within(path, root["path"]):
|
||||
return root
|
||||
return None
|
||||
|
||||
def relative(self, path: str, root: dict[str, Any] | None = None) -> PurePath:
|
||||
"""Path used for ignore rules: relative to its root, else to $HOME."""
|
||||
root = root if root is not None else self.root_for(path)
|
||||
if root is not None:
|
||||
return PurePath(path).relative_to(root["path"])
|
||||
home = str(Path.home())
|
||||
if is_within(path, home):
|
||||
return PurePath(path).relative_to(home)
|
||||
return PurePath(path.lstrip("/"))
|
||||
|
||||
def check_path(self, path: str) -> None:
|
||||
reason = self.filters.path_reason(self.relative(path))
|
||||
if reason:
|
||||
raise Rejected(reason, path)
|
||||
|
||||
def check_content(self, content: bytes) -> None:
|
||||
reason = self.filters.size_reason(len(content)) or self.filters.content_reason(content)
|
||||
if reason:
|
||||
raise Rejected(reason, f"{len(content)} bytes")
|
||||
|
||||
def read_file(self, path: str) -> tuple[bytes, os.stat_result]:
|
||||
"""Read a regular file without following a final symlink."""
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
try:
|
||||
st = os.fstat(fd)
|
||||
if not stat.S_ISREG(st.st_mode):
|
||||
raise Rejected("not-a-regular-file", path)
|
||||
reason = self.filters.size_reason(st.st_size)
|
||||
if reason:
|
||||
raise Rejected(reason, f"{st.st_size} bytes")
|
||||
with os.fdopen(fd, "rb", closefd=False) as fh:
|
||||
content = fh.read(self.filters.max_file_bytes + 1)
|
||||
finally:
|
||||
os.close(fd)
|
||||
return content, st
|
||||
|
||||
# projects
|
||||
|
||||
def _detect_project(self, path: str, root: dict[str, Any] | None) -> tuple[str, str]:
|
||||
directory = str(Path(path).parent)
|
||||
if directory in self._project_cache:
|
||||
return self._project_cache[directory]
|
||||
stop = root["path"] if root else directory
|
||||
current = Path(directory)
|
||||
result = (stop, "root" if root else "directory")
|
||||
while is_within(str(current), stop):
|
||||
found = next(
|
||||
(kind for marker, kind in PROJECT_MARKERS if (current / marker).exists()),
|
||||
None,
|
||||
)
|
||||
if found:
|
||||
result = (str(current), found)
|
||||
break
|
||||
if str(current) == stop or current.parent == current:
|
||||
break
|
||||
current = current.parent
|
||||
self._project_cache[directory] = result
|
||||
return result
|
||||
|
||||
def _project_id(self, path: str, root: dict[str, Any] | None) -> int:
|
||||
project_path, kind = self._detect_project(path, root)
|
||||
row = self.db.one("SELECT id FROM projects WHERE path = ?", (project_path,))
|
||||
if row:
|
||||
return row["id"]
|
||||
cur = self.db.execute(
|
||||
"INSERT INTO projects(root_id, path, detected_by, created_at) VALUES (?, ?, ?, ?)",
|
||||
(root["id"] if root else None, project_path, kind, time.time()),
|
||||
)
|
||||
return cur.lastrowid
|
||||
|
||||
def _ensure_file(self, path: str) -> dict[str, Any]:
|
||||
row = self.db.file_by_path(path)
|
||||
if row:
|
||||
return row
|
||||
root = self.root_for(path)
|
||||
project_id = self._project_id(path, root)
|
||||
self.db.execute(
|
||||
"INSERT INTO files(root_id, project_id, path, first_seen_at) VALUES (?, ?, ?, ?)",
|
||||
(root["id"] if root else None, project_id, path, time.time()),
|
||||
)
|
||||
return self.db.file_by_path(path)
|
||||
|
||||
# commits
|
||||
|
||||
def latest_digest(self, path: str) -> str | None:
|
||||
row = self.db.file_by_path(path)
|
||||
if not row:
|
||||
return None
|
||||
latest = self.db.latest_version(row["id"])
|
||||
return latest["blob_sha256"] if latest else None
|
||||
|
||||
def commit(
|
||||
self,
|
||||
path: str,
|
||||
content: bytes,
|
||||
source: str,
|
||||
reason: str,
|
||||
st: os.stat_result | None = None,
|
||||
digest: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
digest = digest or sha256(content)
|
||||
now = time.time()
|
||||
stat_fields = (
|
||||
(st.st_ino, st.st_mtime_ns, st.st_size) if st else (None, None, len(content))
|
||||
)
|
||||
with self.db.transaction():
|
||||
file = self._ensure_file(path)
|
||||
latest = self.db.latest_version(file["id"])
|
||||
if latest and latest["blob_sha256"] == digest:
|
||||
self.db.execute(
|
||||
"UPDATE files SET exists_on_disk = 1, inode = coalesce(?, inode), "
|
||||
"mtime_ns = coalesce(?, mtime_ns), size = ? WHERE id = ?",
|
||||
(*stat_fields, file["id"]),
|
||||
)
|
||||
return {"status": "unchanged", "version_id": latest["id"], "path": path}
|
||||
if not self.db.one("SELECT 1 FROM blobs WHERE sha256 = ?", (digest,)):
|
||||
_, stored = self.blobs.put(content, digest)
|
||||
self.db.execute(
|
||||
"INSERT INTO blobs(sha256, size, stored_size, created_at) VALUES (?, ?, ?, ?)",
|
||||
(digest, len(content), stored, now),
|
||||
)
|
||||
cur = self.db.execute(
|
||||
"INSERT INTO versions(file_id, blob_sha256, captured_at, source, reason) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
(file["id"], digest, now, source, reason),
|
||||
)
|
||||
self.db.execute(
|
||||
"UPDATE files SET exists_on_disk = 1, inode = ?, mtime_ns = ?, size = ?, "
|
||||
"last_version_at = ? WHERE id = ?",
|
||||
(*stat_fields, now, file["id"]),
|
||||
)
|
||||
if file["project_id"]:
|
||||
self.db.execute(
|
||||
"UPDATE projects SET last_activity_at = ? WHERE id = ?",
|
||||
(now, file["project_id"]),
|
||||
)
|
||||
log.debug("committed %s (%s, %s)", path, source, reason)
|
||||
return {"status": "committed", "version_id": cur.lastrowid, "path": path}
|
||||
|
||||
def mark_deleted(self, path: str) -> None:
|
||||
self.db.execute("UPDATE files SET exists_on_disk = 0 WHERE path = ?", (path,))
|
||||
|
||||
def mark_deleted_tree(self, directory: str) -> None:
|
||||
self.db.execute(
|
||||
"UPDATE files SET exists_on_disk = 0 WHERE path LIKE ? ESCAPE '\\'",
|
||||
(_like_prefix(directory),),
|
||||
)
|
||||
|
||||
def rename(self, old: str, new: str) -> bool:
|
||||
"""Move a file's history to a new path. False if not applicable."""
|
||||
file = self.db.file_by_path(old)
|
||||
if not file or self.db.file_by_path(new):
|
||||
return False
|
||||
with self.db.transaction():
|
||||
self.db.execute("UPDATE files SET path = ? WHERE id = ?", (new, file["id"]))
|
||||
self.db.execute(
|
||||
"INSERT INTO renames(file_id, old_path, new_path, at) VALUES (?, ?, ?, ?)",
|
||||
(file["id"], old, new, time.time()),
|
||||
)
|
||||
return True
|
||||
|
||||
def rename_tree(self, old_dir: str, new_dir: str) -> int:
|
||||
rows = self.db.all(
|
||||
"SELECT id, path FROM files WHERE path LIKE ? ESCAPE '\\'", (_like_prefix(old_dir),)
|
||||
)
|
||||
moved = 0
|
||||
with self.db.transaction():
|
||||
for row in rows:
|
||||
new_path = new_dir + row["path"][len(old_dir):]
|
||||
if self.db.file_by_path(new_path):
|
||||
continue
|
||||
self.db.execute("UPDATE files SET path = ? WHERE id = ?", (new_path, row["id"]))
|
||||
self.db.execute(
|
||||
"INSERT INTO renames(file_id, old_path, new_path, at) VALUES (?, ?, ?, ?)",
|
||||
(row["id"], row["path"], new_path, time.time()),
|
||||
)
|
||||
moved += 1
|
||||
self._project_cache.clear()
|
||||
return moved
|
||||
|
||||
|
||||
def _like_prefix(directory: str) -> str:
|
||||
escaped = directory.rstrip("/").replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
return escaped + "/%"
|
||||
|
||||
|
||||
class TokenBucket:
|
||||
def __init__(self, capacity: float, per_seconds: float, clock: Callable[[], float]):
|
||||
self.capacity = capacity
|
||||
self.rate = capacity / per_seconds
|
||||
self.tokens = capacity
|
||||
self.clock = clock
|
||||
self.stamp = clock()
|
||||
|
||||
def _refill(self) -> None:
|
||||
now = self.clock()
|
||||
self.tokens = min(self.capacity, self.tokens + (now - self.stamp) * self.rate)
|
||||
self.stamp = now
|
||||
|
||||
def take(self) -> bool:
|
||||
self._refill()
|
||||
if self.tokens >= 1:
|
||||
self.tokens -= 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def wait_time(self) -> float:
|
||||
self._refill()
|
||||
return max(0.0, (1 - self.tokens) / self.rate)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Pending:
|
||||
opened_at: float
|
||||
content: bytes | None = None
|
||||
digest: str | None = None
|
||||
source: str = ""
|
||||
reason: str = ""
|
||||
st: os.stat_result | None = None
|
||||
handle: asyncio.TimerHandle | None = None
|
||||
|
||||
|
||||
class Coalescer:
|
||||
"""Keeps the first and last state of a burst of changes to one path.
|
||||
|
||||
The first change is committed immediately and opens a window. Changes
|
||||
inside the window replace each other; when the window closes (debounced,
|
||||
capped by max_hold) only the last one is committed. A per-path token
|
||||
bucket caps committed versions per hour; over the limit, changes keep
|
||||
coalescing until a token is free, so the newest state is never lost.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: Repository,
|
||||
window: float,
|
||||
max_hold: float,
|
||||
per_hour: int,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
):
|
||||
self.repo = repo
|
||||
self.window = window
|
||||
self.max_hold = max_hold
|
||||
self.per_hour = per_hour
|
||||
self.clock = clock
|
||||
self.pending: dict[str, _Pending] = {}
|
||||
self.buckets: dict[str, TokenBucket] = {}
|
||||
|
||||
def _bucket(self, path: str) -> TokenBucket:
|
||||
bucket = self.buckets.get(path)
|
||||
if bucket is None:
|
||||
bucket = self.buckets[path] = TokenBucket(self.per_hour, 3600, self.clock)
|
||||
return bucket
|
||||
|
||||
def _schedule(self, path: str, p: _Pending, delay: float) -> None:
|
||||
if p.handle:
|
||||
p.handle.cancel()
|
||||
p.handle = asyncio.get_running_loop().call_later(delay, self._close, path)
|
||||
|
||||
def submit(
|
||||
self,
|
||||
path: str,
|
||||
content: bytes,
|
||||
source: str,
|
||||
reason: str,
|
||||
st: os.stat_result | None = None,
|
||||
) -> dict[str, Any]:
|
||||
digest = sha256(content)
|
||||
now = self.clock()
|
||||
p = self.pending.get(path)
|
||||
if p is None:
|
||||
if self.repo.latest_digest(path) == digest:
|
||||
return self.repo.commit(path, content, source, reason, st, digest)
|
||||
p = self.pending[path] = _Pending(opened_at=now)
|
||||
if self._bucket(path).take():
|
||||
result = self.repo.commit(path, content, source, reason, st, digest)
|
||||
self._schedule(path, p, self.window)
|
||||
return result
|
||||
p.content, p.digest, p.source, p.reason, p.st = content, digest, source, reason, st
|
||||
self._schedule(path, p, self._bucket(path).wait_time())
|
||||
return {"status": "coalesced", "version_id": None, "path": path}
|
||||
|
||||
if p.content is None and self.repo.latest_digest(path) == digest:
|
||||
return {"status": "unchanged", "version_id": None, "path": path}
|
||||
if p.digest == digest:
|
||||
return {"status": "unchanged", "version_id": None, "path": path}
|
||||
p.content, p.digest, p.source, p.reason, p.st = content, digest, source, reason, st
|
||||
deadline = min(now + self.window, p.opened_at + self.max_hold)
|
||||
self._schedule(path, p, max(0.0, deadline - now))
|
||||
return {"status": "coalesced", "version_id": None, "path": path}
|
||||
|
||||
def _close(self, path: str) -> None:
|
||||
p = self.pending.get(path)
|
||||
if p is None:
|
||||
return
|
||||
if p.content is None:
|
||||
del self.pending[path]
|
||||
return
|
||||
bucket = self._bucket(path)
|
||||
if not bucket.take():
|
||||
self._schedule(path, p, bucket.wait_time())
|
||||
return
|
||||
try:
|
||||
self.repo.commit(path, p.content, p.source, p.reason, p.st, p.digest)
|
||||
except Exception:
|
||||
log.exception("commit failed for %s", path)
|
||||
# Reopen an empty window so continuous editing keeps coalescing.
|
||||
self.pending[path] = fresh = _Pending(opened_at=self.clock())
|
||||
self._schedule(path, fresh, self.window)
|
||||
|
||||
def flush(self) -> int:
|
||||
"""Commit every waiting change immediately (used on shutdown)."""
|
||||
committed = 0
|
||||
for path, p in list(self.pending.items()):
|
||||
if p.handle:
|
||||
p.handle.cancel()
|
||||
if p.content is not None:
|
||||
try:
|
||||
self.repo.commit(path, p.content, p.source, p.reason, p.st, p.digest)
|
||||
committed += 1
|
||||
except Exception:
|
||||
log.exception("flush commit failed for %s", path)
|
||||
self.pending.clear()
|
||||
return committed
|
||||
@@ -0,0 +1,515 @@
|
||||
"""Directory monitoring with inotify, pruned watches and reconciliation scans."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
from asyncinotify import Inotify, Mask, Watch
|
||||
|
||||
from .db import Database
|
||||
from .filters import Filters
|
||||
from .ingest import Coalescer, Rejected, Repository, is_within
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DIR_MASK = (
|
||||
Mask.CLOSE_WRITE
|
||||
| Mask.MOVED_TO
|
||||
| Mask.MOVED_FROM
|
||||
| Mask.CREATE
|
||||
| Mask.DELETE
|
||||
| Mask.DELETE_SELF
|
||||
| Mask.MOVE_SELF
|
||||
| Mask.ONLYDIR
|
||||
| Mask.DONT_FOLLOW
|
||||
| Mask.EXCL_UNLINK
|
||||
)
|
||||
|
||||
NETWORK_FS = frozenset({
|
||||
"nfs", "nfs4", "cifs", "smb3", "smbfs", "sshfs", "9p", "ceph", "glusterfs",
|
||||
"afs", "davfs", "lustre", "gpfs", "virtiofs",
|
||||
})
|
||||
|
||||
MOVE_PAIR_TIMEOUT = 0.5
|
||||
YIELD_EVERY = 200
|
||||
|
||||
|
||||
class RootError(Exception):
|
||||
def __init__(self, code: str, detail: str):
|
||||
super().__init__(detail)
|
||||
self.code = code
|
||||
self.detail = detail
|
||||
|
||||
|
||||
def max_user_watches() -> int:
|
||||
try:
|
||||
return int(Path("/proc/sys/fs/inotify/max_user_watches").read_text())
|
||||
except (OSError, ValueError):
|
||||
return 8192
|
||||
|
||||
|
||||
def watches_in_use() -> int:
|
||||
"""Best-effort count of inotify watches held by this user's processes."""
|
||||
uid = os.getuid()
|
||||
total = 0
|
||||
for proc in Path("/proc").iterdir():
|
||||
if not proc.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
if proc.stat().st_uid != uid:
|
||||
continue
|
||||
for fdinfo in (proc / "fdinfo").iterdir():
|
||||
with open(fdinfo) as fh:
|
||||
total += sum(1 for line in fh if line.startswith("inotify wd:"))
|
||||
except OSError:
|
||||
continue
|
||||
return total
|
||||
|
||||
|
||||
def filesystem_type(path: str) -> str:
|
||||
best, fstype = "", ""
|
||||
try:
|
||||
with open("/proc/self/mounts") as fh:
|
||||
for line in fh:
|
||||
fields = line.split()
|
||||
if len(fields) < 3:
|
||||
continue
|
||||
mountpoint = fields[1].replace("\\040", " ")
|
||||
if is_within(path, mountpoint) and len(mountpoint) > len(best):
|
||||
best, fstype = mountpoint, fields[2]
|
||||
except OSError:
|
||||
pass
|
||||
return fstype
|
||||
|
||||
|
||||
def needs_polling(path: str) -> bool:
|
||||
fstype = filesystem_type(path)
|
||||
return fstype in NETWORK_FS or fstype.startswith("fuse")
|
||||
|
||||
|
||||
class Monitor:
|
||||
def __init__(
|
||||
self,
|
||||
db: Database,
|
||||
repo: Repository,
|
||||
coalescer: Coalescer,
|
||||
filters: Filters,
|
||||
poll_interval: float = 60.0,
|
||||
reconcile_interval: float = 86_400.0,
|
||||
max_watch_fraction: float = 0.5,
|
||||
):
|
||||
self.db = db
|
||||
self.repo = repo
|
||||
self.coalescer = coalescer
|
||||
self.filters = filters
|
||||
self.poll_interval = poll_interval
|
||||
self.reconcile_interval = reconcile_interval
|
||||
self.max_watch_fraction = max_watch_fraction
|
||||
self.inotify: Inotify | None = None
|
||||
self.watches: dict[str, Watch] = {}
|
||||
self.watch_paths: dict[Watch, str] = {}
|
||||
self.polled: dict[int, set[str]] = {}
|
||||
self.missing: set[int] = set()
|
||||
self.moves: dict[int, tuple[str, bool, asyncio.TimerHandle]] = {}
|
||||
self.locks: dict[int, asyncio.Lock] = {}
|
||||
self.tasks: set[asyncio.Task] = set()
|
||||
self._last_full_reconcile = time.monotonic()
|
||||
|
||||
# lifecycle
|
||||
|
||||
async def start(self) -> None:
|
||||
self.inotify = Inotify()
|
||||
self._spawn(self._reader())
|
||||
for root in self.db.roots():
|
||||
await self._activate(root)
|
||||
self._spawn(self._periodic())
|
||||
|
||||
async def stop(self) -> None:
|
||||
for task in list(self.tasks):
|
||||
task.cancel()
|
||||
await asyncio.gather(*self.tasks, return_exceptions=True)
|
||||
for _, _, handle in self.moves.values():
|
||||
handle.cancel()
|
||||
self.moves.clear()
|
||||
if self.inotify is not None:
|
||||
self.inotify.close()
|
||||
self.inotify = None
|
||||
self.watches.clear()
|
||||
self.watch_paths.clear()
|
||||
|
||||
def _spawn(self, coro) -> asyncio.Task:
|
||||
task = asyncio.get_running_loop().create_task(coro)
|
||||
self.tasks.add(task)
|
||||
task.add_done_callback(self._task_done)
|
||||
return task
|
||||
|
||||
def _task_done(self, task: asyncio.Task) -> None:
|
||||
self.tasks.discard(task)
|
||||
if not task.cancelled() and task.exception():
|
||||
log.error("monitor task failed", exc_info=task.exception())
|
||||
|
||||
# roots
|
||||
|
||||
def _lock(self, root_id: int) -> asyncio.Lock:
|
||||
return self.locks.setdefault(root_id, asyncio.Lock())
|
||||
|
||||
async def walk_dirs(self, top: str) -> AsyncIterator[str]:
|
||||
"""Yield top and every non-ignored directory below it (symlinks not followed)."""
|
||||
stack = [top]
|
||||
seen = 0
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
yield current
|
||||
try:
|
||||
with os.scandir(current) as it:
|
||||
for entry in it:
|
||||
if entry.is_dir(follow_symlinks=False) and not self.filters.dir_ignored(entry.name):
|
||||
stack.append(entry.path)
|
||||
except OSError:
|
||||
continue
|
||||
seen += 1
|
||||
if seen % YIELD_EVERY == 0:
|
||||
await asyncio.sleep(0)
|
||||
|
||||
async def count_dirs(self, top: str) -> int:
|
||||
return sum([1 async for _ in self.walk_dirs(top)])
|
||||
|
||||
async def add_root(self, raw_path: str, force: bool = False) -> dict[str, Any]:
|
||||
path = os.path.realpath(os.path.expanduser(raw_path))
|
||||
if not os.path.isdir(path):
|
||||
raise RootError("not-a-directory", f"{path} is not a directory")
|
||||
for root in self.db.roots():
|
||||
if is_within(path, root["path"]):
|
||||
raise RootError("root-overlap", f"{path} is already inside root {root['path']}")
|
||||
if is_within(root["path"], path):
|
||||
raise RootError("root-overlap", f"{path} contains existing root {root['path']}; remove it first")
|
||||
polling = needs_polling(path)
|
||||
if not polling and not force:
|
||||
needed = await self.count_dirs(path)
|
||||
limit = max_user_watches()
|
||||
available = limit - watches_in_use()
|
||||
if needed > available * self.max_watch_fraction:
|
||||
raise RootError(
|
||||
"watch-limit",
|
||||
f"{path} needs {needed} inotify watches but only {available} of {limit} are free; "
|
||||
"choose a smaller directory, raise fs.inotify.max_user_watches, or pass force=true",
|
||||
)
|
||||
cur = self.db.execute(
|
||||
"INSERT INTO roots(path, added_at) VALUES (?, ?)", (path, time.time())
|
||||
)
|
||||
self.db.audit("root.add", path=path)
|
||||
self.repo.reload_roots()
|
||||
root = self.db.root(cur.lastrowid)
|
||||
await self._activate(root)
|
||||
return self.db.root(root["id"])
|
||||
|
||||
async def remove_root(self, root_id: int) -> None:
|
||||
root = self.db.root(root_id)
|
||||
if root is None:
|
||||
raise RootError("not-found", f"root {root_id} does not exist")
|
||||
self._unwatch_tree(root["path"])
|
||||
self.polled.pop(root_id, None)
|
||||
self.missing.discard(root_id)
|
||||
self.db.execute("DELETE FROM roots WHERE id = ?", (root_id,))
|
||||
self.db.audit("root.remove", path=root["path"])
|
||||
self.repo.reload_roots()
|
||||
|
||||
async def _activate(self, root: dict[str, Any]) -> None:
|
||||
root_id, path = root["id"], root["path"]
|
||||
if not os.path.isdir(path):
|
||||
self.missing.add(root_id)
|
||||
self.db.update_root(root_id, mode="missing")
|
||||
log.warning("root %s is missing; waiting for it to reappear", path)
|
||||
return
|
||||
self.missing.discard(root_id)
|
||||
self.polled[root_id] = set()
|
||||
if needs_polling(path):
|
||||
self.polled[root_id].add(path)
|
||||
else:
|
||||
await self._watch_tree(root_id, path)
|
||||
self._refresh_root_status(root_id)
|
||||
reason = "reconcile" if root["baseline_state"] == "done" else "baseline"
|
||||
self._spawn(self.reconcile(root_id, reason))
|
||||
|
||||
def _refresh_root_status(self, root_id: int) -> None:
|
||||
root = self.db.root(root_id)
|
||||
if root is None:
|
||||
return
|
||||
if root_id in self.missing:
|
||||
mode = "missing"
|
||||
elif root["path"] in self.polled.get(root_id, ()):
|
||||
mode = "polling"
|
||||
elif self.polled.get(root_id):
|
||||
mode = "degraded"
|
||||
else:
|
||||
mode = "inotify"
|
||||
self.db.update_root(
|
||||
root_id,
|
||||
mode=mode,
|
||||
watch_count=sum(1 for p in self.watches if is_within(p, root["path"])),
|
||||
polled_dirs=len(self.polled.get(root_id, ())),
|
||||
)
|
||||
|
||||
# watches
|
||||
|
||||
def _add_watch(self, path: str) -> bool:
|
||||
if path in self.watches or self.inotify is None:
|
||||
return True
|
||||
try:
|
||||
watch = self.inotify.add_watch(path, DIR_MASK)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ENOSPC:
|
||||
raise
|
||||
log.debug("cannot watch %s: %s", path, exc)
|
||||
return False
|
||||
self.watches[path] = watch
|
||||
self.watch_paths[watch] = path
|
||||
return True
|
||||
|
||||
async def _watch_tree(self, root_id: int, top: str) -> None:
|
||||
skip: list[str] = []
|
||||
async for directory in self.walk_dirs(top):
|
||||
if any(is_within(directory, s) for s in skip):
|
||||
continue
|
||||
try:
|
||||
self._add_watch(directory)
|
||||
except OSError:
|
||||
log.warning("inotify watch limit reached; polling %s instead", directory)
|
||||
self.polled.setdefault(root_id, set()).add(directory)
|
||||
skip.append(directory)
|
||||
|
||||
def _unwatch_tree(self, top: str) -> None:
|
||||
for path in [p for p in self.watches if is_within(p, top)]:
|
||||
watch = self.watches.pop(path)
|
||||
self.watch_paths.pop(watch, None)
|
||||
if self.inotify is not None:
|
||||
try:
|
||||
self.inotify.rm_watch(watch)
|
||||
except (OSError, ValueError, KeyError):
|
||||
pass
|
||||
|
||||
def _move_watches(self, old: str, new: str) -> None:
|
||||
for path in [p for p in self.watches if is_within(p, old)]:
|
||||
watch = self.watches.pop(path)
|
||||
moved = new + path[len(old):]
|
||||
self.watches[moved] = watch
|
||||
self.watch_paths[watch] = moved
|
||||
|
||||
# events
|
||||
|
||||
async def _reader(self) -> None:
|
||||
assert self.inotify is not None
|
||||
async for event in self.inotify:
|
||||
try:
|
||||
self._handle(event)
|
||||
except Exception:
|
||||
log.exception("failed handling inotify event")
|
||||
|
||||
def _root_id_for(self, path: str) -> int | None:
|
||||
root = self.repo.root_for(path)
|
||||
return root["id"] if root else None
|
||||
|
||||
def _handle(self, event) -> None:
|
||||
mask = event.mask
|
||||
if Mask.Q_OVERFLOW in mask:
|
||||
log.warning("inotify queue overflow; reconciling all roots")
|
||||
for root in self.db.roots():
|
||||
self._spawn(self.reconcile(root["id"], "overflow"))
|
||||
return
|
||||
watch = event.watch
|
||||
if Mask.IGNORED in mask:
|
||||
path = self.watch_paths.pop(watch, None) if watch is not None else None
|
||||
if path is not None and self.watches.get(path) is watch:
|
||||
del self.watches[path]
|
||||
return
|
||||
directory = self.watch_paths.get(watch) if watch is not None else None
|
||||
if directory is None:
|
||||
return
|
||||
|
||||
if event.name is None:
|
||||
if mask & (Mask.DELETE_SELF | Mask.MOVE_SELF):
|
||||
root = self.repo.root_for(directory)
|
||||
if root and root["path"] == directory:
|
||||
log.warning("root %s was removed or moved", directory)
|
||||
self._unwatch_tree(directory)
|
||||
self.missing.add(root["id"])
|
||||
self._refresh_root_status(root["id"])
|
||||
return
|
||||
|
||||
path = os.path.join(directory, str(event.name))
|
||||
is_dir = Mask.ISDIR in mask
|
||||
|
||||
if Mask.MOVED_FROM in mask:
|
||||
handle = asyncio.get_running_loop().call_later(
|
||||
MOVE_PAIR_TIMEOUT, self._moved_out, event.cookie
|
||||
)
|
||||
self.moves[event.cookie] = (path, is_dir, handle)
|
||||
return
|
||||
|
||||
if Mask.MOVED_TO in mask:
|
||||
origin = self.moves.pop(event.cookie, None)
|
||||
if origin is not None:
|
||||
origin[2].cancel()
|
||||
self._moved_in(origin[0] if origin else None, path, is_dir)
|
||||
return
|
||||
|
||||
if is_dir:
|
||||
if Mask.CREATE in mask:
|
||||
self._new_directory(path)
|
||||
elif Mask.DELETE in mask:
|
||||
self._unwatch_tree(path)
|
||||
self.repo.mark_deleted_tree(path)
|
||||
return
|
||||
|
||||
if Mask.CLOSE_WRITE in mask:
|
||||
self.capture(path, "monitor", "modified")
|
||||
elif Mask.DELETE in mask:
|
||||
self.repo.mark_deleted(path)
|
||||
|
||||
def _new_directory(self, path: str) -> None:
|
||||
root_id = self._root_id_for(path)
|
||||
if root_id is None or any(self.filters.dir_ignored(p) for p in self.repo.relative(path).parts):
|
||||
return
|
||||
self._spawn(self._watch_and_scan(root_id, path))
|
||||
|
||||
async def _watch_and_scan(self, root_id: int, path: str) -> None:
|
||||
# Watch first, then scan, so files created before the watch existed are not missed.
|
||||
await self._watch_tree(root_id, path)
|
||||
self._refresh_root_status(root_id)
|
||||
await self.reconcile(root_id, "new-directory", subtree=path)
|
||||
|
||||
def _moved_out(self, cookie: int) -> None:
|
||||
origin = self.moves.pop(cookie, None)
|
||||
if origin is None:
|
||||
return
|
||||
path, is_dir, _ = origin
|
||||
if is_dir:
|
||||
self._unwatch_tree(path)
|
||||
self.repo.mark_deleted_tree(path)
|
||||
else:
|
||||
self.repo.mark_deleted(path)
|
||||
|
||||
def _moved_in(self, old: str | None, new: str, is_dir: bool) -> None:
|
||||
new_ok = self._root_id_for(new) is not None and not any(
|
||||
self.filters.dir_ignored(p) for p in self.repo.relative(new).parts[: None if is_dir else -1]
|
||||
)
|
||||
if is_dir:
|
||||
if old is not None and new_ok and old in self.watches:
|
||||
self._move_watches(old, new)
|
||||
self.repo.rename_tree(old, new)
|
||||
return
|
||||
if old is not None:
|
||||
self._unwatch_tree(old)
|
||||
self.repo.mark_deleted_tree(old)
|
||||
if new_ok:
|
||||
self._new_directory(new)
|
||||
return
|
||||
if old is not None and not self.repo.rename(old, new):
|
||||
self.repo.mark_deleted(old)
|
||||
if new_ok:
|
||||
self.capture(new, "monitor", "renamed" if old else "moved-in")
|
||||
|
||||
def capture(self, path: str, source: str, reason: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
self.repo.check_path(path)
|
||||
content, st = self.repo.read_file(path)
|
||||
self.repo.check_content(content)
|
||||
except Rejected as exc:
|
||||
log.debug("skip %s: %s", path, exc.reason)
|
||||
return None
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except OSError as exc:
|
||||
log.debug("cannot read %s: %s", path, exc)
|
||||
return None
|
||||
return self.coalescer.submit(path, content, source, reason, st)
|
||||
|
||||
# scanning
|
||||
|
||||
async def reconcile(self, root_id: int, reason: str, subtree: str | None = None) -> int:
|
||||
"""Compare disk with the index; store changed files, mark vanished ones deleted."""
|
||||
root = self.db.root(root_id)
|
||||
if root is None or root_id in self.missing:
|
||||
return 0
|
||||
top = subtree or root["path"]
|
||||
committed = 0
|
||||
seen: set[str] = set()
|
||||
async with self._lock(root_id):
|
||||
if reason == "baseline":
|
||||
self.db.update_root(root_id, baseline_state="running")
|
||||
async for directory in self.walk_dirs(top):
|
||||
try:
|
||||
entries = list(os.scandir(directory))
|
||||
except OSError:
|
||||
continue
|
||||
for entry in entries:
|
||||
try:
|
||||
if not entry.is_file(follow_symlinks=False):
|
||||
continue
|
||||
st = entry.stat(follow_symlinks=False)
|
||||
except OSError:
|
||||
continue
|
||||
path = entry.path
|
||||
if self.filters.path_reason(self.repo.relative(path, root)):
|
||||
continue
|
||||
if self.filters.size_reason(st.st_size):
|
||||
continue
|
||||
seen.add(path)
|
||||
if path in self.coalescer.pending:
|
||||
continue
|
||||
row = self.db.file_by_path(path)
|
||||
if (
|
||||
row
|
||||
and row["exists_on_disk"]
|
||||
and row["inode"] == st.st_ino
|
||||
and row["mtime_ns"] == st.st_mtime_ns
|
||||
and row["size"] == st.st_size
|
||||
):
|
||||
continue
|
||||
try:
|
||||
content, fst = self.repo.read_file(path)
|
||||
self.repo.check_content(content)
|
||||
except (Rejected, OSError):
|
||||
continue
|
||||
result = self.repo.commit(path, content, "scan", reason, fst)
|
||||
if result["status"] == "committed":
|
||||
committed += 1
|
||||
await asyncio.sleep(0)
|
||||
rows = self.db.all(
|
||||
"SELECT path FROM files WHERE root_id = ? AND exists_on_disk = 1", (root_id,)
|
||||
)
|
||||
for row in rows:
|
||||
if is_within(row["path"], top) and row["path"] not in seen:
|
||||
if not os.path.isfile(row["path"]):
|
||||
self.repo.mark_deleted(row["path"])
|
||||
fields: dict[str, Any] = {"last_scan_at": time.time()}
|
||||
if reason == "baseline":
|
||||
fields.update(baseline_state="done", baseline_files=committed)
|
||||
self.db.update_root(root_id, **fields)
|
||||
if committed:
|
||||
log.info("%s scan of %s stored %d versions", reason, top, committed)
|
||||
return committed
|
||||
|
||||
async def _periodic(self) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(self.poll_interval)
|
||||
for root_id in list(self.missing):
|
||||
root = self.db.root(root_id)
|
||||
if root is None:
|
||||
self.missing.discard(root_id)
|
||||
elif os.path.isdir(root["path"]):
|
||||
log.info("root %s is back", root["path"])
|
||||
await self._activate(root)
|
||||
for root_id, dirs in list(self.polled.items()):
|
||||
for directory in list(dirs):
|
||||
await self.reconcile(root_id, "poll", subtree=directory)
|
||||
if time.monotonic() - self._last_full_reconcile >= self.reconcile_interval:
|
||||
self._last_full_reconcile = time.monotonic()
|
||||
for root in self.db.roots():
|
||||
await self.reconcile(root["id"], "reconcile")
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Restore planning and execution with path safety."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from .db import Database
|
||||
from .ingest import Repository, is_within
|
||||
from .store import BlobStore, sha256
|
||||
|
||||
PLAN_TTL_SECONDS = 15 * 60
|
||||
|
||||
Conflict = Literal["overwrite", "skip", "rename", "fail"]
|
||||
|
||||
|
||||
class RestoreError(Exception):
|
||||
def __init__(self, code: str, detail: str):
|
||||
super().__init__(detail)
|
||||
self.code = code
|
||||
self.detail = detail
|
||||
|
||||
|
||||
@dataclass
|
||||
class Criteria:
|
||||
as_of: float | None = None
|
||||
paths: list[str] = field(default_factory=list)
|
||||
globs: list[str] = field(default_factory=list)
|
||||
exclude_globs: list[str] = field(default_factory=list)
|
||||
extensions: list[str] = field(default_factory=list)
|
||||
root_id: int | None = None
|
||||
project_id: int | None = None
|
||||
changed_since: float | None = None
|
||||
changed_by_source: str | None = None
|
||||
target_dir: str | None = None
|
||||
conflict: Conflict = "overwrite"
|
||||
remove_newer_files: bool = False
|
||||
|
||||
|
||||
class Restorer:
|
||||
def __init__(self, db: Database, repo: Repository, blobs: BlobStore):
|
||||
self.db = db
|
||||
self.repo = repo
|
||||
self.blobs = blobs
|
||||
|
||||
# path safety
|
||||
|
||||
def allowed_bases(self) -> list[str]:
|
||||
return [str(Path.home())] + [r["path"] for r in self.db.roots()]
|
||||
|
||||
def safe_target(self, target: str) -> str:
|
||||
"""Resolve a write target; refuse traversal, symlink escapes and symlink targets."""
|
||||
if not os.path.isabs(target):
|
||||
raise RestoreError("unsafe-path", f"{target} is not absolute")
|
||||
normalized = os.path.normpath(target)
|
||||
if normalized != target.rstrip("/") or ".." in Path(target).parts:
|
||||
raise RestoreError("unsafe-path", f"{target} is not a normalized path")
|
||||
parent = Path(normalized).parent
|
||||
existing = parent
|
||||
while not existing.exists() and existing != existing.parent:
|
||||
existing = existing.parent
|
||||
resolved_parent = os.path.realpath(existing)
|
||||
if not any(is_within(resolved_parent, base) for base in self.allowed_bases()):
|
||||
raise RestoreError("unsafe-path", f"{target} resolves outside the allowed roots")
|
||||
if os.path.islink(normalized):
|
||||
raise RestoreError("unsafe-path", f"{target} is a symlink")
|
||||
return normalized
|
||||
|
||||
# planning
|
||||
|
||||
def _candidate_files(self, c: Criteria) -> list[dict[str, Any]]:
|
||||
sql = "SELECT * FROM files WHERE 1=1"
|
||||
params: list[Any] = []
|
||||
if c.root_id is not None:
|
||||
sql += " AND root_id = ?"
|
||||
params.append(c.root_id)
|
||||
if c.project_id is not None:
|
||||
sql += " AND project_id = ?"
|
||||
params.append(c.project_id)
|
||||
if c.paths:
|
||||
sql += f" AND path IN ({','.join('?' * len(c.paths))})"
|
||||
params.extend(c.paths)
|
||||
rows = self.db.all(sql, params)
|
||||
out = []
|
||||
for row in rows:
|
||||
path = row["path"]
|
||||
if c.globs and not any(fnmatch.fnmatch(path, g) for g in c.globs):
|
||||
continue
|
||||
if any(fnmatch.fnmatch(path, g) for g in c.exclude_globs):
|
||||
continue
|
||||
if c.extensions and not any(
|
||||
path.endswith(e if e.startswith(".") else "." + e) for e in c.extensions
|
||||
):
|
||||
continue
|
||||
if c.changed_since is not None or c.changed_by_source:
|
||||
sql2 = "SELECT 1 FROM versions WHERE file_id = ?"
|
||||
p2: list[Any] = [row["id"]]
|
||||
if c.changed_since is not None:
|
||||
sql2 += " AND captured_at > ?"
|
||||
p2.append(c.changed_since)
|
||||
if c.changed_by_source:
|
||||
sql2 += " AND source = ?"
|
||||
p2.append(c.changed_by_source)
|
||||
if not self.db.one(sql2 + " LIMIT 1", p2):
|
||||
continue
|
||||
out.append(row)
|
||||
return out
|
||||
|
||||
def _destination(self, file: dict[str, Any], target_dir: str | None) -> str:
|
||||
if not target_dir:
|
||||
return file["path"]
|
||||
root = self.db.root(file["root_id"]) if file["root_id"] else None
|
||||
base = root["path"] if root else str(Path(file["path"]).parent)
|
||||
return os.path.join(target_dir, os.path.relpath(file["path"], base))
|
||||
|
||||
def plan(self, c: Criteria) -> dict[str, Any]:
|
||||
if not (c.paths or c.globs or c.root_id or c.project_id or c.changed_by_source):
|
||||
raise RestoreError("criteria-too-broad", "give at least one of paths, globs, root_id, project_id, changed_by_source")
|
||||
as_of = c.as_of if c.as_of is not None else time.time()
|
||||
target_dir = self.safe_target(os.path.normpath(c.target_dir)) if c.target_dir else None
|
||||
actions = []
|
||||
for file in self._candidate_files(c):
|
||||
version = self.db.version_at(file["id"], as_of)
|
||||
destination = self._destination(file, target_dir)
|
||||
disk_digest = _disk_digest(destination)
|
||||
item: dict[str, Any] = {
|
||||
"file_id": file["id"],
|
||||
"path": file["path"],
|
||||
"destination": destination,
|
||||
"version_id": version["id"] if version else None,
|
||||
}
|
||||
if version is None:
|
||||
if c.remove_newer_files and disk_digest is not None and not target_dir:
|
||||
item["action"] = "remove"
|
||||
else:
|
||||
item["action"] = "skip-not-existing-at-as-of"
|
||||
elif disk_digest == version["blob_sha256"]:
|
||||
item["action"] = "unchanged"
|
||||
elif disk_digest is None:
|
||||
item["action"] = "create"
|
||||
else:
|
||||
item["action"] = {"overwrite": "overwrite", "skip": "skip-conflict",
|
||||
"rename": "write-renamed", "fail": "conflict"}[c.conflict]
|
||||
actions.append(item)
|
||||
plan_id = secrets.token_hex(8)
|
||||
now = time.time()
|
||||
summary: dict[str, int] = {}
|
||||
for a in actions:
|
||||
summary[a["action"]] = summary.get(a["action"], 0) + 1
|
||||
plan = {"plan_id": plan_id, "as_of": as_of, "expires_at": now + PLAN_TTL_SECONDS,
|
||||
"summary": summary, "actions": actions}
|
||||
self.db.execute(
|
||||
"INSERT INTO restore_plans(id, created_at, expires_at, criteria_json, plan_json, status) "
|
||||
"VALUES (?, ?, ?, ?, ?, 'planned')",
|
||||
(plan_id, now, now + PLAN_TTL_SECONDS, json.dumps(c.__dict__), json.dumps(plan)),
|
||||
)
|
||||
return plan
|
||||
|
||||
def get_plan(self, plan_id: str) -> dict[str, Any]:
|
||||
row = self.db.one("SELECT * FROM restore_plans WHERE id = ?", (plan_id,))
|
||||
if row is None:
|
||||
raise RestoreError("not-found", f"restore plan {plan_id} does not exist")
|
||||
plan = json.loads(row["plan_json"])
|
||||
plan["status"] = row["status"]
|
||||
return plan
|
||||
|
||||
# execution
|
||||
|
||||
def execute(self, plan_id: str) -> dict[str, Any]:
|
||||
row = self.db.one("SELECT * FROM restore_plans WHERE id = ?", (plan_id,))
|
||||
if row is None:
|
||||
raise RestoreError("not-found", f"restore plan {plan_id} does not exist")
|
||||
if row["status"] != "planned":
|
||||
raise RestoreError("plan-used", f"restore plan {plan_id} is {row['status']}")
|
||||
if row["expires_at"] < time.time():
|
||||
raise RestoreError("plan-expired", f"restore plan {plan_id} expired; create a new one")
|
||||
plan = json.loads(row["plan_json"])
|
||||
if any(a["action"] == "conflict" for a in plan["actions"]):
|
||||
raise RestoreError("conflict", "plan contains conflicts and conflict policy is 'fail'")
|
||||
self.db.execute("UPDATE restore_plans SET status = 'executing' WHERE id = ?", (plan_id,))
|
||||
results = []
|
||||
for action in plan["actions"]:
|
||||
try:
|
||||
results.append(self._apply(action))
|
||||
except (OSError, RestoreError) as exc:
|
||||
results.append({**action, "result": "error", "error": str(exc)})
|
||||
self.db.execute("UPDATE restore_plans SET status = 'executed' WHERE id = ?", (plan_id,))
|
||||
self.db.audit("restore.execute", plan_id=plan_id, count=len(results))
|
||||
return {"plan_id": plan_id, "results": results}
|
||||
|
||||
def _apply(self, action: dict[str, Any]) -> dict[str, Any]:
|
||||
kind = action["action"]
|
||||
if kind in ("unchanged", "skip-conflict", "skip-not-existing-at-as-of"):
|
||||
return {**action, "result": "skipped"}
|
||||
destination = self.safe_target(action["destination"])
|
||||
if kind == "remove":
|
||||
self._snapshot_current(destination)
|
||||
os.unlink(destination)
|
||||
self.repo.mark_deleted(destination)
|
||||
return {**action, "result": "removed"}
|
||||
version = self.db.version(action["version_id"])
|
||||
content = self.blobs.get(version["blob_sha256"])
|
||||
if kind == "write-renamed":
|
||||
stem, ext = os.path.splitext(destination)
|
||||
destination = f"{stem}.restored-{time.strftime('%Y%m%d-%H%M%S')}{ext}"
|
||||
result = self.write(destination, content, version["id"])
|
||||
return {**action, "result": "restored", "written_to": destination, **result}
|
||||
|
||||
def _snapshot_current(self, path: str) -> int | None:
|
||||
try:
|
||||
content, st = self.repo.read_file(path)
|
||||
except (OSError, Exception):
|
||||
return None
|
||||
if self.repo.filters.size_reason(len(content)) or self.repo.filters.content_reason(content):
|
||||
return None
|
||||
return self.repo.commit(path, content, "restore", "pre-restore", st)["version_id"]
|
||||
|
||||
def write(self, destination: str, content: bytes, version_id: int) -> dict[str, Any]:
|
||||
"""Atomically write content after snapshotting what is there now."""
|
||||
destination = self.safe_target(destination)
|
||||
pre = self._snapshot_current(destination) if os.path.exists(destination) else None
|
||||
Path(destination).parent.mkdir(parents=True, exist_ok=True)
|
||||
mode = os.stat(destination).st_mode & 0o7777 if os.path.exists(destination) else 0o644
|
||||
tmp = f"{destination}.versiond-{secrets.token_hex(4)}.tmp"
|
||||
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as fh:
|
||||
fh.write(content)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(tmp, destination)
|
||||
except BaseException:
|
||||
if os.path.exists(tmp):
|
||||
os.unlink(tmp)
|
||||
raise
|
||||
st = os.stat(destination)
|
||||
committed = self.repo.commit(
|
||||
destination, content, "restore", f"restored from version {version_id}", st
|
||||
)
|
||||
return {"pre_restore_version_id": pre, "new_version_id": committed["version_id"]}
|
||||
|
||||
|
||||
def _disk_digest(path: str) -> str | None:
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
return sha256(fh.read())
|
||||
except OSError:
|
||||
return None
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Local content-addressed blob spool."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from compression import zstd as _codec # Python 3.14+
|
||||
_EXT = ".zst"
|
||||
except ImportError: # pragma: no cover - older Pythons
|
||||
import zlib as _codec
|
||||
_EXT = ".z"
|
||||
|
||||
|
||||
def sha256(content: bytes) -> str:
|
||||
return hashlib.sha256(content).hexdigest()
|
||||
|
||||
|
||||
class BlobStore:
|
||||
def __init__(self, root: Path):
|
||||
self.root = root
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def path_for(self, digest: str) -> Path:
|
||||
return self.root / digest[:2] / digest[2:4] / f"{digest}{_EXT}"
|
||||
|
||||
def has(self, digest: str) -> bool:
|
||||
return self.path_for(digest).exists()
|
||||
|
||||
def put(self, content: bytes, digest: str | None = None) -> tuple[str, int]:
|
||||
"""Store content; returns (digest, stored_size). Idempotent."""
|
||||
digest = digest or sha256(content)
|
||||
target = self.path_for(digest)
|
||||
if target.exists():
|
||||
return digest, target.stat().st_size
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = _codec.compress(content)
|
||||
tmp = target.with_name(f".{target.name}.{os.getpid()}.tmp")
|
||||
with open(tmp, "wb") as fh:
|
||||
fh.write(data)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(tmp, target)
|
||||
return digest, len(data)
|
||||
|
||||
def get(self, digest: str) -> bytes:
|
||||
return _codec.decompress(self.path_for(digest).read_bytes())
|
||||
Reference in New Issue
Block a user