Implement versiond M1: inotify monitor, versioning, diff, restore, systemd install
FastAPI service on 127.0.0.1:9922 that monitors user-chosen directories with pruned inotify watches, stores versions in a local content-addressed spool indexed in SQLite, coalesces bursts (first + last), and offers history, diff and dry-run-first bulk restore. Includes a CLI with a systemd user unit installer. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
retoor
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
3455499cea
commit
ddf09bea88
@@ -0,0 +1,392 @@
|
||||
"""Version commits, project detection, coalescing and rate limiting."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePath
|
||||
from typing import Any, Callable
|
||||
|
||||
from .db import Database
|
||||
from .filters import Filters
|
||||
from .store import BlobStore, sha256
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PROJECT_MARKERS = (
|
||||
(".git", "git"),
|
||||
("pyproject.toml", "python"),
|
||||
("setup.py", "python"),
|
||||
("package.json", "node"),
|
||||
("go.mod", "go"),
|
||||
("Cargo.toml", "rust"),
|
||||
("pom.xml", "maven"),
|
||||
("build.gradle", "gradle"),
|
||||
("composer.json", "php"),
|
||||
("Gemfile", "ruby"),
|
||||
("mix.exs", "elixir"),
|
||||
("CMakeLists.txt", "cmake"),
|
||||
("Makefile", "make"),
|
||||
)
|
||||
|
||||
|
||||
class Rejected(Exception):
|
||||
def __init__(self, reason: str, detail: str = ""):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
self.detail = detail
|
||||
|
||||
|
||||
def is_within(path: str, parent: str) -> bool:
|
||||
return path == parent or path.startswith(parent.rstrip("/") + "/")
|
||||
|
||||
|
||||
class Repository:
|
||||
"""Owns the rules for turning file content into stored versions."""
|
||||
|
||||
def __init__(self, db: Database, blobs: BlobStore, filters: Filters):
|
||||
self.db = db
|
||||
self.blobs = blobs
|
||||
self.filters = filters
|
||||
self._roots: list[dict[str, Any]] = []
|
||||
self._project_cache: dict[str, tuple[str, str]] = {}
|
||||
self.reload_roots()
|
||||
|
||||
def reload_roots(self) -> None:
|
||||
self._roots = sorted(self.db.roots(), key=lambda r: len(r["path"]), reverse=True)
|
||||
self._project_cache.clear()
|
||||
|
||||
def root_for(self, path: str) -> dict[str, Any] | None:
|
||||
for root in self._roots:
|
||||
if is_within(path, root["path"]):
|
||||
return root
|
||||
return None
|
||||
|
||||
def relative(self, path: str, root: dict[str, Any] | None = None) -> PurePath:
|
||||
"""Path used for ignore rules: relative to its root, else to $HOME."""
|
||||
root = root if root is not None else self.root_for(path)
|
||||
if root is not None:
|
||||
return PurePath(path).relative_to(root["path"])
|
||||
home = str(Path.home())
|
||||
if is_within(path, home):
|
||||
return PurePath(path).relative_to(home)
|
||||
return PurePath(path.lstrip("/"))
|
||||
|
||||
def check_path(self, path: str) -> None:
|
||||
reason = self.filters.path_reason(self.relative(path))
|
||||
if reason:
|
||||
raise Rejected(reason, path)
|
||||
|
||||
def check_content(self, content: bytes) -> None:
|
||||
reason = self.filters.size_reason(len(content)) or self.filters.content_reason(content)
|
||||
if reason:
|
||||
raise Rejected(reason, f"{len(content)} bytes")
|
||||
|
||||
def read_file(self, path: str) -> tuple[bytes, os.stat_result]:
|
||||
"""Read a regular file without following a final symlink."""
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
try:
|
||||
st = os.fstat(fd)
|
||||
if not stat.S_ISREG(st.st_mode):
|
||||
raise Rejected("not-a-regular-file", path)
|
||||
reason = self.filters.size_reason(st.st_size)
|
||||
if reason:
|
||||
raise Rejected(reason, f"{st.st_size} bytes")
|
||||
with os.fdopen(fd, "rb", closefd=False) as fh:
|
||||
content = fh.read(self.filters.max_file_bytes + 1)
|
||||
finally:
|
||||
os.close(fd)
|
||||
return content, st
|
||||
|
||||
# projects
|
||||
|
||||
def _detect_project(self, path: str, root: dict[str, Any] | None) -> tuple[str, str]:
|
||||
directory = str(Path(path).parent)
|
||||
if directory in self._project_cache:
|
||||
return self._project_cache[directory]
|
||||
stop = root["path"] if root else directory
|
||||
current = Path(directory)
|
||||
result = (stop, "root" if root else "directory")
|
||||
while is_within(str(current), stop):
|
||||
found = next(
|
||||
(kind for marker, kind in PROJECT_MARKERS if (current / marker).exists()),
|
||||
None,
|
||||
)
|
||||
if found:
|
||||
result = (str(current), found)
|
||||
break
|
||||
if str(current) == stop or current.parent == current:
|
||||
break
|
||||
current = current.parent
|
||||
self._project_cache[directory] = result
|
||||
return result
|
||||
|
||||
def _project_id(self, path: str, root: dict[str, Any] | None) -> int:
|
||||
project_path, kind = self._detect_project(path, root)
|
||||
row = self.db.one("SELECT id FROM projects WHERE path = ?", (project_path,))
|
||||
if row:
|
||||
return row["id"]
|
||||
cur = self.db.execute(
|
||||
"INSERT INTO projects(root_id, path, detected_by, created_at) VALUES (?, ?, ?, ?)",
|
||||
(root["id"] if root else None, project_path, kind, time.time()),
|
||||
)
|
||||
return cur.lastrowid
|
||||
|
||||
def _ensure_file(self, path: str) -> dict[str, Any]:
|
||||
row = self.db.file_by_path(path)
|
||||
if row:
|
||||
return row
|
||||
root = self.root_for(path)
|
||||
project_id = self._project_id(path, root)
|
||||
self.db.execute(
|
||||
"INSERT INTO files(root_id, project_id, path, first_seen_at) VALUES (?, ?, ?, ?)",
|
||||
(root["id"] if root else None, project_id, path, time.time()),
|
||||
)
|
||||
return self.db.file_by_path(path)
|
||||
|
||||
# commits
|
||||
|
||||
def latest_digest(self, path: str) -> str | None:
|
||||
row = self.db.file_by_path(path)
|
||||
if not row:
|
||||
return None
|
||||
latest = self.db.latest_version(row["id"])
|
||||
return latest["blob_sha256"] if latest else None
|
||||
|
||||
def commit(
|
||||
self,
|
||||
path: str,
|
||||
content: bytes,
|
||||
source: str,
|
||||
reason: str,
|
||||
st: os.stat_result | None = None,
|
||||
digest: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
digest = digest or sha256(content)
|
||||
now = time.time()
|
||||
stat_fields = (
|
||||
(st.st_ino, st.st_mtime_ns, st.st_size) if st else (None, None, len(content))
|
||||
)
|
||||
with self.db.transaction():
|
||||
file = self._ensure_file(path)
|
||||
latest = self.db.latest_version(file["id"])
|
||||
if latest and latest["blob_sha256"] == digest:
|
||||
self.db.execute(
|
||||
"UPDATE files SET exists_on_disk = 1, inode = coalesce(?, inode), "
|
||||
"mtime_ns = coalesce(?, mtime_ns), size = ? WHERE id = ?",
|
||||
(*stat_fields, file["id"]),
|
||||
)
|
||||
return {"status": "unchanged", "version_id": latest["id"], "path": path}
|
||||
if not self.db.one("SELECT 1 FROM blobs WHERE sha256 = ?", (digest,)):
|
||||
_, stored = self.blobs.put(content, digest)
|
||||
self.db.execute(
|
||||
"INSERT INTO blobs(sha256, size, stored_size, created_at) VALUES (?, ?, ?, ?)",
|
||||
(digest, len(content), stored, now),
|
||||
)
|
||||
cur = self.db.execute(
|
||||
"INSERT INTO versions(file_id, blob_sha256, captured_at, source, reason) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
(file["id"], digest, now, source, reason),
|
||||
)
|
||||
self.db.execute(
|
||||
"UPDATE files SET exists_on_disk = 1, inode = ?, mtime_ns = ?, size = ?, "
|
||||
"last_version_at = ? WHERE id = ?",
|
||||
(*stat_fields, now, file["id"]),
|
||||
)
|
||||
if file["project_id"]:
|
||||
self.db.execute(
|
||||
"UPDATE projects SET last_activity_at = ? WHERE id = ?",
|
||||
(now, file["project_id"]),
|
||||
)
|
||||
log.debug("committed %s (%s, %s)", path, source, reason)
|
||||
return {"status": "committed", "version_id": cur.lastrowid, "path": path}
|
||||
|
||||
def mark_deleted(self, path: str) -> None:
|
||||
self.db.execute("UPDATE files SET exists_on_disk = 0 WHERE path = ?", (path,))
|
||||
|
||||
def mark_deleted_tree(self, directory: str) -> None:
|
||||
self.db.execute(
|
||||
"UPDATE files SET exists_on_disk = 0 WHERE path LIKE ? ESCAPE '\\'",
|
||||
(_like_prefix(directory),),
|
||||
)
|
||||
|
||||
def rename(self, old: str, new: str) -> bool:
|
||||
"""Move a file's history to a new path. False if not applicable."""
|
||||
file = self.db.file_by_path(old)
|
||||
if not file or self.db.file_by_path(new):
|
||||
return False
|
||||
with self.db.transaction():
|
||||
self.db.execute("UPDATE files SET path = ? WHERE id = ?", (new, file["id"]))
|
||||
self.db.execute(
|
||||
"INSERT INTO renames(file_id, old_path, new_path, at) VALUES (?, ?, ?, ?)",
|
||||
(file["id"], old, new, time.time()),
|
||||
)
|
||||
return True
|
||||
|
||||
def rename_tree(self, old_dir: str, new_dir: str) -> int:
|
||||
rows = self.db.all(
|
||||
"SELECT id, path FROM files WHERE path LIKE ? ESCAPE '\\'", (_like_prefix(old_dir),)
|
||||
)
|
||||
moved = 0
|
||||
with self.db.transaction():
|
||||
for row in rows:
|
||||
new_path = new_dir + row["path"][len(old_dir):]
|
||||
if self.db.file_by_path(new_path):
|
||||
continue
|
||||
self.db.execute("UPDATE files SET path = ? WHERE id = ?", (new_path, row["id"]))
|
||||
self.db.execute(
|
||||
"INSERT INTO renames(file_id, old_path, new_path, at) VALUES (?, ?, ?, ?)",
|
||||
(row["id"], row["path"], new_path, time.time()),
|
||||
)
|
||||
moved += 1
|
||||
self._project_cache.clear()
|
||||
return moved
|
||||
|
||||
|
||||
def _like_prefix(directory: str) -> str:
|
||||
escaped = directory.rstrip("/").replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
return escaped + "/%"
|
||||
|
||||
|
||||
class TokenBucket:
|
||||
def __init__(self, capacity: float, per_seconds: float, clock: Callable[[], float]):
|
||||
self.capacity = capacity
|
||||
self.rate = capacity / per_seconds
|
||||
self.tokens = capacity
|
||||
self.clock = clock
|
||||
self.stamp = clock()
|
||||
|
||||
def _refill(self) -> None:
|
||||
now = self.clock()
|
||||
self.tokens = min(self.capacity, self.tokens + (now - self.stamp) * self.rate)
|
||||
self.stamp = now
|
||||
|
||||
def take(self) -> bool:
|
||||
self._refill()
|
||||
if self.tokens >= 1:
|
||||
self.tokens -= 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def wait_time(self) -> float:
|
||||
self._refill()
|
||||
return max(0.0, (1 - self.tokens) / self.rate)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Pending:
|
||||
opened_at: float
|
||||
content: bytes | None = None
|
||||
digest: str | None = None
|
||||
source: str = ""
|
||||
reason: str = ""
|
||||
st: os.stat_result | None = None
|
||||
handle: asyncio.TimerHandle | None = None
|
||||
|
||||
|
||||
class Coalescer:
|
||||
"""Keeps the first and last state of a burst of changes to one path.
|
||||
|
||||
The first change is committed immediately and opens a window. Changes
|
||||
inside the window replace each other; when the window closes (debounced,
|
||||
capped by max_hold) only the last one is committed. A per-path token
|
||||
bucket caps committed versions per hour; over the limit, changes keep
|
||||
coalescing until a token is free, so the newest state is never lost.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: Repository,
|
||||
window: float,
|
||||
max_hold: float,
|
||||
per_hour: int,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
):
|
||||
self.repo = repo
|
||||
self.window = window
|
||||
self.max_hold = max_hold
|
||||
self.per_hour = per_hour
|
||||
self.clock = clock
|
||||
self.pending: dict[str, _Pending] = {}
|
||||
self.buckets: dict[str, TokenBucket] = {}
|
||||
|
||||
def _bucket(self, path: str) -> TokenBucket:
|
||||
bucket = self.buckets.get(path)
|
||||
if bucket is None:
|
||||
bucket = self.buckets[path] = TokenBucket(self.per_hour, 3600, self.clock)
|
||||
return bucket
|
||||
|
||||
def _schedule(self, path: str, p: _Pending, delay: float) -> None:
|
||||
if p.handle:
|
||||
p.handle.cancel()
|
||||
p.handle = asyncio.get_running_loop().call_later(delay, self._close, path)
|
||||
|
||||
def submit(
|
||||
self,
|
||||
path: str,
|
||||
content: bytes,
|
||||
source: str,
|
||||
reason: str,
|
||||
st: os.stat_result | None = None,
|
||||
) -> dict[str, Any]:
|
||||
digest = sha256(content)
|
||||
now = self.clock()
|
||||
p = self.pending.get(path)
|
||||
if p is None:
|
||||
if self.repo.latest_digest(path) == digest:
|
||||
return self.repo.commit(path, content, source, reason, st, digest)
|
||||
p = self.pending[path] = _Pending(opened_at=now)
|
||||
if self._bucket(path).take():
|
||||
result = self.repo.commit(path, content, source, reason, st, digest)
|
||||
self._schedule(path, p, self.window)
|
||||
return result
|
||||
p.content, p.digest, p.source, p.reason, p.st = content, digest, source, reason, st
|
||||
self._schedule(path, p, self._bucket(path).wait_time())
|
||||
return {"status": "coalesced", "version_id": None, "path": path}
|
||||
|
||||
if p.content is None and self.repo.latest_digest(path) == digest:
|
||||
return {"status": "unchanged", "version_id": None, "path": path}
|
||||
if p.digest == digest:
|
||||
return {"status": "unchanged", "version_id": None, "path": path}
|
||||
p.content, p.digest, p.source, p.reason, p.st = content, digest, source, reason, st
|
||||
deadline = min(now + self.window, p.opened_at + self.max_hold)
|
||||
self._schedule(path, p, max(0.0, deadline - now))
|
||||
return {"status": "coalesced", "version_id": None, "path": path}
|
||||
|
||||
def _close(self, path: str) -> None:
|
||||
p = self.pending.get(path)
|
||||
if p is None:
|
||||
return
|
||||
if p.content is None:
|
||||
del self.pending[path]
|
||||
return
|
||||
bucket = self._bucket(path)
|
||||
if not bucket.take():
|
||||
self._schedule(path, p, bucket.wait_time())
|
||||
return
|
||||
try:
|
||||
self.repo.commit(path, p.content, p.source, p.reason, p.st, p.digest)
|
||||
except Exception:
|
||||
log.exception("commit failed for %s", path)
|
||||
# Reopen an empty window so continuous editing keeps coalescing.
|
||||
self.pending[path] = fresh = _Pending(opened_at=self.clock())
|
||||
self._schedule(path, fresh, self.window)
|
||||
|
||||
def flush(self) -> int:
|
||||
"""Commit every waiting change immediately (used on shutdown)."""
|
||||
committed = 0
|
||||
for path, p in list(self.pending.items()):
|
||||
if p.handle:
|
||||
p.handle.cancel()
|
||||
if p.content is not None:
|
||||
try:
|
||||
self.repo.commit(path, p.content, p.source, p.reason, p.st, p.digest)
|
||||
committed += 1
|
||||
except Exception:
|
||||
log.exception("flush commit failed for %s", path)
|
||||
self.pending.clear()
|
||||
return committed
|
||||
Reference in New Issue
Block a user