Remove encryption; user-data filters; SQLite safety; recovery, retention, scheduler
- No client-side encryption: plaintext content-addressed remote (SHA-256 names), no backup key, no cryptography dependency (server disk encryption is the trust model). - Filters back user data: images, archives, databases, PDFs accepted; 10 MiB cap; binary sniffing removed; temp names hardened (~$, #..#, .temp). - SQLite zero-error policy: backup-API snapshots + integrity_check, journal folding, locked/corrupt loud skips, verified restores. - Recovery: reindex from manifests, remote adopt, on-demand blob fetch, 5-day retention + thinning, GC, date-guarded remote purge, metrics. - Scheduler with WebDAV quota signal and 70% pressure backstop (floor kept). - 37 tests incl. live-monitor capture safety and DB safety.
This commit is contained in:
retoor
@@ -0,0 +1,109 @@
|
||||
"""Bone-level proof: temp files and unfinished writes are never versioned.
|
||||
|
||||
Runs against the real inotify monitor through the HTTP API (same fixture
|
||||
shape as test_service). Each test fails if a single temp/unfinished state
|
||||
is ever stored.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from versiond.api import create_app
|
||||
from versiond.config import Config, Paths
|
||||
|
||||
from test_service import CONFIG, history, wait_for
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("VERSIOND_HOME", str(tmp_path / "home"))
|
||||
paths = Paths.resolve()
|
||||
paths.ensure()
|
||||
paths.config_file.write_text(CONFIG)
|
||||
cfg = Config.load(paths)
|
||||
work = tmp_path / "work"
|
||||
(work / "src").mkdir(parents=True)
|
||||
(work / "src" / "app.py").write_text("v0\n")
|
||||
with TestClient(create_app(cfg), base_url="http://127.0.0.1:9922") as client:
|
||||
client.headers["Authorization"] = f"Bearer {cfg.api_token()}"
|
||||
yield client, work
|
||||
|
||||
|
||||
def _add_root(client, work):
|
||||
r = client.post("/api/v1/roots", json={"path": str(work)})
|
||||
assert r.status_code == 201, r.text
|
||||
root = r.json()
|
||||
wait_for(lambda: client.get(f"/api/v1/roots/{root['id']}").json()["baseline_state"] == "done")
|
||||
return root
|
||||
|
||||
|
||||
def test_temp_names_never_stored(env):
|
||||
client, work = env
|
||||
_add_root(client, work)
|
||||
temps = ["a.tmp", "b.temp", "c.swp", "d.kate-swp", "e~", "f.part",
|
||||
"g.crdownload", "~$lock.docx", "#autosave#", "4913", "h.orig"]
|
||||
for name in temps:
|
||||
(work / "src" / name).write_text("unfinished\n")
|
||||
time.sleep(0.8) # past the 0.2s coalescing window: anything stored would show
|
||||
for name in temps:
|
||||
assert history(client, work / "src" / name) == [], name
|
||||
# Same via the push API: every one must be refused, none stored.
|
||||
for name in temps:
|
||||
body = {"path": str(work / "src" / name),
|
||||
"content_b64": base64.b64encode(b"unfinished\n").decode()}
|
||||
assert client.post("/api/v1/snapshots", json=body).status_code == 422, name
|
||||
|
||||
|
||||
def test_open_file_not_captured_until_close(env):
|
||||
client, work = env
|
||||
_add_root(client, work)
|
||||
target = work / "src" / "slow.py"
|
||||
fh = open(target, "w")
|
||||
try:
|
||||
fh.write("partial-1\n")
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno()) # bytes on disk, but writer still holds it open
|
||||
time.sleep(0.6)
|
||||
assert history(client, target) == [] # no CLOSE_WRITE yet: nothing stored
|
||||
fh.write("partial-2\n")
|
||||
fh.flush()
|
||||
finally:
|
||||
fh.close() # only now is the state finished and capturable
|
||||
wait_for(lambda: history(client, target))
|
||||
versions = history(client, target)
|
||||
assert len(versions) == 1
|
||||
assert client.get(f"/api/v1/versions/{versions[0]['id']}/content").text == "partial-1\npartial-2\n"
|
||||
|
||||
|
||||
def test_burst_collapses_to_first_and_last(env):
|
||||
client, work = env
|
||||
_add_root(client, work)
|
||||
target = work / "src" / "busy.py"
|
||||
target.write_text("v0\n")
|
||||
wait_for(lambda: len(history(client, target)) == 1) # first observed state committed
|
||||
for i in range(1, 6):
|
||||
target.write_text(f"v{i}\n") # burst inside the coalescing window
|
||||
wait_for(lambda: len(history(client, target)) == 2, timeout=8)
|
||||
time.sleep(1.5) # past max_hold: no third version may appear
|
||||
versions = history(client, target)
|
||||
assert len(versions) == 2
|
||||
bodies = [client.get(f"/api/v1/versions/{v['id']}/content").text for v in reversed(versions)]
|
||||
assert bodies == ["v0\n", "v5\n"]
|
||||
|
||||
|
||||
def test_atomic_save_is_one_version(env):
|
||||
client, work = env
|
||||
_add_root(client, work)
|
||||
target = work / "src" / "app.py"
|
||||
wait_for(lambda: history(client, target))
|
||||
tmp = work / "src" / "app.py.tmp"
|
||||
tmp.write_text("atomic\n")
|
||||
os.replace(tmp, target)
|
||||
wait_for(lambda: len(history(client, target)) == 2)
|
||||
time.sleep(0.6)
|
||||
assert len(history(client, target)) == 2 # temp state never stored
|
||||
assert history(client, work / "src" / "app.py.tmp") == []
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Zero-error policy for live databases: only verified snapshots are stored."""
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from versiond import dbsafe
|
||||
from versiond.db import Database
|
||||
from versiond.filters import Filters
|
||||
from versiond.ingest import Rejected, Repository
|
||||
from versiond.store import BlobStore
|
||||
|
||||
|
||||
def _repo(tmp_path):
|
||||
db = Database(tmp_path / "index.sqlite")
|
||||
return Repository(db, BlobStore(tmp_path / "spool"), Filters())
|
||||
|
||||
|
||||
def _open_db(path, **kw):
|
||||
conn = sqlite3.connect(path, **kw)
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS t(id INTEGER PRIMARY KEY, v TEXT)")
|
||||
return conn
|
||||
|
||||
|
||||
def _read_snapshot_bytes(repo, path):
|
||||
data, _ = repo.read_file(path)
|
||||
return data
|
||||
|
||||
|
||||
def _rows_in_snapshot(data):
|
||||
import tempfile, os
|
||||
fd, tmp = tempfile.mkstemp(suffix=".sqlite")
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as fh:
|
||||
fh.write(data)
|
||||
conn = sqlite3.connect(f"file:{tmp}?mode=ro", uri=True)
|
||||
try:
|
||||
assert conn.execute("PRAGMA integrity_check").fetchone()[0] == "ok"
|
||||
return conn.execute("SELECT v FROM t ORDER BY id").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
finally:
|
||||
os.unlink(tmp)
|
||||
|
||||
|
||||
def test_sqlite_snapshot_roundtrip(tmp_path):
|
||||
repo = _repo(tmp_path)
|
||||
db_path = str(tmp_path / "app.db")
|
||||
conn = _open_db(db_path)
|
||||
conn.execute("INSERT INTO t(v) VALUES ('a'), ('b')")
|
||||
conn.commit()
|
||||
data = _read_snapshot_bytes(repo, db_path)
|
||||
assert _rows_in_snapshot(data) == [("a",), ("b",)]
|
||||
result = repo.commit(db_path, data, "test", "edit")
|
||||
assert result["status"] == "committed"
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_snapshot_during_uncommitted_write_is_consistent(tmp_path):
|
||||
"""A writer with an open transaction must not produce a torn version."""
|
||||
repo = _repo(tmp_path)
|
||||
db_path = str(tmp_path / "live.db")
|
||||
writer = _open_db(db_path, isolation_level=None)
|
||||
writer.execute("INSERT INTO t(v) VALUES ('committed')")
|
||||
writer.execute("BEGIN IMMEDIATE")
|
||||
writer.execute("INSERT INTO t(v) VALUES ('uncommitted')")
|
||||
try:
|
||||
data = _read_snapshot_bytes(repo, db_path)
|
||||
finally:
|
||||
writer.execute("ROLLBACK")
|
||||
writer.close()
|
||||
rows = _rows_in_snapshot(data)
|
||||
assert rows == [("committed",)] # committed state only, verified clean
|
||||
|
||||
|
||||
def test_wal_mode_snapshot(tmp_path):
|
||||
repo = _repo(tmp_path)
|
||||
db_path = str(tmp_path / "wal.db")
|
||||
conn = _open_db(db_path)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("INSERT INTO t(v) VALUES ('w1')")
|
||||
conn.commit()
|
||||
conn.execute("INSERT INTO t(v) VALUES ('w2')") # may sit in -wal
|
||||
data = _read_snapshot_bytes(repo, db_path)
|
||||
conn.close()
|
||||
rows = _rows_in_snapshot(data)
|
||||
assert ("w1",) in rows # committed data present, snapshot verified
|
||||
|
||||
|
||||
def test_corrupt_sqlite_magic_rejected(tmp_path):
|
||||
repo = _repo(tmp_path)
|
||||
bad = tmp_path / "bad.db"
|
||||
bad.write_bytes(dbsafe.SQLITE_MAGIC + b"\x00" * 100 + b"garbage")
|
||||
with pytest.raises(Rejected) as exc:
|
||||
repo.read_file(str(bad))
|
||||
assert exc.value.reason in ("database-corrupt", "database-unreadable")
|
||||
|
||||
|
||||
def test_locked_database_rejected_not_stored(tmp_path):
|
||||
repo = _repo(tmp_path)
|
||||
db_path = str(tmp_path / "locked.db")
|
||||
holder = _open_db(db_path, isolation_level=None)
|
||||
holder.execute("INSERT INTO t(v) VALUES ('x')")
|
||||
holder.execute("BEGIN EXCLUSIVE")
|
||||
try:
|
||||
with pytest.raises(dbsafe.DatabaseUnsafe) as exc:
|
||||
dbsafe.snapshot_sqlite(db_path, 10_485_760, timeout=0.3)
|
||||
assert exc.value.reason == "database-locked"
|
||||
finally:
|
||||
holder.execute("ROLLBACK")
|
||||
holder.close()
|
||||
# After the lock clears, the same file snapshots fine.
|
||||
assert _rows_in_snapshot(_read_snapshot_bytes(repo, db_path)) == [("x",)]
|
||||
|
||||
|
||||
def test_pushed_sqlite_bytes_verified(tmp_path):
|
||||
repo = _repo(tmp_path)
|
||||
db_path = str(tmp_path / "push.db")
|
||||
with pytest.raises(Rejected):
|
||||
repo.check_content(dbsafe.SQLITE_MAGIC + b"\x00" * 100 + b"garbage")
|
||||
# Genuine DB bytes pass.
|
||||
conn = _open_db(db_path)
|
||||
conn.execute("INSERT INTO t(v) VALUES ('ok')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
repo.check_content((tmp_path / "push.db").read_bytes())
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Scheduler, quota signal and capacity backstop."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from versiond import maintenance
|
||||
from versiond.api import create_app
|
||||
from versiond.config import Config, Paths
|
||||
from versiond.remote import quota as remote_quota
|
||||
|
||||
from test_service import CONFIG, history, wait_for
|
||||
from test_purge import FakeDAV, REMOTE
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def maint_env(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("VERSIOND_HOME", str(tmp_path / "home"))
|
||||
paths = Paths.resolve()
|
||||
paths.ensure()
|
||||
paths.config_file.write_text(CONFIG + "\n[upload]\nmanifest_interval_seconds = 0.2\n")
|
||||
cfg = Config.load(paths)
|
||||
dav = FakeDAV()
|
||||
project = tmp_path / "proj"
|
||||
project.mkdir()
|
||||
(project / "main.py").write_text("print('hello')\n")
|
||||
app = create_app(cfg, remote_transport=httpx.MockTransport(dav.handler))
|
||||
with TestClient(app, base_url="http://127.0.0.1:9922") as client:
|
||||
client.headers["Authorization"] = f"Bearer {cfg.api_token()}"
|
||||
app.state.services.uploader.offline_sleep = 0.3
|
||||
yield client, dav, project, cfg, app.state.services
|
||||
|
||||
|
||||
def test_quota_signal_and_fallback(maint_env):
|
||||
client, dav, project, cfg, svc = maint_env
|
||||
directory = client.put("/api/v1/config/remote", json=REMOTE).json()["directory"]
|
||||
|
||||
async def get_quota():
|
||||
from versiond.remote import WebDAV
|
||||
settings = svc.uploader.settings()
|
||||
dav_client = WebDAV(settings, transport=svc.uploader.transport)
|
||||
try:
|
||||
return await remote_quota(dav_client, directory)
|
||||
finally:
|
||||
await dav_client.close()
|
||||
|
||||
used, available = asyncio.run(get_quota())
|
||||
assert used is not None and used >= 0 and available == dav.capacity_bytes - used
|
||||
|
||||
async def refresh():
|
||||
return await maintenance.refresh_usage(svc)
|
||||
|
||||
usage = asyncio.run(refresh())
|
||||
assert usage["source"] == "webdav-quota"
|
||||
assert usage["percent"] == round(100 * used / dav.capacity_bytes, 1)
|
||||
assert maintenance.over_limit(svc) is False
|
||||
assert client.get("/health").json()["remote_pressure"] is False
|
||||
|
||||
|
||||
def test_pressure_triggers_pass_but_floor_holds(maint_env):
|
||||
client, dav, project, cfg, svc = maint_env
|
||||
directory = client.put("/api/v1/config/remote", json=REMOTE).json()["directory"]
|
||||
client.post("/api/v1/roots", json={"path": str(project)})
|
||||
wait_for(lambda: history(client, project / "main.py"), timeout=10)
|
||||
# Tiny remote: everything ever uploaded already exceeds 70%.
|
||||
dav.capacity_bytes = 1
|
||||
|
||||
result = asyncio.run(maintenance.run_scheduled_pass(svc, "test-pressure"))
|
||||
assert result["pressure"] is True
|
||||
assert result["versions_deleted"] == 0 # all young: the 5-day floor holds
|
||||
assert len(history(client, project / "main.py")) == 1
|
||||
assert client.get("/health").json()["remote_pressure"] is True
|
||||
assert "remote-over-capacity-limit" in client.get("/health").json()["degraded_roots"]
|
||||
upload = client.get("/api/v1/progress").json()["upload"]
|
||||
assert upload["usage"]["source"] == "webdav-quota"
|
||||
assert upload["usage"]["percent"] >= 70
|
||||
|
||||
|
||||
def test_scheduled_pass_cleans_old_and_reports(maint_env):
|
||||
import sqlite3
|
||||
client, dav, project, cfg, svc = maint_env
|
||||
client.put("/api/v1/config/remote", json=REMOTE)
|
||||
client.post("/api/v1/roots", json={"path": str(project)})
|
||||
wait_for(lambda: history(client, project / "main.py"), timeout=10)
|
||||
(project / "main.py").write_text("v2\n")
|
||||
wait_for(lambda: len(history(client, project / "main.py")) == 2, timeout=10)
|
||||
import time as _t
|
||||
_t.sleep(0.5)
|
||||
(project / "main.py").write_text("v3\n")
|
||||
wait_for(lambda: len(history(client, project / "main.py")) == 3, timeout=10)
|
||||
old = time.time() - 10 * 86400
|
||||
conn = sqlite3.connect(cfg.paths.index_file, timeout=5.0)
|
||||
vids = [v["id"] for v in reversed(history(client, project / "main.py"))]
|
||||
for vid in vids[:2]: # two oldest share one old day: keep daily + latest
|
||||
conn.execute("UPDATE versions SET captured_at = ? WHERE id = ?", (old, vid))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
result = asyncio.run(maintenance.run_scheduled_pass(svc, "test-scheduled"))
|
||||
assert result["versions_deleted"] == 1
|
||||
assert len(history(client, project / "main.py")) == 2
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Purge-remote: date-guarded, async, remote-only deletion of this system's backup."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from versiond.api import create_app
|
||||
from versiond.config import Config, Paths
|
||||
|
||||
from test_service import CONFIG, history, wait_for
|
||||
|
||||
|
||||
class FakeDAV:
|
||||
"""In-memory WebDAV with PROPFIND Depth:0/1 (XML) and recursive DELETE."""
|
||||
|
||||
def __init__(self, user="u1", password="secret"):
|
||||
self.auth = "Basic " + base64.b64encode(f"{user}:{password}".encode()).decode()
|
||||
self.dirs = {"/"}
|
||||
self.files: dict[str, bytes] = {}
|
||||
self.down = False
|
||||
self.capacity_bytes = 10 ** 12 # tiny servers set this low to simulate pressure
|
||||
|
||||
def _usage(self, path):
|
||||
used = sum(len(v) for k, v in self.files.items() if k == path or k.startswith(path + "/"))
|
||||
return used, max(0, self.capacity_bytes - used)
|
||||
|
||||
@staticmethod
|
||||
def parent(path):
|
||||
return path.rsplit("/", 1)[0] or "/"
|
||||
|
||||
def _children(self, path):
|
||||
kids = []
|
||||
for d in sorted(self.dirs):
|
||||
if d != path and self.parent(d) == path:
|
||||
kids.append((d, True))
|
||||
for f in sorted(self.files):
|
||||
if self.parent(f) == path:
|
||||
kids.append((f, False))
|
||||
return kids
|
||||
|
||||
def _multistatus(self, path):
|
||||
parts = ['<?xml version="1.0"?><d:multistatus xmlns:d="DAV:">']
|
||||
parts.append(f"<d:response><d:href>{path}</d:href><d:propstat><d:prop>"
|
||||
"<d:resourcetype><d:collection/></d:resourcetype>"
|
||||
"</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>")
|
||||
for child, is_dir in self._children(path):
|
||||
rtype = "<d:resourcetype><d:collection/></d:resourcetype>" if is_dir else "<d:resourcetype/>"
|
||||
size = "" if is_dir else f"<d:getcontentlength>{len(self.files[child])}</d:getcontentlength>"
|
||||
parts.append(f"<d:response><d:href>{child}</d:href><d:propstat><d:prop>"
|
||||
f"{rtype}{size}</d:prop><d:status>HTTP/1.1 200 OK</d:status>"
|
||||
"</d:propstat></d:response>")
|
||||
parts.append("</d:multistatus>")
|
||||
return "".join(parts).encode()
|
||||
|
||||
def handler(self, request: httpx.Request) -> httpx.Response:
|
||||
if self.down:
|
||||
raise httpx.ConnectError("connection refused", request=request)
|
||||
if request.headers.get("authorization") != self.auth:
|
||||
return httpx.Response(401)
|
||||
path = request.url.path.rstrip("/") or "/"
|
||||
method = request.method
|
||||
if method == "PROPFIND":
|
||||
if path in self.files:
|
||||
body = (f'<?xml version="1.0"?><d:multistatus xmlns:d="DAV:">'
|
||||
f"<d:response><d:href>{path}</d:href><d:propstat><d:prop>"
|
||||
f"<d:resourcetype/><d:getcontentlength>{len(self.files[path])}</d:getcontentlength>"
|
||||
"</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>"
|
||||
"</d:multistatus>").encode()
|
||||
return httpx.Response(207, content=body)
|
||||
if path not in self.dirs:
|
||||
return httpx.Response(404)
|
||||
if request.headers.get("depth", "0") == "1":
|
||||
return httpx.Response(207, content=self._multistatus(path),
|
||||
headers={"Content-Type": "application/xml"})
|
||||
used, available = self._usage(path)
|
||||
body = (f'<?xml version="1.0"?><d:multistatus xmlns:d="DAV:">'
|
||||
f"<d:response><d:href>{path}</d:href><d:propstat><d:prop>"
|
||||
f"<d:quota-used-bytes>{used}</d:quota-used-bytes>"
|
||||
f"<d:quota-available-bytes>{available}</d:quota-available-bytes>"
|
||||
"</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>"
|
||||
"</d:multistatus>").encode()
|
||||
return httpx.Response(207, content=body)
|
||||
if method == "MKCOL":
|
||||
if path in self.dirs:
|
||||
return httpx.Response(405)
|
||||
if self.parent(path) not in self.dirs:
|
||||
return httpx.Response(409)
|
||||
self.dirs.add(path)
|
||||
return httpx.Response(201)
|
||||
if method == "PUT":
|
||||
if self.parent(path) not in self.dirs:
|
||||
return httpx.Response(409)
|
||||
if request.headers.get("if-none-match") == "*" and path in self.files:
|
||||
return httpx.Response(412)
|
||||
self.files[path] = request.read()
|
||||
return httpx.Response(201)
|
||||
if method == "GET":
|
||||
return httpx.Response(200, content=self.files[path]) if path in self.files else httpx.Response(404)
|
||||
if method == "DELETE":
|
||||
for f in [f for f in self.files if f == path or f.startswith(path + "/")]:
|
||||
del self.files[f]
|
||||
for d in [d for d in self.dirs if d != "/" and (d == path or d.startswith(path + "/"))]:
|
||||
self.dirs.discard(d)
|
||||
return httpx.Response(204)
|
||||
return httpx.Response(405)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def purge_env(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("VERSIOND_HOME", str(tmp_path / "home"))
|
||||
paths = Paths.resolve()
|
||||
paths.ensure()
|
||||
paths.config_file.write_text(CONFIG + "\n[upload]\nmanifest_interval_seconds = 0.2\n")
|
||||
cfg = Config.load(paths)
|
||||
dav = FakeDAV()
|
||||
project = tmp_path / "proj"
|
||||
project.mkdir()
|
||||
(project / "main.py").write_text("print('hello')\n")
|
||||
app = create_app(cfg, remote_transport=httpx.MockTransport(dav.handler))
|
||||
with TestClient(app, base_url="http://127.0.0.1:9922") as client:
|
||||
client.headers["Authorization"] = f"Bearer {cfg.api_token()}"
|
||||
app.state.services.uploader.offline_sleep = 0.3
|
||||
yield client, dav, project
|
||||
|
||||
|
||||
REMOTE = {"url": "https://dav.example", "username": "u1", "password": "secret", "base_path": "/versioned/"}
|
||||
|
||||
|
||||
def _configure_and_fill(client, project):
|
||||
r = client.put("/api/v1/config/remote", json=REMOTE)
|
||||
assert r.status_code == 200, r.text
|
||||
directory = r.json()["directory"]
|
||||
client.post("/api/v1/roots", json={"path": str(project)})
|
||||
wait_for(lambda: (lambda p: p["upload"]["versions_local"] == 0
|
||||
and p["upload"]["versions_durable"] >= 1)(
|
||||
client.get("/api/v1/progress").json()), timeout=10)
|
||||
return directory
|
||||
|
||||
|
||||
def test_purge_rejects_bad_date(purge_env):
|
||||
client, _, project = purge_env
|
||||
_configure_and_fill(client, project)
|
||||
r = client.post("/api/v1/admin/purge-remote", json={"today": "01-01-2000"})
|
||||
assert r.status_code == 400, r.text
|
||||
r = client.post("/api/v1/admin/purge-remote", json={"today": "today"})
|
||||
assert r.status_code == 400, r.text
|
||||
|
||||
|
||||
def test_purge_unconfigured_is_409(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("VERSIOND_HOME", str(tmp_path / "home2"))
|
||||
paths = Paths.resolve()
|
||||
paths.ensure()
|
||||
paths.config_file.write_text(CONFIG)
|
||||
cfg = Config.load(paths)
|
||||
app = create_app(cfg)
|
||||
with TestClient(app, base_url="http://127.0.0.1:9922") as client:
|
||||
client.headers["Authorization"] = f"Bearer {cfg.api_token()}"
|
||||
today = datetime.now().strftime("%d-%m-%Y")
|
||||
r = client.post("/api/v1/admin/purge-remote", json={"today": today})
|
||||
assert r.status_code == 409, r.text
|
||||
|
||||
|
||||
def test_purge_deletes_only_this_system(purge_env):
|
||||
client, dav, project = purge_env
|
||||
directory = _configure_and_fill(client, project)
|
||||
before = [p for p in dav.files
|
||||
if p.startswith(directory + "/blobs/") or p.startswith(directory + "/manifests/")]
|
||||
assert before, "expected uploaded blobs/manifests before purge"
|
||||
|
||||
today = datetime.now().strftime("%d-%m-%Y")
|
||||
expected_sizes = {p: len(dav.files[p]) for p in before}
|
||||
r = client.post("/api/v1/admin/purge-remote", json={"today": today})
|
||||
assert r.status_code == 202, r.text
|
||||
body = r.json()
|
||||
assert body["directory"] == directory
|
||||
assert body["files"] == len(before) and body["files"] > 0
|
||||
assert body["bytes"] == sum(expected_sizes.values())
|
||||
|
||||
task_id = body["task_id"]
|
||||
|
||||
def done():
|
||||
s = client.get(f"/api/v1/admin/purge-remote/{task_id}").json()
|
||||
return s if s["status"] == "done" else None
|
||||
|
||||
status = wait_for(done, timeout=10)
|
||||
assert status["files_deleted"] == body["files"]
|
||||
assert status["errors"] == []
|
||||
assert [p for p in dav.files if p.startswith(directory + "/blobs/")] == []
|
||||
assert [p for p in dav.files if p.startswith(directory + "/manifests/")] == []
|
||||
# meta (claim) is kept; local history is untouched
|
||||
assert directory + "/meta/owner.json" in dav.files
|
||||
assert history(client, project / "main.py"), "local index must be untouched"
|
||||
assert client.get(f"/api/v1/admin/purge-remote/nope").status_code == 404
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Disaster recovery, retention, GC, adopt, metrics, verified restores."""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from versiond import dbsafe
|
||||
from versiond.api import create_app
|
||||
from versiond.config import Config, Paths
|
||||
from versiond.store import BlobStore
|
||||
|
||||
from test_service import CONFIG, history, wait_for
|
||||
from test_purge import FakeDAV, REMOTE
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rec_env(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("VERSIOND_HOME", str(tmp_path / "home"))
|
||||
paths = Paths.resolve()
|
||||
paths.ensure()
|
||||
paths.config_file.write_text(CONFIG + "\n[upload]\nmanifest_interval_seconds = 0.2\n")
|
||||
cfg = Config.load(paths)
|
||||
dav = FakeDAV()
|
||||
project = tmp_path / "proj"
|
||||
project.mkdir()
|
||||
(project / "main.py").write_text("print('hello')\n")
|
||||
conn = sqlite3.connect(project / "app.db")
|
||||
conn.execute("CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT)")
|
||||
conn.execute("INSERT INTO t(v) VALUES ('one')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
app = create_app(cfg, remote_transport=httpx.MockTransport(dav.handler))
|
||||
with TestClient(app, base_url="http://127.0.0.1:9922") as client:
|
||||
client.headers["Authorization"] = f"Bearer {cfg.api_token()}"
|
||||
app.state.services.uploader.offline_sleep = 0.3
|
||||
yield client, dav, project, cfg
|
||||
|
||||
|
||||
def _configure_and_fill(client, project, files=2):
|
||||
r = client.put("/api/v1/config/remote", json=REMOTE)
|
||||
assert r.status_code == 200, r.text
|
||||
directory = r.json()["directory"]
|
||||
client.post("/api/v1/roots", json={"path": str(project)})
|
||||
wait_for(lambda: (lambda p: p["upload"]["versions_local"] == 0
|
||||
and p["upload"]["versions_durable"] == files)(
|
||||
client.get("/api/v1/progress").json()), timeout=15)
|
||||
return directory
|
||||
|
||||
|
||||
def _db_index(cfg):
|
||||
return sqlite3.connect(cfg.paths.index_file, timeout=5.0)
|
||||
|
||||
|
||||
def test_reindex_rebuilds_lost_index(rec_env):
|
||||
client, dav, project, cfg = rec_env
|
||||
directory = _configure_and_fill(client, project)
|
||||
before = {v["id"]: v for v in history(client, project / "main.py")}
|
||||
assert before
|
||||
|
||||
# Simulate total local loss of metadata (spool stays, like a surviving disk).
|
||||
conn = _db_index(cfg)
|
||||
conn.execute("DELETE FROM versions")
|
||||
conn.execute("DELETE FROM files")
|
||||
conn.execute("DELETE FROM blobs")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
assert history(client, project / "main.py") == []
|
||||
|
||||
r = client.post("/api/v1/admin/reindex")
|
||||
assert r.status_code == 202, r.text
|
||||
task_id = r.json()["task_id"]
|
||||
status = wait_for(lambda: (lambda s: s if s["status"] == "done" else None)(
|
||||
client.get(f"/api/v1/admin/reindex/{task_id}").json()), timeout=15)
|
||||
assert status["versions"] >= 2 and status["files"] == 2
|
||||
|
||||
after = history(client, project / "main.py")
|
||||
assert len(after) == len(before)
|
||||
assert after[0]["durability"] == "durable"
|
||||
assert client.get(f"/api/v1/versions/{after[0]['id']}/content").text == "print('hello')\n"
|
||||
# Restore works end to end from the rebuilt index.
|
||||
(project / "main.py").write_text("garbage\n")
|
||||
vid = after[0]["id"]
|
||||
out = str(project / "restored.py")
|
||||
assert client.post(f"/api/v1/versions/{vid}/restore", json={"target_path": out}).status_code == 200
|
||||
assert open(out).read() == "print('hello')\n"
|
||||
|
||||
|
||||
def test_reindex_fetches_blobs_from_remote(rec_env):
|
||||
"""True disaster: index AND spool gone; content streams back on demand."""
|
||||
client, dav, project, cfg = rec_env
|
||||
_configure_and_fill(client, project)
|
||||
conn = _db_index(cfg)
|
||||
conn.execute("DELETE FROM versions")
|
||||
conn.execute("DELETE FROM files")
|
||||
conn.execute("DELETE FROM blobs")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
for p in (cfg.paths.spool_dir).rglob("*"):
|
||||
if p.is_file():
|
||||
p.unlink()
|
||||
task_id = client.post("/api/v1/admin/reindex").json()["task_id"]
|
||||
wait_for(lambda: (lambda s: s if s["status"] == "done" else None)(
|
||||
client.get(f"/api/v1/admin/reindex/{task_id}").json()), timeout=15)
|
||||
after = history(client, project / "main.py")
|
||||
assert after
|
||||
assert client.get(f"/api/v1/versions/{after[0]['id']}/content").text == "print('hello')\n"
|
||||
|
||||
|
||||
def test_adopt_takes_over_directory(rec_env):
|
||||
client, dav, project, cfg = rec_env
|
||||
directory = _configure_and_fill(client, project)
|
||||
# A fresh machine against the same server adopts the existing directory.
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
home2 = Path(tempfile.mkdtemp())
|
||||
(home2 / "config").mkdir()
|
||||
(home2 / "config" / "config.toml").write_text(CONFIG)
|
||||
cfg2 = Config.load(Paths(config_dir=home2 / "config", data_dir=home2 / "data", cache_dir=home2 / "cache"))
|
||||
app2 = create_app(cfg2, remote_transport=httpx.MockTransport(dav.handler))
|
||||
with TestClient(app2, base_url="http://127.0.0.1:9922") as c2:
|
||||
c2.headers["Authorization"] = f"Bearer {cfg2.api_token()}"
|
||||
r = c2.post("/api/v1/config/remote/adopt",
|
||||
json={**REMOTE, "password": "secret", "directory": directory})
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["directory"] == directory and r.json()["adopted"] is True
|
||||
|
||||
|
||||
def test_retention_thins_old_keeps_five_days(rec_env):
|
||||
client, dav, project, cfg = rec_env
|
||||
_configure_and_fill(client, project)
|
||||
for i in range(4):
|
||||
(project / "main.py").write_text(f"v{i}\n")
|
||||
time.sleep(0.5) # outside the 0.2s coalescing window: each is a version
|
||||
wait_for(lambda: len(history(client, project / "main.py")) == 5, timeout=10)
|
||||
now = time.time()
|
||||
conn = _db_index(cfg)
|
||||
# Age 3 versions 10 days; pin the oldest; keep the newest fresh.
|
||||
vids = [v["id"] for v in reversed(history(client, project / "main.py"))]
|
||||
for vid in vids[:3]:
|
||||
conn.execute("UPDATE versions SET captured_at = ? WHERE id = ?", (now - 10 * 86400, vid))
|
||||
conn.execute("UPDATE versions SET pinned = 1 WHERE id = ?", (vids[0],))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
dry = client.post("/api/v1/admin/retention/run", json={"dry_run": True}).json()
|
||||
assert dry["dry_run"] and dry["versions_to_delete"] == 1 # 3 old minus pinned minus daily-kept
|
||||
done = client.post("/api/v1/admin/retention/run", json={"dry_run": False}).json()
|
||||
assert done["versions_deleted"] == 1
|
||||
remaining = history(client, project / "main.py")
|
||||
assert len(remaining) == 4 # latest + pinned + one-per-day + fresh middle
|
||||
assert remaining[0]["pinned"] == 0 # newest still the newest
|
||||
assert any(v["pinned"] == 1 for v in remaining)
|
||||
|
||||
|
||||
def test_gc_reclaims_orphans_local_and_remote(rec_env):
|
||||
client, dav, project, cfg = rec_env
|
||||
directory = _configure_and_fill(client, project)
|
||||
(project / "main.py").write_text("v2\n")
|
||||
wait_for(lambda: len(history(client, project / "main.py")) == 2, timeout=10)
|
||||
conn = _db_index(cfg)
|
||||
conn.execute("DELETE FROM versions WHERE id = (SELECT max(id) FROM versions)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
dry = client.post("/api/v1/admin/gc", json={"dry_run": True}).json()
|
||||
assert dry["blobs"] >= 1
|
||||
done = client.post("/api/v1/admin/gc", json={"dry_run": False}).json()
|
||||
assert done["blobs_removed"] >= 1 and done["errors"] == []
|
||||
assert [p for p in dav.files if p.startswith(directory + "/blobs/")] != [] # referenced stay
|
||||
# Remaining history still restorable.
|
||||
assert client.get(f"/api/v1/versions/{history(client, project / 'main.py')[0]['id']}/content").status_code == 200
|
||||
|
||||
|
||||
def test_metrics_and_db_restore_guard(rec_env):
|
||||
client, dav, project, cfg = rec_env
|
||||
_configure_and_fill(client, project)
|
||||
m = client.get("/api/v1/metrics")
|
||||
assert m.status_code == 200 and "versiond_files_tracked" in m.text
|
||||
|
||||
db_hist = history(client, project / "app.db")
|
||||
assert db_hist
|
||||
vid = db_hist[0]["id"]
|
||||
row = _db_index(cfg).execute(
|
||||
"SELECT blob_sha256 FROM versions WHERE id = ?", (vid,)).fetchone()
|
||||
blob_path = None
|
||||
for cand in BlobStore(cfg.paths.spool_dir)._candidates(row[0]):
|
||||
if cand.exists():
|
||||
blob_path = cand
|
||||
assert blob_path is not None
|
||||
blob_path.write_bytes(dbsafe.SQLITE_MAGIC + b"\x00" * 200) # corrupt the stored bytes
|
||||
r = client.post(f"/api/v1/versions/{vid}/restore",
|
||||
json={"target_path": str(project / "evil.db")})
|
||||
assert r.status_code == 422 and "unrestorable" in r.text
|
||||
@@ -10,8 +10,7 @@ from fastapi.testclient import TestClient
|
||||
|
||||
from versiond.api import create_app
|
||||
from versiond.config import Config, Paths
|
||||
from versiond.crypto import KeyRing
|
||||
from versiond.store import _codec
|
||||
from versiond.store import decompress
|
||||
|
||||
from test_service import CONFIG, history, wait_for
|
||||
|
||||
@@ -100,23 +99,26 @@ def test_backup_to_webdav(remote_env):
|
||||
directory = remote["directory"]
|
||||
assert directory.startswith("/versioned/") and remote["adopted"] is False
|
||||
assert "password" not in remote and remote["password_set"] is True
|
||||
assert remote["encryption"] == "none"
|
||||
owner = json.loads(dav.files[directory + "/meta/owner.json"])
|
||||
assert owner["key_id"] == remote["key_id"]
|
||||
assert "installation_id" in owner
|
||||
assert "secret" not in cfg.paths.config_file.read_text() # password only in credentials
|
||||
|
||||
client.post("/api/v1/roots", json={"path": str(project)})
|
||||
wait_for(lambda: progress(client)["upload"]["versions_local"] == 0
|
||||
and progress(client)["upload"]["versions_durable"] == 2, timeout=10)
|
||||
|
||||
keys = KeyRing.load_or_create(cfg.key_file)
|
||||
blob_paths = [p for p in dav.files if p.startswith(directory + "/blobs/")]
|
||||
contents = {_codec.decompress(keys.decrypt(dav.files[p])) for p in blob_paths}
|
||||
# Unencrypted: remote blobs are plain compressed file contents, named by SHA-256.
|
||||
import re
|
||||
assert blob_paths and all(re.fullmatch(r"[0-9a-f]{64}", p.rsplit("/", 1)[-1]) for p in blob_paths)
|
||||
contents = {decompress(dav.files[p]) for p in blob_paths}
|
||||
assert contents == {b"print('hello')\n", b"SECRET=1\n"}
|
||||
assert all(b"SECRET" not in dav.files[p] for p in blob_paths) # encrypted at rest
|
||||
|
||||
manifests = [p for p in dav.files if p.startswith(directory + "/manifests/")]
|
||||
assert manifests and all(p.endswith(".jsonl.zst") for p in manifests)
|
||||
records = [json.loads(line) for p in manifests
|
||||
for line in _codec.decompress(keys.decrypt(dav.files[p])).decode().splitlines()]
|
||||
for line in decompress(dav.files[p]).decode().splitlines()]
|
||||
assert {r["path"] for r in records if r["type"] == "version"} == {str(project / "main.py"), str(project / ".env")}
|
||||
|
||||
# reconfiguring the same machine adopts its own directory
|
||||
|
||||
@@ -155,7 +155,7 @@ def test_push_snapshots_and_limits(env):
|
||||
assert client.post("/api/v1/snapshots", json=body).json()["status"] == "committed"
|
||||
assert client.post("/api/v1/snapshots", json=body).json()["status"] == "unchanged"
|
||||
|
||||
big = {**body, "content_b64": base64.b64encode(b"x" * 204_801).decode()}
|
||||
big = {**body, "content_b64": base64.b64encode(b"x" * 10_485_761).decode()}
|
||||
r = client.post("/api/v1/snapshots", json=big)
|
||||
assert r.status_code == 413 and r.headers["content-type"] == "application/problem+json"
|
||||
|
||||
|
||||
+13
-4
@@ -10,9 +10,15 @@ from versiond.store import BlobStore
|
||||
|
||||
def test_filters_paths():
|
||||
f = Filters()
|
||||
ok = ["app.py", "src/main.go", ".env", ".env.production", "Makefile", "pkg/package.json", ".gitignore"]
|
||||
ok = ["app.py", "src/main.go", ".env", ".env.production", "Makefile", "pkg/package.json", ".gitignore",
|
||||
# user data: images, archives, databases, documents (backed up as raw files)
|
||||
"photos/vacation.jpg", "img/logo.png", "backup/site.tar.gz", "data/archive.zip",
|
||||
"app.db", "app.sqlite3", "report.pdf", "song.mp3", "clip.mp4"]
|
||||
for p in ok:
|
||||
assert f.path_reason(PurePath(p)) is None, p
|
||||
journals = ["app.db-wal", "app.db-shm", "app.sqlite-journal", "data/x-wal"]
|
||||
for p in journals:
|
||||
assert f.path_reason(PurePath(p)) == "database-journal", p
|
||||
rejected = {
|
||||
"node_modules/x/index.js": "ignored-directory",
|
||||
".git/config": "ignored-directory",
|
||||
@@ -21,6 +27,10 @@ def test_filters_paths():
|
||||
".bashrc_local": "hidden-file",
|
||||
"notes.txt~": "temporary-file",
|
||||
"main.py.swp": "ignored-extension",
|
||||
"draft.temp": "ignored-extension",
|
||||
"x.kate-swp": "ignored-extension",
|
||||
"~$lock.docx": "temporary-file",
|
||||
"#autosave#": "temporary-file",
|
||||
"venv/lib/x.py": "ignored-directory",
|
||||
"target/debug/build.rs": "ignored-directory",
|
||||
"4913": "temporary-file",
|
||||
@@ -28,9 +38,8 @@ def test_filters_paths():
|
||||
}
|
||||
for p, reason in rejected.items():
|
||||
assert f.path_reason(PurePath(p)) == reason, p
|
||||
assert f.size_reason(204_800) is None
|
||||
assert f.size_reason(204_801) == "file-too-large"
|
||||
assert f.content_reason(b"abc\x00def") == "binary-content"
|
||||
assert f.size_reason(10_485_760) is None
|
||||
assert f.size_reason(10_485_761) == "file-too-large"
|
||||
|
||||
|
||||
def test_filter_patterns():
|
||||
|
||||
Reference in New Issue
Block a user