retoor
3499f6cda0
- No client-side encryption: plaintext content-addressed remote (SHA-256 names), no backup key, no cryptography dependency (server disk encryption is the trust model). - Filters back user data: images, archives, databases, PDFs accepted; 10 MiB cap; binary sniffing removed; temp names hardened (~$, #..#, .temp). - SQLite zero-error policy: backup-API snapshots + integrity_check, journal folding, locked/corrupt loud skips, verified restores. - Recovery: reindex from manifests, remote adopt, on-demand blob fetch, 5-day retention + thinning, GC, date-guarded remote purge, metrics. - Scheduler with WebDAV quota signal and 70% pressure backstop (floor kept). - 37 tests incl. live-monitor capture safety and DB safety.
199 lines
8.7 KiB
Python
199 lines
8.7 KiB
Python
"""Purge-remote: date-guarded, async, remote-only deletion of this system's backup."""
|
|
|
|
import base64
|
|
import json
|
|
import time
|
|
from datetime import datetime
|
|
|
|
import httpx
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from versiond.api import create_app
|
|
from versiond.config import Config, Paths
|
|
|
|
from test_service import CONFIG, history, wait_for
|
|
|
|
|
|
class FakeDAV:
|
|
"""In-memory WebDAV with PROPFIND Depth:0/1 (XML) and recursive DELETE."""
|
|
|
|
def __init__(self, user="u1", password="secret"):
|
|
self.auth = "Basic " + base64.b64encode(f"{user}:{password}".encode()).decode()
|
|
self.dirs = {"/"}
|
|
self.files: dict[str, bytes] = {}
|
|
self.down = False
|
|
self.capacity_bytes = 10 ** 12 # tiny servers set this low to simulate pressure
|
|
|
|
def _usage(self, path):
|
|
used = sum(len(v) for k, v in self.files.items() if k == path or k.startswith(path + "/"))
|
|
return used, max(0, self.capacity_bytes - used)
|
|
|
|
@staticmethod
|
|
def parent(path):
|
|
return path.rsplit("/", 1)[0] or "/"
|
|
|
|
def _children(self, path):
|
|
kids = []
|
|
for d in sorted(self.dirs):
|
|
if d != path and self.parent(d) == path:
|
|
kids.append((d, True))
|
|
for f in sorted(self.files):
|
|
if self.parent(f) == path:
|
|
kids.append((f, False))
|
|
return kids
|
|
|
|
def _multistatus(self, path):
|
|
parts = ['<?xml version="1.0"?><d:multistatus xmlns:d="DAV:">']
|
|
parts.append(f"<d:response><d:href>{path}</d:href><d:propstat><d:prop>"
|
|
"<d:resourcetype><d:collection/></d:resourcetype>"
|
|
"</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>")
|
|
for child, is_dir in self._children(path):
|
|
rtype = "<d:resourcetype><d:collection/></d:resourcetype>" if is_dir else "<d:resourcetype/>"
|
|
size = "" if is_dir else f"<d:getcontentlength>{len(self.files[child])}</d:getcontentlength>"
|
|
parts.append(f"<d:response><d:href>{child}</d:href><d:propstat><d:prop>"
|
|
f"{rtype}{size}</d:prop><d:status>HTTP/1.1 200 OK</d:status>"
|
|
"</d:propstat></d:response>")
|
|
parts.append("</d:multistatus>")
|
|
return "".join(parts).encode()
|
|
|
|
def handler(self, request: httpx.Request) -> httpx.Response:
|
|
if self.down:
|
|
raise httpx.ConnectError("connection refused", request=request)
|
|
if request.headers.get("authorization") != self.auth:
|
|
return httpx.Response(401)
|
|
path = request.url.path.rstrip("/") or "/"
|
|
method = request.method
|
|
if method == "PROPFIND":
|
|
if path in self.files:
|
|
body = (f'<?xml version="1.0"?><d:multistatus xmlns:d="DAV:">'
|
|
f"<d:response><d:href>{path}</d:href><d:propstat><d:prop>"
|
|
f"<d:resourcetype/><d:getcontentlength>{len(self.files[path])}</d:getcontentlength>"
|
|
"</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>"
|
|
"</d:multistatus>").encode()
|
|
return httpx.Response(207, content=body)
|
|
if path not in self.dirs:
|
|
return httpx.Response(404)
|
|
if request.headers.get("depth", "0") == "1":
|
|
return httpx.Response(207, content=self._multistatus(path),
|
|
headers={"Content-Type": "application/xml"})
|
|
used, available = self._usage(path)
|
|
body = (f'<?xml version="1.0"?><d:multistatus xmlns:d="DAV:">'
|
|
f"<d:response><d:href>{path}</d:href><d:propstat><d:prop>"
|
|
f"<d:quota-used-bytes>{used}</d:quota-used-bytes>"
|
|
f"<d:quota-available-bytes>{available}</d:quota-available-bytes>"
|
|
"</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>"
|
|
"</d:multistatus>").encode()
|
|
return httpx.Response(207, content=body)
|
|
if method == "MKCOL":
|
|
if path in self.dirs:
|
|
return httpx.Response(405)
|
|
if self.parent(path) not in self.dirs:
|
|
return httpx.Response(409)
|
|
self.dirs.add(path)
|
|
return httpx.Response(201)
|
|
if method == "PUT":
|
|
if self.parent(path) not in self.dirs:
|
|
return httpx.Response(409)
|
|
if request.headers.get("if-none-match") == "*" and path in self.files:
|
|
return httpx.Response(412)
|
|
self.files[path] = request.read()
|
|
return httpx.Response(201)
|
|
if method == "GET":
|
|
return httpx.Response(200, content=self.files[path]) if path in self.files else httpx.Response(404)
|
|
if method == "DELETE":
|
|
for f in [f for f in self.files if f == path or f.startswith(path + "/")]:
|
|
del self.files[f]
|
|
for d in [d for d in self.dirs if d != "/" and (d == path or d.startswith(path + "/"))]:
|
|
self.dirs.discard(d)
|
|
return httpx.Response(204)
|
|
return httpx.Response(405)
|
|
|
|
|
|
@pytest.fixture
|
|
def purge_env(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("VERSIOND_HOME", str(tmp_path / "home"))
|
|
paths = Paths.resolve()
|
|
paths.ensure()
|
|
paths.config_file.write_text(CONFIG + "\n[upload]\nmanifest_interval_seconds = 0.2\n")
|
|
cfg = Config.load(paths)
|
|
dav = FakeDAV()
|
|
project = tmp_path / "proj"
|
|
project.mkdir()
|
|
(project / "main.py").write_text("print('hello')\n")
|
|
app = create_app(cfg, remote_transport=httpx.MockTransport(dav.handler))
|
|
with TestClient(app, base_url="http://127.0.0.1:9922") as client:
|
|
client.headers["Authorization"] = f"Bearer {cfg.api_token()}"
|
|
app.state.services.uploader.offline_sleep = 0.3
|
|
yield client, dav, project
|
|
|
|
|
|
REMOTE = {"url": "https://dav.example", "username": "u1", "password": "secret", "base_path": "/versioned/"}
|
|
|
|
|
|
def _configure_and_fill(client, project):
|
|
r = client.put("/api/v1/config/remote", json=REMOTE)
|
|
assert r.status_code == 200, r.text
|
|
directory = r.json()["directory"]
|
|
client.post("/api/v1/roots", json={"path": str(project)})
|
|
wait_for(lambda: (lambda p: p["upload"]["versions_local"] == 0
|
|
and p["upload"]["versions_durable"] >= 1)(
|
|
client.get("/api/v1/progress").json()), timeout=10)
|
|
return directory
|
|
|
|
|
|
def test_purge_rejects_bad_date(purge_env):
|
|
client, _, project = purge_env
|
|
_configure_and_fill(client, project)
|
|
r = client.post("/api/v1/admin/purge-remote", json={"today": "01-01-2000"})
|
|
assert r.status_code == 400, r.text
|
|
r = client.post("/api/v1/admin/purge-remote", json={"today": "today"})
|
|
assert r.status_code == 400, r.text
|
|
|
|
|
|
def test_purge_unconfigured_is_409(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("VERSIOND_HOME", str(tmp_path / "home2"))
|
|
paths = Paths.resolve()
|
|
paths.ensure()
|
|
paths.config_file.write_text(CONFIG)
|
|
cfg = Config.load(paths)
|
|
app = create_app(cfg)
|
|
with TestClient(app, base_url="http://127.0.0.1:9922") as client:
|
|
client.headers["Authorization"] = f"Bearer {cfg.api_token()}"
|
|
today = datetime.now().strftime("%d-%m-%Y")
|
|
r = client.post("/api/v1/admin/purge-remote", json={"today": today})
|
|
assert r.status_code == 409, r.text
|
|
|
|
|
|
def test_purge_deletes_only_this_system(purge_env):
|
|
client, dav, project = purge_env
|
|
directory = _configure_and_fill(client, project)
|
|
before = [p for p in dav.files
|
|
if p.startswith(directory + "/blobs/") or p.startswith(directory + "/manifests/")]
|
|
assert before, "expected uploaded blobs/manifests before purge"
|
|
|
|
today = datetime.now().strftime("%d-%m-%Y")
|
|
expected_sizes = {p: len(dav.files[p]) for p in before}
|
|
r = client.post("/api/v1/admin/purge-remote", json={"today": today})
|
|
assert r.status_code == 202, r.text
|
|
body = r.json()
|
|
assert body["directory"] == directory
|
|
assert body["files"] == len(before) and body["files"] > 0
|
|
assert body["bytes"] == sum(expected_sizes.values())
|
|
|
|
task_id = body["task_id"]
|
|
|
|
def done():
|
|
s = client.get(f"/api/v1/admin/purge-remote/{task_id}").json()
|
|
return s if s["status"] == "done" else None
|
|
|
|
status = wait_for(done, timeout=10)
|
|
assert status["files_deleted"] == body["files"]
|
|
assert status["errors"] == []
|
|
assert [p for p in dav.files if p.startswith(directory + "/blobs/")] == []
|
|
assert [p for p in dav.files if p.startswith(directory + "/manifests/")] == []
|
|
# meta (claim) is kept; local history is untouched
|
|
assert directory + "/meta/owner.json" in dav.files
|
|
assert history(client, project / "main.py"), "local index must be untouched"
|
|
assert client.get(f"/api/v1/admin/purge-remote/nope").status_code == 404
|