Remove encryption; user-data filters; SQLite safety; recovery, retention, scheduler
- No client-side encryption: plaintext content-addressed remote (SHA-256 names), no backup key, no cryptography dependency (server disk encryption is the trust model). - Filters back user data: images, archives, databases, PDFs accepted; 10 MiB cap; binary sniffing removed; temp names hardened (~$, #..#, .temp). - SQLite zero-error policy: backup-API snapshots + integrity_check, journal folding, locked/corrupt loud skips, verified restores. - Recovery: reindex from manifests, remote adopt, on-demand blob fetch, 5-day retention + thinning, GC, date-guarded remote purge, metrics. - Scheduler with WebDAV quota signal and 70% pressure backstop (floor kept). - 37 tests incl. live-monitor capture safety and DB safety.
This commit is contained in:
retoor
+135
-13
@@ -10,13 +10,15 @@ import re
|
||||
import secrets
|
||||
import socket
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
INSTALL_APP_ID = b"versiond:4c1b9e0f7a2d4e8b"
|
||||
FORMAT_VERSION = 1
|
||||
FORMAT_VERSION = 2
|
||||
|
||||
|
||||
class RemoteError(Exception):
|
||||
@@ -53,8 +55,12 @@ def installation_id() -> str:
|
||||
machine_id = ""
|
||||
if not machine_id:
|
||||
return secrets.token_hex(8)
|
||||
try:
|
||||
key = bytes.fromhex(machine_id)
|
||||
except ValueError:
|
||||
return secrets.token_hex(8)
|
||||
msg = INSTALL_APP_ID + b":" + str(os.getuid()).encode()
|
||||
return hmac.new(bytes.fromhex(machine_id), msg, hashlib.sha256).hexdigest()[:16]
|
||||
return hmac.new(key, msg, hashlib.sha256).hexdigest()[:16]
|
||||
|
||||
|
||||
def hostname_slug() -> str:
|
||||
@@ -155,11 +161,13 @@ async def check_connection(dav: WebDAV) -> None:
|
||||
await dav.delete(probe)
|
||||
|
||||
|
||||
async def claim_directory(dav: WebDAV, key_id: str) -> dict[str, Any]:
|
||||
async def claim_directory(dav: WebDAV) -> dict[str, Any]:
|
||||
"""Pick and claim this installation's unique directory below base_path.
|
||||
|
||||
Returns {"directory": ..., "adopted": bool}. Never takes over a directory that
|
||||
belongs to a different installation id.
|
||||
belongs to a different installation id. Remote data is stored unencrypted
|
||||
(relying on server disk encryption); blobs are plain content-addressed
|
||||
objects named by their SHA-256.
|
||||
"""
|
||||
base = join(dav.settings.base_path)
|
||||
await dav.makedirs(base)
|
||||
@@ -177,12 +185,11 @@ async def claim_directory(dav: WebDAV, key_id: str) -> dict[str, Any]:
|
||||
"hostname": socket.gethostname(),
|
||||
"user": os.environ.get("USER", ""),
|
||||
"created_at": time.time(),
|
||||
"key_id": key_id,
|
||||
}
|
||||
if not await dav.put(owner_path, json.dumps(owner, indent=2).encode(), if_none_match=True):
|
||||
continue # someone created it between our GET and PUT; re-read
|
||||
fmt = {"format": FORMAT_VERSION, "encryption": "aes-256-gcm", "compression": "zstd",
|
||||
"blob_layout": "blobs/<hmac[:2]>/<hmac>", "key_id": key_id}
|
||||
fmt = {"format": FORMAT_VERSION, "encryption": "none", "compression": "zstd-or-zlib",
|
||||
"blob_layout": "blobs/<sha256[:2]>/<sha256>"}
|
||||
await dav.put(join(directory, "meta", "format.json"), json.dumps(fmt, indent=2).encode())
|
||||
return {"directory": directory, "adopted": False}
|
||||
try:
|
||||
@@ -190,12 +197,127 @@ async def claim_directory(dav: WebDAV, key_id: str) -> dict[str, Any]:
|
||||
except ValueError:
|
||||
owner = {}
|
||||
if owner.get("installation_id") == install_id:
|
||||
if owner.get("key_id") not in (None, key_id):
|
||||
raise RemoteError(
|
||||
"key-mismatch",
|
||||
f"{directory} was written with a different backup key; restore that key "
|
||||
"(~/.config/versiond/backup.key) or remove the directory",
|
||||
)
|
||||
return {"directory": directory, "adopted": True}
|
||||
name = f"{hostname_slug()}-{install_id[:8]}-{secrets.token_hex(2)}"
|
||||
raise RemoteError("claim-failed", "could not claim a unique remote directory")
|
||||
|
||||
|
||||
# remote listing (read-only PROPFIND) for the purge endpoint
|
||||
|
||||
DAV_NS = "DAV:"
|
||||
_PROPFIND_BODY = (
|
||||
'<?xml version="1.0" encoding="utf-8"?>'
|
||||
'<d:propfind xmlns:d="DAV:"><d:prop>'
|
||||
"<d:resourcetype/><d:getcontentlength/>"
|
||||
"</d:prop></d:propfind>"
|
||||
).encode()
|
||||
|
||||
_QUOTA_BODY = (
|
||||
'<?xml version="1.0" encoding="utf-8"?>'
|
||||
'<d:propfind xmlns:d="DAV:"><d:prop>'
|
||||
"<d:quota-used-bytes/><d:quota-available-bytes/>"
|
||||
"</d:prop></d:propfind>"
|
||||
).encode()
|
||||
|
||||
|
||||
@dataclass
|
||||
class RemoteEntry:
|
||||
path: str
|
||||
is_dir: bool
|
||||
size: int | None = None
|
||||
|
||||
|
||||
def _href_to_path(href: str) -> str:
|
||||
path = unquote(urlparse(href).path or href)
|
||||
return path.rstrip("/") or "/"
|
||||
|
||||
|
||||
async def list_dir(dav: "WebDAV", path: str) -> list[RemoteEntry]:
|
||||
"""Immediate children of a collection via PROPFIND Depth:1 (read-only)."""
|
||||
resp = await dav.request(
|
||||
"PROPFIND", path,
|
||||
headers={"Depth": "1", "Content-Type": "application/xml"},
|
||||
content=_PROPFIND_BODY,
|
||||
)
|
||||
if resp.status_code == 404:
|
||||
return []
|
||||
WebDAV._check(resp, 207)
|
||||
try:
|
||||
root = ET.fromstring(resp.content)
|
||||
except ET.ParseError as exc:
|
||||
raise RemoteError("remote-http", f"PROPFIND {path} returned unparseable XML") from exc
|
||||
entries: list[RemoteEntry] = []
|
||||
for response in root.findall(f"{{{DAV_NS}}}response"):
|
||||
href = response.findtext(f"{{{DAV_NS}}}href")
|
||||
if not href:
|
||||
continue
|
||||
child = _href_to_path(href)
|
||||
if child == path.rstrip("/") or child == path:
|
||||
continue # the collection itself, not a child
|
||||
is_dir = False
|
||||
size: int | None = None
|
||||
for propstat in response.findall(f"{{{DAV_NS}}}propstat"):
|
||||
prop = propstat.find(f"{{{DAV_NS}}}prop")
|
||||
if prop is None:
|
||||
continue
|
||||
restype = prop.find(f"{{{DAV_NS}}}resourcetype")
|
||||
if restype is not None and restype.find(f"{{{DAV_NS}}}collection") is not None:
|
||||
is_dir = True
|
||||
length = prop.findtext(f"{{{DAV_NS}}}getcontentlength")
|
||||
if length is not None:
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError:
|
||||
size = None
|
||||
entries.append(RemoteEntry(child, is_dir, size))
|
||||
return entries
|
||||
|
||||
|
||||
async def list_tree(dav: "WebDAV", top: str) -> tuple[list[RemoteEntry], list[RemoteEntry]]:
|
||||
"""All files and directories strictly below top. Returns (files, dirs)."""
|
||||
files: list[RemoteEntry] = []
|
||||
dirs: list[RemoteEntry] = []
|
||||
stack = [top]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
for entry in await list_dir(dav, current):
|
||||
if entry.is_dir:
|
||||
dirs.append(entry)
|
||||
stack.append(entry.path)
|
||||
else:
|
||||
files.append(entry)
|
||||
return files, dirs
|
||||
|
||||
|
||||
async def quota(dav: "WebDAV", path: str) -> tuple[int | None, int | None]:
|
||||
"""RFC 4331 quota properties. Returns (used_bytes, available_bytes or None).
|
||||
|
||||
(None, None) when the server exposes no quota information.
|
||||
"""
|
||||
resp = await dav.request(
|
||||
"PROPFIND", path,
|
||||
headers={"Depth": "0", "Content-Type": "application/xml"},
|
||||
content=_QUOTA_BODY,
|
||||
)
|
||||
if resp.status_code == 404:
|
||||
return None, None
|
||||
WebDAV._check(resp, 207)
|
||||
try:
|
||||
root = ET.fromstring(resp.content)
|
||||
except ET.ParseError:
|
||||
return None, None
|
||||
used = available = None
|
||||
for prop in root.findall(f".//{{{DAV_NS}}}prop"):
|
||||
used_text = prop.findtext(f"{{{DAV_NS}}}quota-used-bytes")
|
||||
avail_text = prop.findtext(f"{{{DAV_NS}}}quota-available-bytes")
|
||||
if used_text is not None:
|
||||
try:
|
||||
used = int(used_text)
|
||||
except ValueError:
|
||||
used = None
|
||||
if avail_text is not None:
|
||||
try:
|
||||
available = int(avail_text)
|
||||
except ValueError:
|
||||
available = None
|
||||
return used, available
|
||||
|
||||
Reference in New Issue
Block a user