versioning/tests/test_recovery.py

"""Disaster recovery, retention, GC, adopt, metrics, verified restores."""

import json
import sqlite3
import time
from datetime import datetime

import httpx
import pytest
from fastapi.testclient import TestClient

from versiond import dbsafe
from versiond.api import create_app
from versiond.config import Config, Paths
from versiond.store import BlobStore

from test_service import CONFIG, history, wait_for
from test_purge import FakeDAV, REMOTE


@pytest.fixture
def rec_env(tmp_path, monkeypatch):
    monkeypatch.setenv("VERSIOND_HOME", str(tmp_path / "home"))
    paths = Paths.resolve()
    paths.ensure()
    paths.config_file.write_text(CONFIG + "\n[upload]\nmanifest_interval_seconds = 0.2\n")
    cfg = Config.load(paths)
    dav = FakeDAV()
    project = tmp_path / "proj"
    project.mkdir()
    (project / "main.py").write_text("print('hello')\n")
    conn = sqlite3.connect(project / "app.db")
    conn.execute("CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT)")
    conn.execute("INSERT INTO t(v) VALUES ('one')")
    conn.commit()
    conn.close()
    app = create_app(cfg, remote_transport=httpx.MockTransport(dav.handler))
    with TestClient(app, base_url="http://127.0.0.1:9922") as client:
        client.headers["Authorization"] = f"Bearer {cfg.api_token()}"
        app.state.services.uploader.offline_sleep = 0.3
        yield client, dav, project, cfg


def _configure_and_fill(client, project, files=2):
    r = client.put("/api/v1/config/remote", json=REMOTE)
    assert r.status_code == 200, r.text
    directory = r.json()["directory"]
    client.post("/api/v1/roots", json={"path": str(project)})
    wait_for(lambda: (lambda p: p["upload"]["versions_local"] == 0
                      and p["upload"]["versions_durable"] == files)(
                          client.get("/api/v1/progress").json()), timeout=15)
    return directory


def _db_index(cfg):
    return sqlite3.connect(cfg.paths.index_file, timeout=5.0)


def test_reindex_rebuilds_lost_index(rec_env):
    client, dav, project, cfg = rec_env
    directory = _configure_and_fill(client, project)
    before = {v["id"]: v for v in history(client, project / "main.py")}
    assert before

    # Simulate total local loss of metadata (spool stays, like a surviving disk).
    conn = _db_index(cfg)
    conn.execute("DELETE FROM versions")
    conn.execute("DELETE FROM files")
    conn.execute("DELETE FROM blobs")
    conn.commit()
    conn.close()
    assert history(client, project / "main.py") == []

    r = client.post("/api/v1/admin/reindex")
    assert r.status_code == 202, r.text
    task_id = r.json()["task_id"]
    status = wait_for(lambda: (lambda s: s if s["status"] == "done" else None)(
        client.get(f"/api/v1/admin/reindex/{task_id}").json()), timeout=15)
    assert status["versions"] >= 2 and status["files"] == 2

    after = history(client, project / "main.py")
    assert len(after) == len(before)
    assert after[0]["durability"] == "durable"
    assert client.get(f"/api/v1/versions/{after[0]['id']}/content").text == "print('hello')\n"
    # Restore works end to end from the rebuilt index.
    (project / "main.py").write_text("garbage\n")
    vid = after[0]["id"]
    out = str(project / "restored.py")
    assert client.post(f"/api/v1/versions/{vid}/restore", json={"target_path": out}).status_code == 200
    assert open(out).read() == "print('hello')\n"


def test_reindex_fetches_blobs_from_remote(rec_env):
    """True disaster: index AND spool gone; content streams back on demand."""
    client, dav, project, cfg = rec_env
    _configure_and_fill(client, project)
    conn = _db_index(cfg)
    conn.execute("DELETE FROM versions")
    conn.execute("DELETE FROM files")
    conn.execute("DELETE FROM blobs")
    conn.commit()
    conn.close()
    for p in (cfg.paths.spool_dir).rglob("*"):
        if p.is_file():
            p.unlink()
    task_id = client.post("/api/v1/admin/reindex").json()["task_id"]
    wait_for(lambda: (lambda s: s if s["status"] == "done" else None)(
        client.get(f"/api/v1/admin/reindex/{task_id}").json()), timeout=15)
    after = history(client, project / "main.py")
    assert after
    assert client.get(f"/api/v1/versions/{after[0]['id']}/content").text == "print('hello')\n"


def test_adopt_takes_over_directory(rec_env):
    client, dav, project, cfg = rec_env
    directory = _configure_and_fill(client, project)
    # A fresh machine against the same server adopts the existing directory.
    import tempfile
    from pathlib import Path
    home2 = Path(tempfile.mkdtemp())
    (home2 / "config").mkdir()
    (home2 / "config" / "config.toml").write_text(CONFIG)
    cfg2 = Config.load(Paths(config_dir=home2 / "config", data_dir=home2 / "data", cache_dir=home2 / "cache"))
    app2 = create_app(cfg2, remote_transport=httpx.MockTransport(dav.handler))
    with TestClient(app2, base_url="http://127.0.0.1:9922") as c2:
        c2.headers["Authorization"] = f"Bearer {cfg2.api_token()}"
        r = c2.post("/api/v1/config/remote/adopt",
                    json={**REMOTE, "password": "secret", "directory": directory})
        assert r.status_code == 200, r.text
        assert r.json()["directory"] == directory and r.json()["adopted"] is True


def test_retention_thins_old_keeps_five_days(rec_env):
    client, dav, project, cfg = rec_env
    _configure_and_fill(client, project)
    for i in range(4):
        (project / "main.py").write_text(f"v{i}\n")
        time.sleep(0.5)  # outside the 0.2s coalescing window: each is a version
    wait_for(lambda: len(history(client, project / "main.py")) == 5, timeout=10)
    now = time.time()
    conn = _db_index(cfg)
    # Age 3 versions 10 days; pin the oldest; keep the newest fresh.
    vids = [v["id"] for v in reversed(history(client, project / "main.py"))]
    for vid in vids[:3]:
        conn.execute("UPDATE versions SET captured_at = ? WHERE id = ?", (now - 10 * 86400, vid))
    conn.execute("UPDATE versions SET pinned = 1 WHERE id = ?", (vids[0],))
    conn.commit()
    conn.close()

    dry = client.post("/api/v1/admin/retention/run", json={"dry_run": True}).json()
    assert dry["dry_run"] and dry["versions_to_delete"] == 1  # 3 old minus pinned minus daily-kept
    done = client.post("/api/v1/admin/retention/run", json={"dry_run": False}).json()
    assert done["versions_deleted"] == 1
    assert done["index_reclaimed"]["freelist_after"] == 0  # hard delete, not freelist bloat
    remaining = history(client, project / "main.py")
    assert len(remaining) == 4  # latest + pinned + one-per-day + fresh middle
    assert remaining[0]["pinned"] == 0  # newest still the newest
    assert any(v["pinned"] == 1 for v in remaining)


def test_gc_reclaims_orphans_local_and_remote(rec_env):
    client, dav, project, cfg = rec_env
    directory = _configure_and_fill(client, project)
    (project / "main.py").write_text("v2\n")
    wait_for(lambda: len(history(client, project / "main.py")) == 2, timeout=10)
    conn = _db_index(cfg)
    conn.execute("DELETE FROM versions WHERE id = (SELECT max(id) FROM versions)")
    conn.commit()
    conn.close()
    dry = client.post("/api/v1/admin/gc", json={"dry_run": True}).json()
    assert dry["blobs"] >= 1
    done = client.post("/api/v1/admin/gc", json={"dry_run": False}).json()
    assert done["blobs_removed"] >= 1 and done["errors"] == []
    assert [p for p in dav.files if p.startswith(directory + "/blobs/")] != []  # referenced stay
    # Remaining history still restorable.
    assert client.get(f"/api/v1/versions/{history(client, project / 'main.py')[0]['id']}/content").status_code == 200


def test_metrics_and_db_restore_guard(rec_env):
    client, dav, project, cfg = rec_env
    _configure_and_fill(client, project)
    m = client.get("/api/v1/metrics")
    assert m.status_code == 200 and "versiond_files_tracked" in m.text

    db_hist = history(client, project / "app.db")
    assert db_hist
    vid = db_hist[0]["id"]
    row = _db_index(cfg).execute(
        "SELECT blob_sha256 FROM versions WHERE id = ?", (vid,)).fetchone()
    blob_path = None
    for cand in BlobStore(cfg.paths.spool_dir)._candidates(row[0]):
        if cand.exists():
            blob_path = cand
    assert blob_path is not None
    blob_path.write_bytes(dbsafe.SQLITE_MAGIC + b"\x00" * 200)  # corrupt the stored bytes
    r = client.post(f"/api/v1/versions/{vid}/restore",
                    json={"target_path": str(project / "evil.db")})
    assert r.status_code == 422 and "unrestorable" in r.text