DevPlace CI / test (push) Failing after 1h5m29s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
136 lines
4.8 KiB
Python
136 lines
4.8 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import asyncio
|
|
import json
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
from devplacepy import config
|
|
from devplacepy.services.backup import store
|
|
|
|
|
|
async def _run_rclone(*args: str, timeout: float = 1800.0) -> tuple[int, str, str]:
|
|
proc = await asyncio.create_subprocess_exec(
|
|
config.RCLONE_BIN,
|
|
"--config",
|
|
config.RCLONE_CONFIG_FILE,
|
|
*args,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
try:
|
|
out, err = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
|
except asyncio.TimeoutError:
|
|
proc.kill()
|
|
await proc.wait()
|
|
return 124, "", "rclone timed out"
|
|
return proc.returncode, out.decode("utf-8", "replace"), err.decode("utf-8", "replace")
|
|
|
|
|
|
def _remote_dir(target: str) -> str:
|
|
return f"{config.BACKUP_OFFLOAD_REMOTE}/{target}"
|
|
|
|
|
|
def _is_auth_error(err: str) -> bool:
|
|
text = err.lower()
|
|
return (
|
|
"401" in text
|
|
or "unauthorized" in text
|
|
or "didn't find section" in text
|
|
or "did not find section" in text
|
|
)
|
|
|
|
|
|
async def upload_pending(log=lambda message: None) -> int:
|
|
uploaded = 0
|
|
for row in store.list_pending_offload():
|
|
local_path = Path(row["local_path"])
|
|
remote_path = f"{_remote_dir(row['target'])}/{row['filename']}"
|
|
code, _, err = await _run_rclone("copyto", str(local_path), remote_path)
|
|
if code != 0:
|
|
log(f"Offload failed for {row['filename']}: {err.strip()[:300]}")
|
|
if _is_auth_error(err):
|
|
log(
|
|
"Offload halted: remote storage rejected credentials; "
|
|
"remaining uploads skipped until the next cycle"
|
|
)
|
|
break
|
|
continue
|
|
size_code, size_out, size_err = await _run_rclone("size", remote_path, "--json")
|
|
if size_code != 0:
|
|
log(f"Offload verify failed for {row['filename']}: {size_err.strip()[:300]}")
|
|
continue
|
|
try:
|
|
remote_bytes = json.loads(size_out).get("bytes", -1)
|
|
except (ValueError, TypeError):
|
|
remote_bytes = -1
|
|
expected_bytes = int(row.get("size_bytes") or 0)
|
|
if remote_bytes != expected_bytes:
|
|
log(
|
|
f"Offload size mismatch for {row['filename']} "
|
|
f"(local {expected_bytes}, remote {remote_bytes}), will retry"
|
|
)
|
|
continue
|
|
store.mark_remote_uploaded(row["uid"], remote_path)
|
|
uploaded += 1
|
|
log(f"Offloaded {row['filename']} ({store.human_bytes(expected_bytes)}) to {remote_path}")
|
|
return uploaded
|
|
|
|
|
|
async def enforce_local_retention(keep_local: int, log=lambda message: None) -> int:
|
|
keep_local = max(1, keep_local)
|
|
purged = 0
|
|
for target in store.BACKUP_TARGETS:
|
|
rows = store.list_offloaded_local_by_target(target)
|
|
for row in rows[keep_local:]:
|
|
store.mark_local_purged(row["uid"])
|
|
purged += 1
|
|
log(f"Purged local copy of {row['filename']} (kept remotely at {row['remote_path']})")
|
|
return purged
|
|
|
|
|
|
async def enforce_remote_retention(keep_remote: int, log=lambda message: None) -> int:
|
|
if keep_remote < 1:
|
|
return 0
|
|
removed = 0
|
|
for target in store.BACKUP_TARGETS:
|
|
remote_dir = _remote_dir(target)
|
|
code, out, _ = await _run_rclone("lsjson", remote_dir)
|
|
if code != 0:
|
|
continue
|
|
try:
|
|
entries = json.loads(out)
|
|
except (ValueError, TypeError):
|
|
continue
|
|
files = sorted(
|
|
(entry for entry in entries if not entry.get("IsDir")),
|
|
key=lambda entry: entry.get("Name", ""),
|
|
reverse=True,
|
|
)
|
|
for entry in files[keep_remote:]:
|
|
remote_path = f"{remote_dir}/{entry['Name']}"
|
|
del_code, _, del_err = await _run_rclone("deletefile", remote_path)
|
|
if del_code == 0:
|
|
removed += 1
|
|
log(f"Removed old remote backup {remote_path}")
|
|
else:
|
|
log(f"Could not remove remote backup {remote_path}: {del_err.strip()[:200]}")
|
|
return removed
|
|
|
|
|
|
async def run_cycle(*, keep_local: int, keep_remote: int, log=lambda message: None) -> dict:
|
|
if shutil.which(config.RCLONE_BIN) is None:
|
|
log(
|
|
f"Offload skipped: rclone binary not found ({config.RCLONE_BIN}); "
|
|
"install rclone or disable backup_offload_enabled"
|
|
)
|
|
return {"uploaded": 0, "purged_local": 0, "removed_remote": 0}
|
|
uploaded = await upload_pending(log=log)
|
|
purged_local = await enforce_local_retention(keep_local, log=log)
|
|
removed_remote = await enforce_remote_retention(keep_remote, log=log)
|
|
return {
|
|
"uploaded": uploaded,
|
|
"purged_local": purged_local,
|
|
"removed_remote": removed_remote,
|
|
}
|