forked from retoor/devplacepy
feat: add backup CLI commands, AI correction/modifier services, and timezone-aware date display
- Add `devplace backups` CLI subcommands (list, run, prune, clear) with job enqueueing and orphan cleanup - Introduce `BACKUPS_DIR` and `BACKUP_STAGING_DIR` config paths for backup storage - Implement `schedule_correction` and `schedule_modification` calls in content creation, comment creation, and comment editing flows - Add `DEFAULT_CORRECTION_PROMPT` and `DEFAULT_MODIFIER_PROMPT` config constants for AI content processing - Document timezone-aware date display using `local_dt`/`dt_ago` Jinja globals with client-side `Intl` localization - Update README with AI content correction/modifier support in direct messages via `@ai` inline instructions - Add `track_action(user["uid"], "vote")` call on upvote in `apply_vote`
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from devplacepy.services.backup.service import BackupService
|
||||
|
||||
__all__ = ["BackupService"]
|
||||
@@ -0,0 +1,238 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy import config
|
||||
from devplacepy.attachments import _directory_for
|
||||
from devplacepy.services.backup import store
|
||||
from devplacepy.services.devii.tasks.schedule import next_run as schedule_next_run
|
||||
from devplacepy.services.devii.tasks.schedule import now_utc, to_iso
|
||||
from devplacepy.services.jobs import queue
|
||||
from devplacepy.services.jobs.base import JobService, _human_bytes
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
WORKER_MODULE = "devplacepy.services.jobs.backup_worker"
|
||||
|
||||
|
||||
class BackupService(JobService):
|
||||
kind = "backup"
|
||||
title = "Backup"
|
||||
description = (
|
||||
"Builds compressed tar.gz backups of selected data targets in a subprocess off the "
|
||||
"request path, snapshots the database consistently, records full statistics, fires "
|
||||
"scheduled backups, and rotates them by retention."
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="backup", interval_seconds=15)
|
||||
|
||||
async def run_once(self) -> None:
|
||||
await super().run_once()
|
||||
try:
|
||||
self._fire_due_schedules()
|
||||
except Exception as exc:
|
||||
self.log(f"Schedule pass failed: {exc}")
|
||||
|
||||
async def process(self, job: dict) -> dict:
|
||||
from devplacepy.services.audit import record as audit
|
||||
|
||||
payload = job["payload"]
|
||||
target = payload.get("target", "")
|
||||
schedule_uid = payload.get("schedule_uid", "")
|
||||
keep_last = int(payload.get("keep_last") or 0)
|
||||
job_uid = job["uid"]
|
||||
record = store.get_backup_by_job(job_uid)
|
||||
if record is None:
|
||||
backup_uid = store.create_backup(
|
||||
target=target,
|
||||
created_by=job.get("owner_id", ""),
|
||||
job_uid=job_uid,
|
||||
schedule_uid=schedule_uid,
|
||||
)
|
||||
record = store.get_backup(backup_uid)
|
||||
backup_uid = record["uid"]
|
||||
|
||||
if not store.is_valid_target(target):
|
||||
store.fail_backup(backup_uid, f"unknown backup target: {target}")
|
||||
raise ValueError(f"unknown backup target: {target}")
|
||||
|
||||
store.mark_running(backup_uid)
|
||||
staging = config.BACKUP_STAGING_DIR / job_uid
|
||||
try:
|
||||
sources = await asyncio.to_thread(self._materialize, target, staging)
|
||||
spec_path = staging / "spec.json"
|
||||
await asyncio.to_thread(
|
||||
spec_path.write_text, json.dumps({"sources": sources})
|
||||
)
|
||||
final_dir = config.BACKUPS_DIR / _directory_for(backup_uid)
|
||||
await asyncio.to_thread(final_dir.mkdir, parents=True, exist_ok=True)
|
||||
filename = self._archive_name(target, backup_uid)
|
||||
tmp_path = final_dir / f"{generate_uid()}.partial"
|
||||
stats = await self._run_worker(spec_path, tmp_path)
|
||||
final_path = final_dir / filename
|
||||
await asyncio.to_thread(tmp_path.replace, final_path)
|
||||
except Exception as exc:
|
||||
store.fail_backup(backup_uid, str(exc) or exc.__class__.__name__)
|
||||
audit.record_system(
|
||||
"job.backup.failed",
|
||||
actor_kind="system",
|
||||
origin="scheduler" if schedule_uid else "web",
|
||||
result="failure",
|
||||
target_type="backup",
|
||||
target_uid=backup_uid,
|
||||
metadata={"target": target, "schedule_uid": schedule_uid},
|
||||
summary=f"backup {target} failed",
|
||||
links=[audit.job(job_uid)],
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
await asyncio.to_thread(shutil.rmtree, staging, ignore_errors=True)
|
||||
|
||||
store.finalize_backup(
|
||||
backup_uid,
|
||||
filename=filename,
|
||||
local_path=str(final_path),
|
||||
stats=stats,
|
||||
)
|
||||
removed = store.rotate_schedule(schedule_uid, keep_last)
|
||||
audit.record_system(
|
||||
"job.backup.complete",
|
||||
actor_kind="system",
|
||||
origin="scheduler" if schedule_uid else "web",
|
||||
target_type="backup",
|
||||
target_uid=backup_uid,
|
||||
metadata={
|
||||
"target": target,
|
||||
"schedule_uid": schedule_uid,
|
||||
"bytes_out": stats["bytes_out"],
|
||||
"sha256": stats["sha256"],
|
||||
"file_count": stats["file_count"],
|
||||
"rotated": removed,
|
||||
},
|
||||
summary=f"backup {target} completed ({_human_bytes(stats['bytes_out'])})",
|
||||
links=[audit.job(job_uid)],
|
||||
)
|
||||
return {
|
||||
"backup_uid": backup_uid,
|
||||
"target": target,
|
||||
"filename": filename,
|
||||
"local_path": str(final_path),
|
||||
"sha256": stats["sha256"],
|
||||
"file_count": stats["file_count"],
|
||||
"dir_count": stats["dir_count"],
|
||||
"bytes_in": stats["bytes_in"],
|
||||
"bytes_out": stats["bytes_out"],
|
||||
"item_count": stats["file_count"],
|
||||
}
|
||||
|
||||
def cleanup(self, job: dict) -> None:
|
||||
shutil.rmtree(config.BACKUP_STAGING_DIR / job["uid"], ignore_errors=True)
|
||||
|
||||
def _materialize(self, target: str, staging: Path) -> list[dict]:
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
sources: list[dict] = []
|
||||
if target in ("database", "full"):
|
||||
snapshot = staging / "db_snapshot"
|
||||
self._snapshot_databases(snapshot)
|
||||
sources.append({"root": "database", "path": str(snapshot)})
|
||||
if target in ("uploads", "full"):
|
||||
sources.append({"root": "uploads", "path": str(config.UPLOADS_DIR)})
|
||||
if target in ("keys", "full"):
|
||||
sources.append({"root": "keys", "path": str(config.KEYS_DIR)})
|
||||
return sources
|
||||
|
||||
def _snapshot_databases(self, dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
database_file = Path(str(config.DATABASE_URL).replace("sqlite:///", ""))
|
||||
for source in (database_file, config.DEVII_TASKS_DB, config.DEVII_LESSONS_DB):
|
||||
if Path(source).exists():
|
||||
self._sqlite_backup(Path(source), dest / Path(source).name)
|
||||
|
||||
def _sqlite_backup(self, source: Path, destination: Path) -> None:
|
||||
origin = sqlite3.connect(f"file:{source}?mode=ro", uri=True)
|
||||
try:
|
||||
target = sqlite3.connect(str(destination))
|
||||
try:
|
||||
origin.backup(target)
|
||||
finally:
|
||||
target.close()
|
||||
finally:
|
||||
origin.close()
|
||||
|
||||
async def _run_worker(self, spec_path: Path, output_path: Path) -> dict:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-m",
|
||||
WORKER_MODULE,
|
||||
str(spec_path),
|
||||
str(output_path),
|
||||
cwd=str(config.BASE_DIR),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
out, err = await proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"backup worker exited {proc.returncode}: "
|
||||
f"{err.decode('utf-8', 'replace')[:500]}"
|
||||
)
|
||||
return json.loads(out.decode("utf-8"))
|
||||
|
||||
def _archive_name(self, target: str, backup_uid: str) -> str:
|
||||
stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
|
||||
tail = backup_uid.replace("-", "")[-8:]
|
||||
return f"{target}-{stamp}-{tail}.tar.gz"
|
||||
|
||||
def _fire_due_schedules(self) -> None:
|
||||
reference = to_iso(now_utc())
|
||||
for schedule in store.list_due_schedules(reference):
|
||||
self._enqueue_scheduled(schedule, reference)
|
||||
|
||||
def _enqueue_scheduled(self, schedule: dict, reference: str) -> None:
|
||||
target = schedule["target"]
|
||||
if not store.is_valid_target(target):
|
||||
self.log(f"Schedule {schedule['uid']} skipped: unknown target {target}")
|
||||
return
|
||||
keep_last = int(schedule.get("keep_last") or 0)
|
||||
created_by = schedule.get("created_by", "")
|
||||
job_uid = queue.enqueue(
|
||||
"backup",
|
||||
{
|
||||
"target": target,
|
||||
"schedule_uid": schedule["uid"],
|
||||
"created_by": created_by,
|
||||
"keep_last": keep_last,
|
||||
},
|
||||
owner_kind="system",
|
||||
owner_id=created_by,
|
||||
preferred_name=f"{schedule.get('name', target)} ({target})",
|
||||
)
|
||||
store.create_backup(
|
||||
target=target,
|
||||
created_by=created_by,
|
||||
job_uid=job_uid,
|
||||
schedule_uid=schedule["uid"],
|
||||
)
|
||||
moment = schedule_next_run(
|
||||
schedule["kind"],
|
||||
int(schedule.get("every_seconds") or 0),
|
||||
schedule.get("cron") or None,
|
||||
now_utc(),
|
||||
)
|
||||
store.set_schedule_runtime(
|
||||
schedule["uid"],
|
||||
next_run_at=to_iso(moment) if moment else "",
|
||||
last_run_at=reference,
|
||||
last_job_uid=job_uid,
|
||||
run_count=int(schedule.get("run_count") or 0) + 1,
|
||||
)
|
||||
self.log(f"Scheduled backup '{schedule.get('name')}' enqueued ({job_uid})")
|
||||
@@ -0,0 +1,437 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import shutil
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy import config
|
||||
from devplacepy.database import _index, db, get_table
|
||||
from devplacepy.utils import generate_uid
|
||||
|
||||
BACKUP_TARGETS: dict[str, dict] = {
|
||||
"database": {
|
||||
"label": "Database",
|
||||
"description": "Consistent snapshot of the SQLite database and the Devii task and lesson databases.",
|
||||
},
|
||||
"uploads": {
|
||||
"label": "Uploads",
|
||||
"description": "All attachments and project files under the uploads directory.",
|
||||
},
|
||||
"keys": {
|
||||
"label": "Keys and config",
|
||||
"description": "VAPID notification keys and other small config artifacts.",
|
||||
},
|
||||
"full": {
|
||||
"label": "Full data directory",
|
||||
"description": "Database, uploads, and keys in one archive, excluding regenerable staging, locks, and caches.",
|
||||
},
|
||||
}
|
||||
|
||||
STATUS_PENDING = "pending"
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_DONE = "done"
|
||||
STATUS_FAILED = "failed"
|
||||
|
||||
STORAGE_CACHE_TTL_SECONDS = 30
|
||||
|
||||
_storage_cache: dict = {"at": 0.0, "data": None}
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def is_valid_target(key: str) -> bool:
|
||||
return key in BACKUP_TARGETS
|
||||
|
||||
|
||||
def target_label(key: str) -> str:
|
||||
return BACKUP_TARGETS.get(key, {}).get("label", key)
|
||||
|
||||
|
||||
def human_bytes(size: int) -> str:
|
||||
value = float(max(0, int(size or 0)))
|
||||
for unit in ("B", "KB", "MB", "GB", "TB"):
|
||||
if value < 1024 or unit == "TB":
|
||||
return f"{value:.0f} {unit}" if unit == "B" else f"{value:.1f} {unit}"
|
||||
value /= 1024
|
||||
return f"{value:.1f} TB"
|
||||
|
||||
|
||||
def ensure_tables() -> None:
|
||||
backups = get_table("backups")
|
||||
for column, example in (
|
||||
("uid", ""),
|
||||
("job_uid", ""),
|
||||
("target", ""),
|
||||
("label", ""),
|
||||
("status", STATUS_PENDING),
|
||||
("filename", ""),
|
||||
("local_path", ""),
|
||||
("size_bytes", 0),
|
||||
("bytes_in", 0),
|
||||
("file_count", 0),
|
||||
("dir_count", 0),
|
||||
("sha256", ""),
|
||||
("schedule_uid", ""),
|
||||
("created_by", ""),
|
||||
("created_at", ""),
|
||||
("completed_at", ""),
|
||||
("error", ""),
|
||||
):
|
||||
if not backups.has_column(column):
|
||||
backups.create_column_by_example(column, example)
|
||||
|
||||
schedules = get_table("backup_schedules")
|
||||
for column, example in (
|
||||
("uid", ""),
|
||||
("name", ""),
|
||||
("target", ""),
|
||||
("kind", "interval"),
|
||||
("every_seconds", 0),
|
||||
("cron", ""),
|
||||
("enabled", 1),
|
||||
("keep_last", 0),
|
||||
("next_run_at", ""),
|
||||
("last_run_at", ""),
|
||||
("last_job_uid", ""),
|
||||
("run_count", 0),
|
||||
("created_by", ""),
|
||||
("created_at", ""),
|
||||
("updated_at", ""),
|
||||
("deleted_at", None),
|
||||
("deleted_by", None),
|
||||
):
|
||||
if not schedules.has_column(column):
|
||||
schedules.create_column_by_example(column, example)
|
||||
|
||||
_index(db, "backups", "idx_backups_status", ["status"])
|
||||
_index(db, "backups", "idx_backups_job", ["job_uid"])
|
||||
_index(db, "backups", "idx_backups_schedule", ["schedule_uid", "created_at"])
|
||||
_index(db, "backup_schedules", "idx_backup_schedules_enabled", ["enabled"])
|
||||
|
||||
|
||||
def create_backup(
|
||||
*, target: str, created_by: str, job_uid: str, schedule_uid: str = ""
|
||||
) -> str:
|
||||
uid = generate_uid()
|
||||
get_table("backups").insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"job_uid": job_uid,
|
||||
"target": target,
|
||||
"label": target_label(target),
|
||||
"status": STATUS_PENDING,
|
||||
"filename": "",
|
||||
"local_path": "",
|
||||
"size_bytes": 0,
|
||||
"bytes_in": 0,
|
||||
"file_count": 0,
|
||||
"dir_count": 0,
|
||||
"sha256": "",
|
||||
"schedule_uid": schedule_uid,
|
||||
"created_by": created_by,
|
||||
"created_at": now_iso(),
|
||||
"completed_at": "",
|
||||
"error": "",
|
||||
}
|
||||
)
|
||||
return uid
|
||||
|
||||
|
||||
def get_backup(uid: str) -> dict | None:
|
||||
if "backups" not in db.tables:
|
||||
return None
|
||||
return get_table("backups").find_one(uid=uid)
|
||||
|
||||
|
||||
def get_backup_by_job(job_uid: str) -> dict | None:
|
||||
if "backups" not in db.tables:
|
||||
return None
|
||||
return get_table("backups").find_one(job_uid=job_uid)
|
||||
|
||||
|
||||
def list_backups(limit: int = 100) -> list[dict]:
|
||||
if "backups" not in db.tables:
|
||||
return []
|
||||
return list(get_table("backups").find(order_by=["-created_at"], _limit=limit))
|
||||
|
||||
|
||||
def mark_running(uid: str) -> None:
|
||||
get_table("backups").update(
|
||||
{"uid": uid, "status": STATUS_RUNNING}, ["uid"]
|
||||
)
|
||||
|
||||
|
||||
def finalize_backup(uid: str, *, filename: str, local_path: str, stats: dict) -> None:
|
||||
get_table("backups").update(
|
||||
{
|
||||
"uid": uid,
|
||||
"status": STATUS_DONE,
|
||||
"filename": filename,
|
||||
"local_path": local_path,
|
||||
"size_bytes": int(stats.get("bytes_out", 0)),
|
||||
"bytes_in": int(stats.get("bytes_in", 0)),
|
||||
"file_count": int(stats.get("file_count", 0)),
|
||||
"dir_count": int(stats.get("dir_count", 0)),
|
||||
"sha256": stats.get("sha256", ""),
|
||||
"completed_at": now_iso(),
|
||||
"error": "",
|
||||
},
|
||||
["uid"],
|
||||
)
|
||||
|
||||
|
||||
def fail_backup(uid: str, error: str) -> None:
|
||||
get_table("backups").update(
|
||||
{
|
||||
"uid": uid,
|
||||
"status": STATUS_FAILED,
|
||||
"completed_at": now_iso(),
|
||||
"error": error[:2000],
|
||||
},
|
||||
["uid"],
|
||||
)
|
||||
|
||||
|
||||
def delete_backup(uid: str) -> dict | None:
|
||||
row = get_backup(uid)
|
||||
if not row:
|
||||
return None
|
||||
_unlink_archive(row)
|
||||
get_table("backups").delete(uid=uid)
|
||||
return row
|
||||
|
||||
|
||||
def rotate_schedule(schedule_uid: str, keep_last: int) -> int:
|
||||
if keep_last <= 0 or not schedule_uid:
|
||||
return 0
|
||||
rows = list(
|
||||
get_table("backups").find(
|
||||
schedule_uid=schedule_uid,
|
||||
status=STATUS_DONE,
|
||||
order_by=["-created_at"],
|
||||
)
|
||||
)
|
||||
removed = 0
|
||||
for row in rows[keep_last:]:
|
||||
delete_backup(row["uid"])
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
|
||||
def prune_orphans() -> int:
|
||||
removed = 0
|
||||
for row in list_backups(limit=100000):
|
||||
if row.get("status") != STATUS_DONE:
|
||||
continue
|
||||
local_path = row.get("local_path") or ""
|
||||
if not local_path or not Path(local_path).is_file():
|
||||
get_table("backups").delete(uid=row["uid"])
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
|
||||
def clear_all() -> int:
|
||||
rows = list_backups(limit=100000)
|
||||
for row in rows:
|
||||
_unlink_archive(row)
|
||||
get_table("backups").delete()
|
||||
return len(rows)
|
||||
|
||||
|
||||
def _unlink_archive(row: dict) -> None:
|
||||
local_path = row.get("local_path") or ""
|
||||
if local_path:
|
||||
Path(local_path).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def create_schedule(
|
||||
*,
|
||||
name: str,
|
||||
target: str,
|
||||
kind: str,
|
||||
every_seconds: int,
|
||||
cron: str,
|
||||
keep_last: int,
|
||||
created_by: str,
|
||||
next_run_at: str,
|
||||
) -> str:
|
||||
uid = generate_uid()
|
||||
get_table("backup_schedules").insert(
|
||||
{
|
||||
"uid": uid,
|
||||
"name": name,
|
||||
"target": target,
|
||||
"kind": kind,
|
||||
"every_seconds": every_seconds,
|
||||
"cron": cron,
|
||||
"enabled": 1,
|
||||
"keep_last": keep_last,
|
||||
"next_run_at": next_run_at,
|
||||
"last_run_at": "",
|
||||
"last_job_uid": "",
|
||||
"run_count": 0,
|
||||
"created_by": created_by,
|
||||
"created_at": now_iso(),
|
||||
"updated_at": now_iso(),
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
)
|
||||
return uid
|
||||
|
||||
|
||||
def get_schedule(uid: str) -> dict | None:
|
||||
if "backup_schedules" not in db.tables:
|
||||
return None
|
||||
return get_table("backup_schedules").find_one(uid=uid, deleted_at=None)
|
||||
|
||||
|
||||
def list_schedules() -> list[dict]:
|
||||
if "backup_schedules" not in db.tables:
|
||||
return []
|
||||
return list(
|
||||
get_table("backup_schedules").find(
|
||||
deleted_at=None, order_by=["-created_at"]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def list_due_schedules(reference: str) -> list[dict]:
|
||||
if "backup_schedules" not in db.tables:
|
||||
return []
|
||||
rows = get_table("backup_schedules").find(deleted_at=None, enabled=1)
|
||||
return [
|
||||
row
|
||||
for row in rows
|
||||
if (row.get("next_run_at") or "") and row["next_run_at"] <= reference
|
||||
]
|
||||
|
||||
|
||||
def update_schedule(uid: str, changes: dict) -> None:
|
||||
changes = {**changes, "uid": uid, "updated_at": now_iso()}
|
||||
get_table("backup_schedules").update(changes, ["uid"])
|
||||
|
||||
|
||||
def set_schedule_runtime(
|
||||
uid: str, *, next_run_at: str, last_run_at: str, last_job_uid: str, run_count: int
|
||||
) -> None:
|
||||
get_table("backup_schedules").update(
|
||||
{
|
||||
"uid": uid,
|
||||
"next_run_at": next_run_at,
|
||||
"last_run_at": last_run_at,
|
||||
"last_job_uid": last_job_uid,
|
||||
"run_count": run_count,
|
||||
"updated_at": now_iso(),
|
||||
},
|
||||
["uid"],
|
||||
)
|
||||
|
||||
|
||||
def delete_schedule(uid: str, deleted_by: str) -> bool:
|
||||
row = get_schedule(uid)
|
||||
if not row:
|
||||
return False
|
||||
get_table("backup_schedules").update(
|
||||
{"uid": uid, "deleted_at": now_iso(), "deleted_by": deleted_by}, ["uid"]
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _path_size(path: Path) -> tuple[int, int]:
|
||||
if not path.exists():
|
||||
return 0, 0
|
||||
if path.is_file():
|
||||
return path.stat().st_size, 1
|
||||
total = 0
|
||||
files = 0
|
||||
for entry in path.rglob("*"):
|
||||
try:
|
||||
if entry.is_file() and not entry.is_symlink():
|
||||
total += entry.stat().st_size
|
||||
files += 1
|
||||
except OSError:
|
||||
continue
|
||||
return total, files
|
||||
|
||||
|
||||
def _storage_paths() -> list[tuple[str, str, Path]]:
|
||||
database_file = Path(str(config.DATABASE_URL).replace("sqlite:///", ""))
|
||||
return [
|
||||
("database", "Database file", database_file),
|
||||
("devii_tasks", "Devii tasks DB", config.DEVII_TASKS_DB),
|
||||
("devii_lessons", "Devii lessons DB", config.DEVII_LESSONS_DB),
|
||||
("uploads", "Uploads", config.UPLOADS_DIR),
|
||||
("attachments", "Attachments", config.ATTACHMENTS_DIR),
|
||||
("project_files", "Project files", config.PROJECT_FILES_DIR),
|
||||
("keys", "Keys", config.KEYS_DIR),
|
||||
("zips", "Zip archives", config.ZIPS_DIR),
|
||||
("deepsearch", "DeepSearch", config.DEEPSEARCH_DIR),
|
||||
("container_workspaces", "Container workspaces", config.CONTAINER_WORKSPACES_DIR),
|
||||
("backups", "Backups", config.BACKUPS_DIR),
|
||||
]
|
||||
|
||||
|
||||
def compute_storage_stats() -> dict:
|
||||
now = time.monotonic()
|
||||
if (
|
||||
_storage_cache["data"] is not None
|
||||
and (now - _storage_cache["at"]) < STORAGE_CACHE_TTL_SECONDS
|
||||
):
|
||||
return _storage_cache["data"]
|
||||
|
||||
paths = []
|
||||
for key, label, path in _storage_paths():
|
||||
size, files = _path_size(path)
|
||||
paths.append(
|
||||
{
|
||||
"key": key,
|
||||
"label": label,
|
||||
"path": str(path),
|
||||
"size_bytes": size,
|
||||
"size_human": human_bytes(size),
|
||||
"file_count": files,
|
||||
"exists": path.exists(),
|
||||
}
|
||||
)
|
||||
|
||||
data_size, data_files = _path_size(config.DATA_DIR)
|
||||
backups_size, backups_files = _path_size(config.BACKUPS_DIR)
|
||||
backup_count = len(
|
||||
[b for b in list_backups(limit=100000) if b.get("status") == STATUS_DONE]
|
||||
)
|
||||
usage = shutil.disk_usage(str(config.DATA_DIR))
|
||||
|
||||
data = {
|
||||
"paths": paths,
|
||||
"data_dir": {
|
||||
"path": str(config.DATA_DIR),
|
||||
"size_bytes": data_size,
|
||||
"size_human": human_bytes(data_size),
|
||||
"file_count": data_files,
|
||||
},
|
||||
"backups_total": {
|
||||
"count": backup_count,
|
||||
"size_bytes": backups_size,
|
||||
"size_human": human_bytes(backups_size),
|
||||
"file_count": backups_files,
|
||||
},
|
||||
"disk": {
|
||||
"total_bytes": usage.total,
|
||||
"used_bytes": usage.used,
|
||||
"free_bytes": usage.free,
|
||||
"total_human": human_bytes(usage.total),
|
||||
"used_human": human_bytes(usage.used),
|
||||
"free_human": human_bytes(usage.free),
|
||||
"used_percent": round(usage.used / usage.total * 100, 1)
|
||||
if usage.total
|
||||
else 0.0,
|
||||
},
|
||||
"generated_at": now_iso(),
|
||||
}
|
||||
_storage_cache["data"] = data
|
||||
_storage_cache["at"] = now
|
||||
return data
|
||||
Reference in New Issue
Block a user