|
# retoor <retoor@molodetz.nl>
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from devplacepy.config import BASE_DIR, DATA_DIR
|
|
from devplacepy import project_files
|
|
from devplacepy.services.jobs.base import JobService
|
|
from devplacepy.utils import generate_uid, slugify
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
ZIPS_DIR = DATA_DIR / "zips"
|
|
STAGING_DIR = DATA_DIR / "zip_staging"
|
|
WORKER_MODULE = "devplacepy.services.jobs.zip_worker"
|
|
|
|
|
|
def _shard(uid: str) -> str:
|
|
return f"{uid[:2]}/{uid[2:4]}"
|
|
|
|
|
|
class ZipService(JobService):
|
|
kind = "zip"
|
|
title = "Zip"
|
|
description = (
|
|
"Builds downloadable zip archives in a subprocess off the request path, tracks each "
|
|
"job with full statistics, and deletes archives once they go unused for the retention "
|
|
"window."
|
|
)
|
|
|
|
def __init__(self):
|
|
super().__init__(name="zip", interval_seconds=2)
|
|
|
|
async def process(self, job: dict) -> dict:
|
|
source = job["payload"].get("source", {})
|
|
uid = job["uid"]
|
|
staging = STAGING_DIR / uid
|
|
try:
|
|
item_count = await asyncio.to_thread(self._materialize, source, staging)
|
|
tmp_dir = ZIPS_DIR / _shard(uid)
|
|
tmp_dir.mkdir(parents=True, exist_ok=True)
|
|
tmp_zip = tmp_dir / f"{generate_uid()}.zip"
|
|
stats = await self._run_worker(staging, tmp_zip)
|
|
final_name = self._final_name(job.get("preferred_name", ""), stats["crc32"])
|
|
final_path = tmp_dir / final_name
|
|
os.replace(tmp_zip, final_path)
|
|
finally:
|
|
await asyncio.to_thread(shutil.rmtree, staging, ignore_errors=True)
|
|
|
|
return {
|
|
"download_url": f"/zips/{uid}/download",
|
|
"local_path": str(final_path),
|
|
"final_name": final_name,
|
|
"crc32": f"{stats['crc32']:08x}",
|
|
"file_count": stats["file_count"],
|
|
"dir_count": stats["dir_count"],
|
|
"bytes_in": stats["bytes_in"],
|
|
"bytes_out": stats["bytes_out"],
|
|
"item_count": item_count,
|
|
}
|
|
|
|
def _materialize(self, source: dict, staging: Path) -> int:
|
|
if source.get("type") == "project_tree":
|
|
return project_files.export_to_dir(source["project_uid"], source.get("path", ""), staging)
|
|
raise ValueError(f"unsupported zip source: {source.get('type')}")
|
|
|
|
async def _run_worker(self, source_dir: Path, output_path: Path) -> dict:
|
|
proc = await asyncio.create_subprocess_exec(
|
|
sys.executable, "-m", WORKER_MODULE, str(source_dir), str(output_path),
|
|
cwd=str(BASE_DIR),
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
out, err = await proc.communicate()
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f"zip worker exited {proc.returncode}: {err.decode('utf-8', 'replace')[:500]}")
|
|
return json.loads(out.decode("utf-8"))
|
|
|
|
def _final_name(self, preferred_name: str, crc32: int) -> str:
|
|
name = preferred_name or "download"
|
|
if name.lower().endswith(".zip"):
|
|
name = name[:-4]
|
|
slug = slugify(name) or "download"
|
|
return f"{crc32:08x}.{slug}.zip"
|
|
|
|
def cleanup(self, job: dict) -> None:
|
|
result = job.get("result", {})
|
|
local_path = result.get("local_path")
|
|
if local_path:
|
|
Path(local_path).unlink(missing_ok=True)
|
|
shutil.rmtree(STAGING_DIR / job["uid"], ignore_errors=True)
|