devplacepy/git.py at 499f91e16a6d908b2c7545a32147ac255f31ebcd

 # retoor <retoor@molodetz.nl>
 from __future__ import annotations
 import asyncio
 import logging
 import re
 from dataclasses import dataclass
 from pathlib import Path
 from typing import AsyncIterator, Optional
 from urllib.parse import urlparse
 import httpx
 from devplacepy.stealth import stealth_async_client
 from devplacepy.services.jobs.isslop.acquisition.workspace import directory_size_bytes
 from devplacepy.services.jobs.isslop.config import (
     GIT_CLONE_TIMEOUT_SECONDS,
     GIT_SIZE_LIMIT_BYTES,
     GIT_SIZE_POLL_SECONDS,
     WEBSITE_REQUEST_TIMEOUT_SECONDS,
 )
 logger = logging.getLogger(__name__)
 class RepositoryTooLargeError(Exception):
     pass
 class CloneFailedError(Exception):
     pass
 @dataclass(frozen=True)
 class SizePreflight:
     known: bool
     size_bytes: int
     origin: str
 def _owner_repo(url: str) -> Optional[tuple[str, str, str]]:
     match = re.match(r"^git@([\w.-]+):(.+)$", url)
     if match:
         host, path = match.group(1), match.group(2)
     else:
         parsed = urlparse(url)
         host, path = parsed.hostname or "", parsed.path
     parts = [part for part in path.strip("/").removesuffix(".git").split("/") if part]
     if len(parts) < 2 or not host:
         return None
     return host, parts[0], parts[1]
 async def preflight_size(url: str) -> SizePreflight:
     located = _owner_repo(url)
     if located is None:
         return SizePreflight(known=False, size_bytes=0, origin="unparseable")
     host, owner, repo = located
     candidates: list[tuple[str, str]] = []
     if host == "github.com":
         candidates.append((f"https://api.github.com/repos/{owner}/{repo}", "github"))
     else:
         candidates.append((f"https://{host}/api/v1/repos/{owner}/{repo}", "gitea"))
     async with stealth_async_client(timeout=WEBSITE_REQUEST_TIMEOUT_SECONDS, follow_redirects=True) as client:
         for api_url, origin in candidates:
             try:
                 response = await client.get(api_url, headers={"accept": "application/json"})
             except httpx.HTTPError as error:
                 logger.info("Size preflight unavailable via %s: %s", origin, error)
                 continue
             if response.status_code != 200:
                 logger.info("Size preflight %s returned HTTP %d", origin, response.status_code)
                 continue
             try:
                 body = response.json()
             except ValueError:
                 continue
             size_kb = body.get("size")
             if isinstance(size_kb, (int, float)) and size_kb > 0:
                 size_bytes = int(size_kb) * 1024
                 logger.info("Preflight size via %s: %d bytes", origin, size_bytes)
                 return SizePreflight(known=True, size_bytes=size_bytes, origin=origin)
     return SizePreflight(known=False, size_bytes=0, origin="unknown")
 async def clone_repository(url: str, workspace: Path, size_limit: int = GIT_SIZE_LIMIT_BYTES) -> AsyncIterator[str]:
     preflight = await preflight_size(url)
     if preflight.known:
         yield f"Repository size reported by {preflight.origin}: {preflight.size_bytes / (1024 * 1024):.1f} MB"
         if preflight.size_bytes > size_limit:
             raise RepositoryTooLargeError(
                 f"Repository is {preflight.size_bytes / (1024 ** 3):.2f} GB which exceeds the 3 GB limit"
             )
     else:
         yield "Repository size not determinable up front, monitoring during clone"
     argv = [
         "git",
         "clone",
         "--depth",
         "1",
         "--single-branch",
         "--no-tags",
         url,
         str(workspace),
     ]
     yield f"Cloning with depth 1: {url}"
     proc = await asyncio.create_subprocess_exec(
         *argv,
         stdin=asyncio.subprocess.DEVNULL,
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.STDOUT,
         env={"GIT_TERMINAL_PROMPT": "0", "GIT_ASKPASS": "true", "PATH": "/usr/bin:/bin:/usr/local/bin"},
     )
     aborted_reason: list[str] = []
     async def monitor() -> None:
         while proc.returncode is None:
             await asyncio.sleep(GIT_SIZE_POLL_SECONDS)
             if not workspace.exists():
                 continue
             size = await asyncio.to_thread(directory_size_bytes, workspace)
             logger.debug("Clone size check: %d bytes", size)
             if size > size_limit:
                 aborted_reason.append(f"Clone exceeded 3 GB limit at {size / (1024 ** 3):.2f} GB, cancelled")
                 proc.kill()
                 return
     monitor_task = asyncio.create_task(monitor())
     output_lines: list[str] = []
     try:
         assert proc.stdout is not None
         while True:
             try:
                 line = await asyncio.wait_for(proc.stdout.readline(), timeout=GIT_CLONE_TIMEOUT_SECONDS)
             except asyncio.TimeoutError:
                 proc.kill()
                 raise CloneFailedError("Clone timed out")
             if not line:
                 break
             text = line.decode("utf-8", errors="replace").strip()
             if text:
                 output_lines.append(text)
                 yield f"git: {text}"
         await proc.wait()
     finally:
         monitor_task.cancel()
         try:
             await monitor_task
         except asyncio.CancelledError:
             logger.debug("Clone size monitor stopped")
     if aborted_reason:
         raise RepositoryTooLargeError(aborted_reason[0])
     if proc.returncode != 0:
         tail = " | ".join(output_lines[-4:])
         raise CloneFailedError(f"git clone failed with code {proc.returncode}: {tail}")
     final_size = await asyncio.to_thread(directory_size_bytes, workspace)
     if final_size > size_limit:
         raise RepositoryTooLargeError(f"Repository is {final_size / (1024 ** 3):.2f} GB which exceeds the 3 GB limit")
     yield f"Clone complete: {final_size / (1024 * 1024):.1f} MB on disk"