feat: add project fork async job service with cli prune/clear commands and shared container image build target
This commit is contained in:
@@ -10,11 +10,9 @@ import httpx
|
||||
|
||||
from devplacepy import config, project_files
|
||||
from devplacepy.services.containers import store
|
||||
from devplacepy.services.containers.locks import NUMBERING_LOCK
|
||||
from devplacepy.services.containers.templates import DEFAULT_DOCKERFILE
|
||||
from devplacepy.services.containers.backend.base import Mount, PortMapping, RunSpec
|
||||
from devplacepy.services.containers.runtime import get_backend
|
||||
from devplacepy.services.devii.tasks.schedule import Schedule, next_run, now_utc, to_iso, from_iso
|
||||
from devplacepy.services.jobs import queue
|
||||
|
||||
IMAGE_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,62}$")
|
||||
INGRESS_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
|
||||
@@ -31,13 +29,6 @@ class ContainerError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def validate_image_name(name: str) -> str:
|
||||
name = (name or "").strip().lower()
|
||||
if not IMAGE_NAME_RE.match(name):
|
||||
raise ContainerError("name must be lowercase letters, digits, '.', '_' or '-' (max 63 chars)")
|
||||
return name
|
||||
|
||||
|
||||
def _validate_limits(cpu_limit: str, mem_limit: str) -> None:
|
||||
if cpu_limit and not CPU_RE.match(str(cpu_limit)):
|
||||
raise ContainerError("cpu limit must be a number, e.g. 1 or 1.5")
|
||||
@@ -126,41 +117,6 @@ def parse_env(value) -> dict:
|
||||
return env
|
||||
|
||||
|
||||
# ---------------- dockerfiles + builds ----------------
|
||||
|
||||
async def enqueue_build(dockerfile: dict, version: dict, *, also_latest: bool = True, owner=("system", "system")) -> dict:
|
||||
async with NUMBERING_LOCK:
|
||||
head = store.get_dockerfile(dockerfile["uid"])
|
||||
build_number = int(head.get("latest_build_number") or 0) + 1
|
||||
store.update_dockerfile(head["uid"], {"latest_build_number": build_number})
|
||||
image_tag = f"{head['name']}:{build_number}"
|
||||
build = store.create_build(head, version, build_number, image_tag, also_latest)
|
||||
job_uid = queue.enqueue("container_build", {"build_uid": build["uid"]}, owner[0], owner[1], f"build {image_tag}")
|
||||
store.update_build(build["uid"], {"job_uid": job_uid})
|
||||
return store.get_build(build["uid"])
|
||||
|
||||
|
||||
async def create_dockerfile(project: dict, user: dict, *, name: str, description: str = "",
|
||||
tags: str = "", content: str = None) -> dict:
|
||||
name = validate_image_name(name)
|
||||
for existing in store.list_dockerfiles(project["uid"]):
|
||||
if existing["name"] == name:
|
||||
raise ContainerError(f"a Dockerfile named '{name}' already exists in this project")
|
||||
content = content if content is not None and content.strip() else DEFAULT_DOCKERFILE
|
||||
dockerfile = store.create_dockerfile(project["uid"], user, name, description, tags)
|
||||
version = store.create_version(dockerfile, content, user)
|
||||
build = await enqueue_build(dockerfile, version, owner=("user", user["uid"]))
|
||||
return {"dockerfile": store.get_dockerfile(dockerfile["uid"]), "version": version, "build": build}
|
||||
|
||||
|
||||
async def save_version(dockerfile: dict, user: dict, content: str, *, force: bool = False) -> dict:
|
||||
if not force and store.content_hash(content) == dockerfile.get("content_hash"):
|
||||
return {"version": None, "build": None, "changed": False}
|
||||
version = store.create_version(dockerfile, content, user)
|
||||
build = await enqueue_build(store.get_dockerfile(dockerfile["uid"]), version, owner=("user", user["uid"]))
|
||||
return {"version": version, "build": build, "changed": True}
|
||||
|
||||
|
||||
# ---------------- instances ----------------
|
||||
|
||||
def validate_ingress(slug: str, port, port_list) -> tuple:
|
||||
@@ -181,13 +137,13 @@ def validate_ingress(slug: str, port, port_list) -> tuple:
|
||||
return slug, ingress_port
|
||||
|
||||
|
||||
async def create_instance(project: dict, dockerfile: dict, build: dict, *, name: str,
|
||||
async def create_instance(project: dict, *, name: str,
|
||||
boot_command: str = "", env="", cpu_limit: str = "", mem_limit: str = "",
|
||||
ports="", volumes="", restart_policy: str = "never",
|
||||
autostart: bool = True, ingress_slug: str = "", ingress_port=None,
|
||||
actor=("system", "system")) -> dict:
|
||||
if build.get("status") != store.BUILD_SUCCESS:
|
||||
raise ContainerError("the selected build has not completed successfully")
|
||||
if not await get_backend().image_exists(config.CONTAINER_IMAGE):
|
||||
raise ContainerError(f"the '{config.CONTAINER_IMAGE}' image is not built - run 'make ppy'")
|
||||
if restart_policy not in store.RESTART_POLICIES:
|
||||
raise ContainerError(f"restart policy must be one of {', '.join(store.RESTART_POLICIES)}")
|
||||
_validate_limits(cpu_limit, mem_limit)
|
||||
@@ -202,7 +158,9 @@ async def create_instance(project: dict, dockerfile: dict, build: dict, *, name:
|
||||
await asyncio.to_thread(project_files.export_to_dir, project["uid"], "", str(workspace))
|
||||
|
||||
row = {
|
||||
"project_uid": project["uid"], "dockerfile_uid": dockerfile["uid"], "build_uid": build["uid"],
|
||||
"project_uid": project["uid"],
|
||||
"created_by": actor[1] if actor and actor[0] == "user" else "",
|
||||
"owner_uid": project.get("user_uid", ""),
|
||||
"name": name, "boot_command": boot_command or "",
|
||||
"env_json": json.dumps(env_map),
|
||||
"cpu_limit": str(cpu_limit or ""), "mem_limit": str(mem_limit or ""),
|
||||
@@ -215,7 +173,7 @@ async def create_instance(project: dict, dockerfile: dict, build: dict, *, name:
|
||||
"workspace_dir": str(workspace),
|
||||
}
|
||||
instance = store.create_instance(row)
|
||||
store.record_event(instance, "created", actor[0], actor[1], {"build": build["uid"]})
|
||||
store.record_event(instance, "created", actor[0], actor[1], {"image": config.CONTAINER_IMAGE})
|
||||
return instance
|
||||
|
||||
|
||||
@@ -238,18 +196,40 @@ def mark_for_removal(instance: dict, *, actor=("system", "system")) -> None:
|
||||
store.record_event(instance, "remove", actor[0], actor[1])
|
||||
|
||||
|
||||
def pravda_env(instance: dict) -> dict:
|
||||
from devplacepy import database, seo
|
||||
base_url = seo.public_base_url()
|
||||
api_key = ""
|
||||
for uid in (instance.get("created_by"), instance.get("owner_uid")):
|
||||
if not uid:
|
||||
continue
|
||||
user = database.get_users_by_uids([uid]).get(uid)
|
||||
if user and user.get("api_key"):
|
||||
api_key = user["api_key"]
|
||||
break
|
||||
slug = instance.get("ingress_slug") or ""
|
||||
ingress_url = (f"{base_url}/p/{slug}" if base_url else f"/p/{slug}") if slug else ""
|
||||
return {
|
||||
"PRAVDA_BASE_URL": base_url,
|
||||
"PRAVDA_OPENAI_URL": f"{base_url}/openai/v1" if base_url else "",
|
||||
"PRAVDA_API_KEY": api_key,
|
||||
"PRAVDA_USER_UID": instance.get("owner_uid") or "",
|
||||
"PRAVDA_CONTAINER_NAME": instance.get("name") or "",
|
||||
"PRAVDA_CONTAINER_UID": instance.get("uid") or "",
|
||||
"PRAVDA_INGRESS_URL": ingress_url,
|
||||
}
|
||||
|
||||
|
||||
def run_spec_for(instance: dict, image_tag: str) -> RunSpec:
|
||||
env = json.loads(instance.get("env_json") or "{}")
|
||||
env = {**json.loads(instance.get("env_json") or "{}"), **pravda_env(instance)}
|
||||
ports = [PortMapping(p["host"], p["container"], p.get("proto", "tcp"))
|
||||
for p in json.loads(instance.get("ports_json") or "[]")]
|
||||
mounts = [Mount(instance["workspace_dir"], "/app", "rw")]
|
||||
for extra in json.loads(instance.get("volumes_json") or "[]"):
|
||||
if isinstance(extra, dict) and extra.get("host") and extra.get("container"):
|
||||
mounts.append(Mount(extra["host"], extra["container"], extra.get("mode", "rw")))
|
||||
command = []
|
||||
boot = (instance.get("boot_command") or "").strip()
|
||||
if boot:
|
||||
command = ["/bin/sh", "-c", boot]
|
||||
command = ["/bin/sh", "-c", boot] if boot else ["sleep", "infinity"]
|
||||
return RunSpec(
|
||||
image=image_tag, name=instance["slug"],
|
||||
labels={INSTANCE_LABEL: instance["uid"], PROJECT_LABEL: instance["project_uid"]},
|
||||
@@ -284,21 +264,6 @@ def _percentile(values: list, pct: float) -> float:
|
||||
return float(ordered[index])
|
||||
|
||||
|
||||
def dockerfile_stats(dockerfile_uid: str) -> dict:
|
||||
builds = store.list_builds(dockerfile_uid, limit=1000)
|
||||
done = [b for b in builds if b["status"] in (store.BUILD_SUCCESS, store.BUILD_FAILED)]
|
||||
success = [b for b in done if b["status"] == store.BUILD_SUCCESS]
|
||||
durations = [int(b.get("duration_ms") or 0) for b in success if b.get("duration_ms")]
|
||||
return {
|
||||
"total_builds": len(builds),
|
||||
"success": len(success),
|
||||
"failed": len(done) - len(success),
|
||||
"success_rate": round(len(success) / len(done) * 100, 1) if done else 0.0,
|
||||
"avg_build_ms": int(sum(durations) / len(durations)) if durations else 0,
|
||||
"p95_build_ms": int(_percentile(durations, 95)),
|
||||
}
|
||||
|
||||
|
||||
def instance_stats(instance_uid: str) -> dict:
|
||||
metrics = store.recent_metrics(instance_uid, limit=720)
|
||||
cpu = [m.get("cpu_pct", 0) for m in metrics]
|
||||
|
||||
Reference in New Issue
Block a user