# retoor import asyncio import json import re import socket from pathlib import Path from devplacepy import config, project_files, stealth from devplacepy.services.containers import store from devplacepy.services.containers.backend.base import ( WORKSPACE_MOUNT, Mount, PortMapping, RunSpec, ) from devplacepy.services.containers.runtime import get_backend from devplacepy.services.devii.tasks.schedule import ( Schedule, now_utc, to_iso, ) IMAGE_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,62}$") INGRESS_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") MEM_RE = re.compile(r"^\d+(\.\d+)?[bkmgBKMG]?$") CPU_RE = re.compile(r"^\d+(\.\d+)?$") ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") INSTANCE_LABEL = "devplace.instance" PROJECT_LABEL = "devplace.project" HOST_PORT_MIN = 20001 HOST_PORT_MAX = 65535 BOOT_LANGUAGES = ("none", "python", "bash") BOOT_SCRIPT_FILES = {"python": ".devplace_boot.py", "bash": ".devplace_boot.sh"} BOOT_SCRIPT_RUNNERS = {"python": "python", "bash": "bash"} MAX_BOOT_SCRIPT_CHARS = 100_000 class ContainerError(ValueError): pass def _validate_limits(cpu_limit: str, mem_limit: str) -> None: if cpu_limit and not CPU_RE.match(str(cpu_limit)): raise ContainerError("cpu limit must be a number, e.g. 1 or 1.5") if mem_limit and not MEM_RE.match(str(mem_limit)): raise ContainerError("memory limit must look like 512m, 1g, or a byte count") def validate_run_as(run_as_uid) -> str: uid = str(run_as_uid or "").strip() if not uid: return "" from devplacepy import database user = database.get_users_by_uids([uid]).get(uid) if not user: raise ContainerError(f"run-as user not found: {uid}") return uid def validate_boot(boot_language, boot_script) -> tuple: language = str(boot_language or "none").strip().lower() or "none" if language not in BOOT_LANGUAGES: raise ContainerError( f"boot language must be one of {', '.join(BOOT_LANGUAGES)}" ) script = str(boot_script or "") if language == "none": script = "" if len(script) > MAX_BOOT_SCRIPT_CHARS: raise ContainerError( f"boot script exceeds the {MAX_BOOT_SCRIPT_CHARS}-character limit" ) if language != "none" and not script.strip(): raise ContainerError("boot script is required when a boot language is set") return language, script def parse_ports(value) -> list: ports = [] if not value: return ports items = ( value if isinstance(value, list) else str(value).replace(",", "\n").splitlines() ) for item in items: item = str(item).strip() if not item: continue proto = "tcp" if "/" in item: item, proto = item.split("/", 1) if ":" in item: host, container = item.split(":", 1) else: host, container = "0", item if not host.isdigit() or not container.isdigit(): raise ContainerError( f"port '{item}' must be numeric host:container or a bare container port" ) ports.append(PortMapping(int(host), int(container), proto.strip() or "tcp")) return ports def used_host_ports() -> set: ports = set() for instance in store.all_instances(): for mapping in json.loads(instance.get("ports_json") or "[]"): host = int(mapping.get("host") or 0) if host: ports.add(host) return ports def _host_port_free(port: int) -> bool: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: sock.bind(("0.0.0.0", port)) return True except OSError: return False def allocate_host_port(reserved: set) -> int: for port in range(HOST_PORT_MIN, HOST_PORT_MAX + 1): if port in reserved: continue if _host_port_free(port): return port raise ContainerError( f"no free host port available in range {HOST_PORT_MIN}-{HOST_PORT_MAX}" ) def assign_host_ports(port_list: list) -> list: reserved = used_host_ports() assigned = [] for mapping in port_list: host = mapping.host if not host: host = allocate_host_port(reserved) elif host in reserved: raise ContainerError( f"host port {host} is already published by another instance" ) reserved.add(host) assigned.append(PortMapping(host, mapping.container, mapping.proto)) return assigned def parse_env(value) -> dict: env = {} if not value: return env if isinstance(value, dict): items = value.items() else: items = (line.split("=", 1) for line in str(value).splitlines() if "=" in line) for key, val in items: key = str(key).strip() if not ENV_KEY_RE.match(key): raise ContainerError(f"invalid environment variable name: {key}") env[key] = str(val) return env # ---------------- instances ---------------- def validate_ingress(slug: str, port, port_list) -> tuple: slug = (slug or "").strip().lower() if not slug: return "", 0 if not INGRESS_SLUG_RE.match(slug): raise ContainerError( "ingress slug must be lowercase letters, digits, or '-' (max 63 chars)" ) for other in store.all_instances(): if other.get("ingress_slug") == slug: raise ContainerError(f"ingress slug '{slug}' is already in use") ingress_port = int(port) if port else 0 container_ports = {p.container for p in port_list} if ingress_port and ingress_port not in container_ports: raise ContainerError( f"ingress_port {ingress_port} must be one of the container ports you mapped" ) if not ingress_port and len(container_ports) != 1: raise ContainerError( "set ingress_port to choose which mapped container port to publish" ) return slug, ingress_port async def create_instance( project: dict, *, name: str, boot_command: str = "", boot_language: str = "none", boot_script: str = "", run_as_uid: str = "", start_on_boot: bool = False, env="", cpu_limit: str = "", mem_limit: str = "", ports="", volumes="", restart_policy: str = "never", autostart: bool = True, ingress_slug: str = "", ingress_port=None, actor=("system", "system"), ) -> dict: if not await get_backend().image_exists(config.CONTAINER_IMAGE): raise ContainerError( f"the '{config.CONTAINER_IMAGE}' image is not built - run 'make ppy'" ) if restart_policy not in store.RESTART_POLICIES: raise ContainerError( f"restart policy must be one of {', '.join(store.RESTART_POLICIES)}" ) _validate_limits(cpu_limit, mem_limit) run_as_uid = validate_run_as(run_as_uid) boot_language, boot_script = validate_boot(boot_language, boot_script) port_list = assign_host_ports(parse_ports(ports)) env_map = parse_env(env) name = (name or "").strip() if not name: raise ContainerError("instance name is required") ingress_slug, ingress_port = validate_ingress(ingress_slug, ingress_port, port_list) workspace = Path(config.CONTAINER_WORKSPACES_DIR) / project["uid"] await asyncio.to_thread( project_files.export_to_dir, project["uid"], "", str(workspace) ) row = { "project_uid": project["uid"], "created_by": actor[1] if actor and actor[0] == "user" else "", "owner_uid": project.get("user_uid", ""), "run_as_uid": run_as_uid, "name": name, "boot_command": boot_command or "", "boot_language": boot_language, "boot_script": boot_script, "start_on_boot": 1 if start_on_boot else 0, "env_json": json.dumps(env_map), "cpu_limit": str(cpu_limit or ""), "mem_limit": str(mem_limit or ""), "ports_json": json.dumps( [ {"host": p.host, "container": p.container, "proto": p.proto} for p in port_list ] ), "volumes_json": volumes if isinstance(volumes, str) else json.dumps(volumes or []), "restart_policy": restart_policy, "ingress_slug": ingress_slug, "ingress_port": ingress_port, "desired_state": store.DESIRED_RUNNING if autostart else store.DESIRED_STOPPED, "status": store.ST_CREATED, "workspace_dir": str(workspace), } instance = store.create_instance(row) store.record_event( instance, "created", actor[0], actor[1], {"image": config.CONTAINER_IMAGE} ) if actor and actor[0] == "user": from devplacepy.utils import track_action track_action(actor[1], "container") return instance def set_desired_state( instance: dict, desired: str, *, actor=("system", "system") ) -> dict: if desired not in ( store.DESIRED_RUNNING, store.DESIRED_STOPPED, store.DESIRED_PAUSED, ): raise ContainerError("desired state must be running, stopped, or paused") store.update_instance(instance["uid"], {"desired_state": desired}) store.record_event(instance, f"desire_{desired}", actor[0], actor[1]) return store.get_instance(instance["uid"]) def request_restart(instance: dict, *, actor=("system", "system")) -> dict: store.update_instance( instance["uid"], {"desired_state": store.DESIRED_RUNNING, "status": store.ST_RESTARTING}, ) store.record_event(instance, "restart", actor[0], actor[1]) return store.get_instance(instance["uid"]) def mark_for_removal(instance: dict, *, actor=("system", "system")) -> None: store.update_instance( instance["uid"], {"desired_state": store.DESIRED_STOPPED, "status": store.ST_REMOVING}, ) store.record_event(instance, "remove", actor[0], actor[1]) def update_instance_config( instance: dict, *, run_as_uid=None, boot_language=None, boot_script=None, boot_command=None, restart_policy=None, start_on_boot=None, cpu_limit=None, mem_limit=None, actor=("system", "system"), ) -> dict: changes: dict = {} if run_as_uid is not None: changes["run_as_uid"] = validate_run_as(run_as_uid) if boot_language is not None or boot_script is not None: language = ( boot_language if boot_language is not None else instance.get("boot_language", "none") ) script = ( boot_script if boot_script is not None else instance.get("boot_script", "") ) language, script = validate_boot(language, script) changes["boot_language"] = language changes["boot_script"] = script if boot_command is not None: changes["boot_command"] = str(boot_command or "")[:500] if restart_policy is not None: if restart_policy not in store.RESTART_POLICIES: raise ContainerError( f"restart policy must be one of {', '.join(store.RESTART_POLICIES)}" ) changes["restart_policy"] = restart_policy if start_on_boot is not None: changes["start_on_boot"] = 1 if start_on_boot else 0 if cpu_limit is not None or mem_limit is not None: cpu = cpu_limit if cpu_limit is not None else instance.get("cpu_limit", "") mem = mem_limit if mem_limit is not None else instance.get("mem_limit", "") _validate_limits(cpu, mem) changes["cpu_limit"] = str(cpu or "") changes["mem_limit"] = str(mem or "") if not changes: return store.get_instance(instance["uid"]) store.update_instance(instance["uid"], changes) store.record_event( instance, "configure", actor[0], actor[1], {"fields": sorted(changes)} ) return store.get_instance(instance["uid"]) def set_start_on_boot( instance: dict, enabled: bool, *, actor=("system", "system") ) -> dict: store.update_instance(instance["uid"], {"start_on_boot": 1 if enabled else 0}) store.record_event( instance, "start_on_boot", actor[0], actor[1], {"enabled": bool(enabled)} ) return store.get_instance(instance["uid"]) def materialize_boot_script(instance: dict) -> None: language = (instance.get("boot_language") or "none").strip().lower() workspace = instance.get("workspace_dir") if not workspace: return for filename in BOOT_SCRIPT_FILES.values(): stale = Path(workspace) / filename if stale.is_file(): try: stale.unlink() except OSError: pass if language not in BOOT_SCRIPT_FILES: return script = instance.get("boot_script") or "" if not script.strip(): return target = Path(workspace) / BOOT_SCRIPT_FILES[language] try: Path(workspace).mkdir(parents=True, exist_ok=True) target.write_text(script, encoding="utf-8") except OSError: pass def pravda_env(instance: dict) -> dict: from devplacepy import database, seo base_url = seo.public_base_url() api_key = "" user_uid = instance.get("owner_uid") or "" for uid in ( instance.get("run_as_uid"), instance.get("created_by"), instance.get("owner_uid"), ): if not uid: continue user = database.get_users_by_uids([uid]).get(uid) if user and user.get("api_key"): api_key = user["api_key"] break slug = instance.get("ingress_slug") or "" ingress_url = (f"{base_url}/p/{slug}" if base_url else f"/p/{slug}") if slug else "" return { "PRAVDA_BASE_URL": base_url, "PRAVDA_OPENAI_URL": f"{base_url}/openai/v1" if base_url else "", "PRAVDA_API_KEY": api_key, "PRAVDA_USER_UID": instance.get("run_as_uid") or user_uid, "PRAVDA_CONTAINER_NAME": instance.get("name") or "", "PRAVDA_CONTAINER_UID": instance.get("uid") or "", "PRAVDA_INGRESS_URL": ingress_url, } def run_spec_for(instance: dict, image_tag: str) -> RunSpec: env = {**json.loads(instance.get("env_json") or "{}"), **pravda_env(instance)} ports = [ PortMapping(p["host"], p["container"], p.get("proto", "tcp")) for p in json.loads(instance.get("ports_json") or "[]") ] mounts = [Mount(instance["workspace_dir"], WORKSPACE_MOUNT, "rw")] for extra in json.loads(instance.get("volumes_json") or "[]"): if isinstance(extra, dict) and extra.get("host") and extra.get("container"): mounts.append( Mount(extra["host"], extra["container"], extra.get("mode", "rw")) ) language = (instance.get("boot_language") or "none").strip().lower() boot = (instance.get("boot_command") or "").strip() if language in BOOT_SCRIPT_FILES and (instance.get("boot_script") or "").strip(): script_path = f"{WORKSPACE_MOUNT}/{BOOT_SCRIPT_FILES[language]}" command = [BOOT_SCRIPT_RUNNERS[language], script_path] elif boot: command = ["/bin/sh", "-c", boot] else: command = ["sleep", "infinity"] return RunSpec( image=image_tag, name=instance["slug"], labels={ INSTANCE_LABEL: instance["uid"], PROJECT_LABEL: instance["project_uid"], }, env=env, cpu_limit=instance.get("cpu_limit", ""), mem_limit=instance.get("mem_limit", ""), ports=ports, mounts=mounts, restart_policy=instance.get("restart_policy", "never"), command=command, ) async def sync_workspace(instance: dict, user: dict) -> dict: workspace = instance.get("workspace_dir") if not workspace: raise ContainerError("instance has no workspace") counts = await asyncio.to_thread( project_files.sync_dir_bidirectional, instance["project_uid"], workspace, user ) store.record_event( instance, "sync", "user", user["uid"], {"exported": counts["exported"], "imported": counts["imported"]}, ) return counts def sync_bidirectional_sync(instance: dict, user: dict) -> dict: workspace = instance.get("workspace_dir") if not workspace: return {"exported": 0, "imported": 0} counts = project_files.sync_dir_bidirectional( instance["project_uid"], workspace, user ) if counts["exported"] or counts["imported"]: store.record_event( instance, "sync", "service", "system", {"exported": counts["exported"], "imported": counts["imported"]}, ) return counts def add_schedule(instance: dict, action: str, schedule: Schedule) -> dict: if action not in ("start", "stop"): raise ContainerError("schedule action must be start or stop") first = schedule.first_run(now_utc()) return store.create_schedule(instance, action, schedule.columns(), to_iso(first)) # ---------------- aggregation ---------------- def _percentile(values: list, pct: float) -> float: if not values: return 0.0 ordered = sorted(values) index = min(len(ordered) - 1, int(round((pct / 100.0) * (len(ordered) - 1)))) return float(ordered[index]) def instance_stats(instance_uid: str) -> dict: metrics = store.recent_metrics(instance_uid, limit=720) cpu = [m.get("cpu_pct", 0) for m in metrics] mem = [m.get("mem_bytes", 0) for m in metrics] return { "samples": len(metrics), "cpu_avg": round(sum(cpu) / len(cpu), 2) if cpu else 0.0, "cpu_p95": round(_percentile(cpu, 95), 2), "mem_max": max(mem) if mem else 0, "mem_avg": int(sum(mem) / len(mem)) if mem else 0, } def _port_reachable(host: str, port: int, timeout: float = 0.3) -> bool: try: with socket.create_connection((host, port), timeout=timeout): return True except OSError: return False def _http_probe(host: str, port: int, timeout: float = 1.0) -> str: try: with stealth.stealth_sync_client(timeout=timeout) as client: response = client.get(f"http://{host}:{port}/") return f"HTTP {response.status_code}" except Exception as exc: # noqa: BLE001 - diagnostic, any failure is informative return f"unreachable: {type(exc).__name__}" def _net_entry(data: dict) -> dict: net = (data or {}).get("NetworkSettings") or {} if net.get("IPAddress") or net.get("Gateway"): return net for entry in (net.get("Networks") or {}).values(): if entry and (entry.get("IPAddress") or entry.get("Gateway")): return entry return {} def container_ip_from_inspect(data: dict) -> str: return (_net_entry(data).get("IPAddress") or "").strip() def container_gateway_from_inspect(data: dict) -> str: return (_net_entry(data).get("Gateway") or "").strip() def _ingress_container_port(instance: dict, port_maps: list) -> int: ingress_port = int(instance.get("ingress_port") or 0) if ingress_port: for mapping in port_maps: if int(mapping.get("container") or 0) == ingress_port: return ingress_port return 0 return int(port_maps[0].get("container") or 0) if port_maps else 0 def _host_port_for(port_maps: list, container_port: int) -> int: for mapping in port_maps: if int(mapping.get("container") or 0) == container_port: return int(mapping.get("host") or 0) return 0 def proxy_target(instance: dict) -> tuple: port_maps = json.loads(instance.get("ports_json") or "[]") container_port = _ingress_container_port(instance, port_maps) if not container_port: return None, None host_port = _host_port_for(port_maps, container_port) if config.CONTAINER_PROXY_HOST: return (config.CONTAINER_PROXY_HOST, host_port) if host_port else (None, None) if not host_port: return None, None gateway = (instance.get("container_gateway") or "").strip() return (gateway or "127.0.0.1", host_port) def instance_runtime(instance: dict) -> dict: boot = (instance.get("boot_command") or "").strip() port_maps = json.loads(instance.get("ports_json") or "[]") container_ip = (instance.get("container_ip") or "").strip() container_gateway = (instance.get("container_gateway") or "").strip() target_host, target_port = proxy_target(instance) probe_host = config.CONTAINER_PROXY_HOST or container_gateway or "127.0.0.1" ports = [] for mapping in port_maps: host_port = int(mapping.get("host") or 0) ports.append( { "container": int(mapping.get("container") or 0), "host": host_port, "proto": mapping.get("proto", "tcp"), "reachable": _port_reachable(probe_host, host_port) if host_port else False, } ) ingress_serving = ( _http_probe(target_host, target_port) if target_host and target_port else "no ingress port mapped" ) return { "command": boot or "image CMD (no boot_command set)", "ports": ports, "container_ip": container_ip, "container_gateway": container_gateway, "ingress_target": ( f"{target_host}:{target_port}" if target_host and target_port else "" ), "ingress_port": int(instance.get("ingress_port") or 0), "ingress_serving": ingress_serving, "status_ok_for_ingress": instance.get("status") == store.ST_RUNNING, "restart_count": int(instance.get("restart_count") or 0), "exit_code": instance.get("exit_code"), "container_id": (instance.get("container_id") or "")[:12], }