# retoor <retoor@molodetz.nl>
from __future__ import annotations
import os
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from devplacepy import config
from devplacepy.database import get_table
from devplacepy.services.base import BaseService, ConfigField
from devplacepy.services.containers import store
from devplacepy.services.containers.workspace import (
certs,
flags,
quota,
tunnels,
)
DISK_SAMPLE_DEFAULT_MINUTES = 10
def _now() -> datetime:
return datetime.now(timezone.utc)
def _parse(value: str | None) -> datetime | None:
if not value:
return None
try:
parsed = datetime.fromisoformat(value)
except (TypeError, ValueError):
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed
def _minutes_since(value: str | None) -> float | None:
parsed = _parse(value)
if parsed is None:
return None
return (_now() - parsed).total_seconds() / 60.0
def _directory_size(path: Path) -> int:
total = 0
stack = [path]
while stack:
current = stack.pop()
try:
with os.scandir(current) as entries:
for entry in entries:
try:
if entry.is_dir(follow_symlinks=False):
stack.append(Path(entry.path))
elif entry.is_file(follow_symlinks=False):
total += entry.stat().st_size
except OSError:
continue
except OSError:
continue
return total
class WorkspaceService(BaseService):
title = "Dev Workspaces"
description = (
"Reconciles browser IDE workspaces: tunnel certificates, disk and egress "
"metering, abuse flags, idle and retention lifecycle, and purge sweeps."
)
default_enabled = False
min_interval = 5
METRICS_SECONDS = 30
config_fields = [
ConfigField("workspace_enabled", "Enabled", type="bool", default="0",
group="General", help="Master switch for the workspace feature."),
ConfigField("workspace_editor_version", "Editor version", default="",
group="General",
help="code-server release. Empty means the image default."),
ConfigField("workspace_extensions_gallery", "Extensions gallery", type="text",
default="", group="General",
help="EXTENSIONS_GALLERY JSON. Empty means the default."),
ConfigField("workspace_molohttp_base_url", "molohttp base URL", type="url",
default="https://pravda.education/molohttp", group="molohttp"),
ConfigField("workspace_molohttp_api_key", "molohttp API key", type="password",
default="", group="molohttp", secret=True,
help="Preferred credential. Sent as the x-api-key header."),
ConfigField("workspace_molohttp_username", "molohttp username", default="",
group="molohttp"),
ConfigField("workspace_molohttp_password", "molohttp password",
type="password", default="", group="molohttp", secret=True),
ConfigField("workspace_tunnel_domain", "Tunnel domain",
default=config.WORKSPACE_TUNNEL_DOMAIN, group="Tunnels"),
ConfigField("workspace_hostname_pattern", "Hostname pattern",
default="{name}.{domain}", group="Tunnels"),
ConfigField("workspace_port_hostname_pattern", "Port hostname pattern",
default="{port}-{name}.{domain}", group="Tunnels"),
ConfigField("workspace_cert_mode", "Certificate mode", type="select",
default="per_host", options=[{"value": "per_host", "label": "Per host"}, {"value": "wildcard", "label": "Wildcard"}],
group="Tunnels"),
ConfigField("workspace_acme_email", "ACME email", default="", group="Tunnels"),
ConfigField("workspace_max_per_user", "Max workspaces per user", type="int",
default="2", minimum=0, group="Quotas"),
ConfigField("workspace_max_tunnels", "Max tunnels per workspace", type="int",
default="5", minimum=0, group="Quotas"),
ConfigField("workspace_disk_quota_mb", "Disk quota (MB)", type="int",
default="2048", minimum=0, group="Quotas"),
ConfigField("workspace_egress_quota_mb", "Egress quota (MB)", type="int",
default="10240", minimum=0, group="Quotas"),
ConfigField("workspace_disk_warn_percent", "Disk warn percent", type="int",
default="80", minimum=1, maximum=100, group="Quotas"),
ConfigField("workspace_idle_warn_minutes", "Idle warn (minutes)", type="int",
default="45", minimum=1, group="Lifecycle"),
ConfigField("workspace_idle_stop_minutes", "Idle stop (minutes)", type="int",
default="60", minimum=1, group="Lifecycle"),
ConfigField("workspace_retention_days", "Retention (days)", type="int",
default="14", minimum=1, group="Lifecycle"),
ConfigField("workspace_purge_after_days", "Purge after (days)", type="int",
default="7", minimum=1, group="Lifecycle"),
ConfigField("workspace_auto_stop", "Auto stop", type="select", default="auto",
options=[{"value": "off", "label": "Off"}, {"value": "notify", "label": "Notify"}, {"value": "auto", "label": "Auto"}], group="Automation"),
ConfigField("workspace_auto_delete", "Auto delete", type="select",
default="auto", options=[{"value": "off", "label": "Off"}, {"value": "notify", "label": "Notify"}, {"value": "auto", "label": "Auto"}],
group="Automation"),
ConfigField("workspace_auto_suspend", "Auto suspend", type="select",
default="notify", options=[{"value": "off", "label": "Off"}, {"value": "notify", "label": "Notify"}, {"value": "auto", "label": "Auto"}],
group="Automation"),
ConfigField("workspace_auto_flag", "Auto flag", type="select", default="auto",
options=[{"value": "off", "label": "Off"}, {"value": "flag", "label": "Flag"}, {"value": "auto", "label": "Auto"}], group="Automation"),
ConfigField("workspace_flag_cpu_percent", "Flag CPU percent", type="int",
default="95", minimum=1, maximum=100, group="Abuse"),
ConfigField("workspace_flag_cpu_minutes", "Flag CPU minutes", type="int",
default="120", minimum=1, group="Abuse"),
ConfigField("workspace_flag_egress_mb_per_hour", "Flag egress MB/hour",
type="int", default="2048", minimum=1, group="Abuse"),
ConfigField("workspace_flag_request_rate", "Flag request rate", type="int",
default="6000", minimum=1, group="Abuse"),
ConfigField("workspace_disk_sample_minutes", "Disk sample (minutes)",
type="int", default=str(DISK_SAMPLE_DEFAULT_MINUTES), minimum=1,
group="Advanced"),
]
def __init__(self) -> None:
super().__init__("workspace", interval_seconds=30)
self._last_disk_sample = 0.0
def _workspaces(self) -> list[dict]:
table = get_table("instances")
return list(table.find(is_workspace=1, deleted_at=None))
async def run_once(self) -> None:
cfg = self.get_config()
if cfg.get("workspace_enabled") not in (True, "1", 1):
return
rows = self._workspaces()
for phase in (
self._sample_disk,
self._evaluate_flags,
self._advance_lifecycle,
self._sweep_purge,
):
try:
phase(rows, cfg)
except Exception as error:
self.log(f"{phase.__name__} failed: {error}")
try:
await self._issue_tunnel_certificates()
except Exception as error:
self.log(f"_issue_tunnel_certificates failed: {error}")
async def _issue_tunnel_certificates(self) -> None:
if not certs.configured():
return
for row in tunnels.awaiting_certificate():
await certs.certify(row, self.log)
def _sample_disk(self, rows: list[dict], cfg: dict) -> None:
interval = int(cfg.get("workspace_disk_sample_minutes") or
DISK_SAMPLE_DEFAULT_MINUTES) * 60
if time.monotonic() - self._last_disk_sample < interval:
return
self._last_disk_sample = time.monotonic()
for row in rows:
workspace = config.CONTAINER_WORKSPACES_DIR / row.get("project_uid", "")
state = config.WORKSPACE_STATE_DIR / row["uid"]
total = _directory_size(workspace) + _directory_size(state)
store.update_instance(
row["uid"],
{"disk_bytes": total, "disk_sampled_at": _now().isoformat()},
)
row["disk_bytes"] = total
def _evaluate_flags(self, rows: list[dict], cfg: dict) -> None:
mode = cfg.get("workspace_auto_flag") or "auto"
if mode == "off":
return
egress_ceiling = int(cfg.get("workspace_flag_egress_mb_per_hour") or 2048)
egress_bytes = egress_ceiling * 1024 * 1024
requests_ceiling = int(cfg.get("workspace_flag_request_rate") or 6000)
for row in rows:
limits = quota.resolve(row.get("workspace_owner_uid", ""), row)
used = int(row.get("egress_bytes") or 0)
if used > egress_bytes:
flags.raise_flag(
row,
flags.KIND_EGRESS,
"warn",
f"egress {used} bytes exceeds hourly ceiling {egress_bytes}",
float(used),
float(egress_bytes),
)
if int(row.get("request_count") or 0) > requests_ceiling:
flags.raise_flag(
row,
flags.KIND_REQUESTS,
"warn",
"request rate above configured ceiling",
float(row.get("request_count") or 0),
float(requests_ceiling),
)
disk_quota = limits.disk_quota_bytes()
if disk_quota and int(row.get("disk_bytes") or 0) > disk_quota:
flags.raise_flag(
row,
flags.KIND_DISK,
"warn",
"disk usage above quota",
float(row.get("disk_bytes") or 0),
float(disk_quota),
)
def _advance_lifecycle(self, rows: list[dict], cfg: dict) -> None:
from devplacepy.utils import create_notification
stop_mode = cfg.get("workspace_auto_stop") or "auto"
delete_mode = cfg.get("workspace_auto_delete") or "auto"
for row in rows:
if row.get("suspended_at"):
continue
limits = quota.resolve(row.get("workspace_owner_uid", ""), row)
owner = row.get("workspace_owner_uid", "")
idle = _minutes_since(row.get("last_active_at"))
if idle is None:
continue
running = row.get("status") == "running"
if running and stop_mode != "off":
if idle >= limits.idle_stop_minutes:
if stop_mode == "auto":
store.update_instance(
row["uid"], {"desired_state": "stopped"}
)
if owner:
create_notification(
owner,
"workspace",
f"Workspace {row.get('name', '')} stopped after "
f"{limits.idle_stop_minutes} minutes idle.",
row["uid"],
f"/projects/{row.get('project_uid', '')}/workspace",
)
store.update_instance(row["uid"], {"idle_warned_at": ""})
elif idle >= limits.idle_warn_minutes and not row.get("idle_warned_at"):
store.update_instance(
row["uid"], {"idle_warned_at": _now().isoformat()}
)
if owner:
remaining = int(limits.idle_stop_minutes - idle)
create_notification(
owner,
"workspace",
f"Workspace {row.get('name', '')} stops in about "
f"{remaining} minutes unless you use it.",
row["uid"],
f"/projects/{row.get('project_uid', '')}/workspace",
)
if delete_mode == "off" or running:
continue
idle_days = idle / (60 * 24)
warn_at = max(1, limits.retention_days - 3)
if idle_days >= limits.retention_days:
if delete_mode == "auto":
store.delete_instance(row["uid"], "system")
tunnels.suspend_for_instance(row["uid"])
if owner:
create_notification(
owner,
"workspace",
f"Workspace {row.get('name', '')} was removed after "
f"{limits.retention_days} days idle. An administrator can "
"restore it from Trash.",
row["uid"],
"/projects",
)
elif idle_days >= warn_at and not row.get("delete_warned_at"):
store.update_instance(
row["uid"], {"delete_warned_at": _now().isoformat()}
)
if owner:
due = _now() + timedelta(days=limits.retention_days - idle_days)
create_notification(
owner,
"workspace",
f"Workspace {row.get('name', '')} is scheduled for deletion "
f"on {due.strftime('%d/%m/%Y')} unless you use it.",
row["uid"],
f"/projects/{row.get('project_uid', '')}/workspace",
)
def _sweep_purge(self, rows: list[dict], cfg: dict) -> None:
limits = quota.resolve()
cutoff = _now() - timedelta(days=limits.purge_after_days)
table = get_table("instances")
for row in table.find(is_workspace=1):
deleted = _parse(row.get("deleted_at"))
if not deleted or deleted > cutoff:
continue
for tunnel in tunnels.list_for_instance(row["uid"]):
tunnels.soft_delete(tunnel["uid"], "system")
def collect_metrics(self) -> dict:
rows = self._workspaces()
running = [r for r in rows if r.get("status") == "running"]
suspended = [r for r in rows if r.get("suspended_at")]
disk = sum(int(r.get("disk_bytes") or 0) for r in rows)
egress = sum(int(r.get("egress_bytes") or 0) for r in rows)
open_flags = flags.list_flags()
tunnel_rows = list(get_table("tunnels").find(deleted_at=None))
by_status: dict[str, int] = {}
for row in tunnel_rows:
key = row.get("status") or "unknown"
by_status[key] = by_status.get(key, 0) + 1
return {
"stats": [
{"label": "Workspaces", "value": len(rows)},
{"label": "Running", "value": len(running)},
{"label": "Suspended", "value": len(suspended)},
{"label": "Disk MB", "value": disk // (1024 * 1024)},
{"label": "Egress MB", "value": egress // (1024 * 1024)},
{"label": "Open flags", "value": len(open_flags)},
{"label": "Tunnels", "value": len(tunnel_rows)},
],
"table": {
"columns": ["Tunnel status", "Count"],
"rows": [[key, value] for key, value in sorted(by_status.items())],
},
}
View Git Blame Copy Permalink