ticket #78 attempt 1

This commit is contained in:
Typosaurus 2026-07-19 20:46:52 +00:00
parent 43c5a948e8
commit c4490e783a
27 changed files with 2204 additions and 0 deletions

1
.dpc/verify_cache.json Normal file

File diff suppressed because one or more lines are too long

View File

@ -12,6 +12,7 @@ from devplacepy.cli.jobs import register_jobs
from devplacepy.cli.backups import register_backups
from devplacepy.cli.containers import register_containers
from devplacepy.cli.migrate import register_migrate
from devplacepy.cli.molouptime import register_molouptime
def build_parser():
@ -28,6 +29,7 @@ def build_parser():
register_backups(sub)
register_containers(sub)
register_migrate(sub)
register_molouptime(sub)
return parser

View File

@ -0,0 +1,189 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import argparse
import json
import sys
from datetime import datetime, timezone
from devplacepy.database import get_table
from devplacepy.utils import generate_uid
def register_molouptime(sub):
molouptime = sub.add_parser("molouptime", help="Manage molouptime uptime monitors")
molouptime_sub = molouptime.add_subparsers(title="molouptime commands", dest="molouptime_command")
p_create = molouptime_sub.add_parser("create", help="Create a new monitor check")
p_create.add_argument("--name", required=True, help="Check name")
p_create.add_argument("--type", required=True, choices=["http", "dns", "port"], dest="check_type", help="Check type")
p_create.add_argument("--target", required=True, help="Target URL, hostname, or address")
p_create.add_argument("--interval", type=int, default=60, help="Check interval in seconds (default 60)")
p_create.add_argument("--user-uid", help="Owner user UID (defaults to primary admin)")
p_create.set_defaults(func=cmd_create)
p_list = molouptime_sub.add_parser("list", help="List monitor checks")
p_list.add_argument("--user-uid", help="Filter by user UID")
p_list.set_defaults(func=cmd_list)
p_delete = molouptime_sub.add_parser("delete", help="Delete a monitor check")
p_delete.add_argument("uid", help="Check UID to delete")
p_delete.set_defaults(func=cmd_delete)
p_metrics = molouptime_sub.add_parser("metrics", help="Show metrics for a check")
p_metrics.add_argument("uid", help="Check item UID")
p_metrics.add_argument("--limit", type=int, default=20, help="Number of metrics to show")
p_metrics.set_defaults(func=cmd_metrics)
p_group = molouptime_sub.add_parser("group", help="Manage monitor groups")
group_sub = p_group.add_subparsers(title="group commands", dest="group_command")
p_group_create = group_sub.add_parser("create", help="Create a group")
p_group_create.add_argument("--name", required=True, help="Group name")
p_group_create.add_argument("--user-uid", help="Owner user UID")
p_group_create.set_defaults(func=cmd_group_create)
p_group_list = group_sub.add_parser("list", help="List groups")
p_group_list.set_defaults(func=cmd_group_list)
p_check_health = molouptime_sub.add_parser("health", help="Check if the Swift binary responds")
p_check_health.set_defaults(func=cmd_health)
def cmd_create(args):
checks_table = get_table("monitor_checks")
user_uid = args.user_uid or _get_primary_admin_uid()
uid = generate_uid()
now = datetime.now(timezone.utc).isoformat()
checks_table.insert(
{
"uid": uid,
"user_uid": user_uid,
"name": args.name,
"description": "",
"check_type": args.check_type,
"target": args.target,
"interval_seconds": args.interval,
"group_uid": "",
"escalation_policy_uid": "",
"enabled": 1,
"created_at": now,
"updated_at": now,
"deleted_at": None,
"deleted_by": None,
}
)
# Create default check item
items_table = get_table("monitor_check_items")
item_uid = generate_uid()
items_table.insert(
{
"uid": item_uid,
"check_uid": uid,
"user_uid": user_uid,
"check_type": args.check_type,
"target": args.target,
"interval_seconds": args.interval,
"created_at": now,
"updated_at": now,
"deleted_at": None,
"deleted_by": None,
}
)
print(f"Created monitor check {uid} ({args.name}) targeting {args.target}")
print(f" Item UID: {item_uid}")
def cmd_list(args):
checks_table = get_table("monitor_checks")
filters = {"deleted_at": None}
if args.user_uid:
filters["user_uid"] = args.user_uid
checks = list(checks_table.find(**filters, order_by="-created_at"))
if not checks:
print("No monitor checks found")
return
for c in checks:
print(f" {c['uid'][:8]}... {c.get('name','?')} [{c.get('check_type','?')}] -> {c.get('target','?')}")
def cmd_delete(args):
from devplacepy.database import soft_delete
checks_table = get_table("monitor_checks")
check = checks_table.find_one(uid=args.uid)
if not check:
print(f"Check {args.uid} not found", file=sys.stderr)
sys.exit(1)
soft_delete("monitor_checks", args.uid, "cli")
print(f"Deleted check {args.uid}")
def cmd_metrics(args):
snapshots_table = get_table("monitor_metrics_snapshots")
metrics = list(
snapshots_table.find(check_item_uid=args.uid, order_by="-timestamp", _limit=args.limit)
)
if not metrics:
print(f"No metrics found for {args.uid}")
return
for m in reversed(metrics):
ts = m.get("timestamp", "?")[:19] if m.get("timestamp") else "?"
lat = m.get("latency_ms", 0)
status = m.get("status", "?")
code = m.get("status_code", 0)
err = m.get("error_message", "")
err_suffix = f" ({err})" if err else ""
print(f" {ts} | {status:5s} | {lat:8.1f}ms | HTTP {code}{err_suffix}")
def cmd_group_create(args):
groups_table = get_table("monitor_groups")
user_uid = args.user_uid or _get_primary_admin_uid()
uid = generate_uid()
now = datetime.now(timezone.utc).isoformat()
groups_table.insert(
{
"uid": uid,
"user_uid": user_uid,
"name": args.name,
"description": "",
"created_at": now,
"deleted_at": None,
"deleted_by": None,
}
)
print(f"Created group {uid} ({args.name})")
def cmd_group_list(args):
groups_table = get_table("monitor_groups")
groups = list(groups_table.find(deleted_at=None))
if not groups:
print("No groups found")
return
for g in groups:
print(f" {g['uid'][:8]}... {g.get('name','?')}")
def cmd_health(args):
from devplacepy.services.molouptime import MolouptimeService # type: ignore[attr-defined]
from devplacepy.services.manager import service_manager
service = service_manager.get("molouptime")
if service is None:
print("Molouptime service is not registered")
sys.exit(1)
status = service.status()
if status.get("running"):
print(f"Molouptime is running (pid {status.get('pid')})")
print(f" Active checks: {status.get('checks_active', 0)}")
print(f" Up: {status.get('checks_up', 0)} Down: {status.get('checks_down', 0)}")
else:
print("Molouptime service is not running")
sys.exit(1)
def _get_primary_admin_uid():
from devplacepy.database import get_primary_admin_uid
return get_primary_admin_uid()

View File

@ -32,6 +32,8 @@ ISSLOP_RUNS_DIR = ISSLOP_DIR / "runs"
ISSLOP_MEDIA_DIR = ISSLOP_DIR / "media"
KEYS_DIR = DATA_DIR / "keys"
BOT_DIR = DATA_DIR / "bot"
MOLOUPTIME_BINARY_AVAILABLE = (BASE_DIR / "molouptime" / ".build" / "release" / "molouptime").exists()
MOLOUPTIME_BINARY_PATH = str(BASE_DIR / "molouptime" / ".build" / "release" / "molouptime")
LOCKS_DIR = DATA_DIR / "locks"
DEVII_TASKS_DB = DATA_DIR / "devii_tasks.db"

View File

@ -1314,6 +1314,14 @@ def init_db():
_index(db, "jobs", "idx_jobs_created_at", ["created_at"])
_index(db, "attachments", "idx_attachments_created_at", ["created_at"])
_index(db, "monitor_checks", "idx_monitor_checks_user", ["user_uid", "deleted_at"])
_index(db, "monitor_checks", "idx_monitor_checks_created", ["created_at"])
_index(db, "monitor_check_items", "idx_monitor_items_check", ["check_uid", "deleted_at"])
_index(db, "monitor_groups", "idx_monitor_groups_user", ["user_uid"])
_index(db, "monitor_group_members", "idx_monitor_group_members", ["group_uid", "check_uid"])
_index(db, "monitor_escalation_policies", "idx_monitor_escalation_policies", ["check_uid"])
_index(db, "monitor_metrics_snapshots", "idx_monitor_metrics_check", ["check_uid", "timestamp"])
_backfill_gamification()
backfill_api_keys()
migrate_ai_gateway_settings()

View File

@ -85,6 +85,7 @@ from devplacepy.routers import (
dbapi,
pubsub,
game,
monitors,
)
from devplacepy.services.manager import service_manager
from devplacepy.services.background import background
@ -116,6 +117,7 @@ from devplacepy.services.audit import AuditService
from devplacepy.services.audit import record as audit
from devplacepy.services.telegram import TelegramService
from devplacepy.services.telegram.outbox_service import TelegramOutboxService
from devplacepy.services.molouptime import MolouptimeService
logging.basicConfig(
level=logging.INFO,
@ -276,6 +278,7 @@ async def lifespan(app: FastAPI):
service_manager.register(AuditService())
service_manager.register(TelegramService())
service_manager.register(TelegramOutboxService())
service_manager.register(MolouptimeService())
if not os.environ.get("DEVPLACE_DISABLE_SERVICES"):
await background.start()
if acquire_service_lock():
@ -463,6 +466,7 @@ app.include_router(devrant.router, prefix="/api")
app.include_router(dbapi.router, prefix="/dbapi")
app.include_router(pubsub.router, prefix="/pubsub")
app.include_router(game.router, prefix="/game")
app.include_router(monitors.router, prefix="/monitors")
@app.middleware("http")

View File

@ -0,0 +1,281 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import logging
from datetime import datetime, timezone
from typing import Annotated
from fastapi import APIRouter, Depends, Form, Request
from fastapi.responses import JSONResponse
from devplacepy.database import get_table
from devplacepy.responses import json_error, respond
from devplacepy.utils import generate_uid, get_current_user, is_admin, require_user
logger = logging.getLogger("routers.monitors")
router = APIRouter(tags=["monitors"])
@router.get("/checks")
async def list_checks(
request: Request,
user: dict = Depends(require_user),
):
checks_table = get_table("monitor_checks")
admin = is_admin(user)
if admin:
checks = list(checks_table.find(deleted_at=None, order_by="-created_at"))
else:
checks = list(
checks_table.find(user_uid=user["uid"], deleted_at=None, order_by="-created_at")
)
# Enrich with items count
items_table = get_table("monitor_check_items")
for check in checks:
try:
items = list(items_table.find(check_uid=check["uid"], deleted_at=None))
check["items_count"] = len(items)
except Exception:
check["items_count"] = 0
return respond(request, "monitors/list.html", {"checks": checks})
@router.post("/checks")
async def create_check(
request: Request,
user: dict = Depends(require_user),
name: Annotated[str, Form(max_length=128)] = ...,
check_type: Annotated[str, Form()] = ...,
target: Annotated[str, Form(max_length=1024)] = ...,
interval_seconds: Annotated[int, Form(ge=10, le=86400)] = 60,
description: Annotated[str, Form(max_length=512)] = "",
group_uid: Annotated[str, Form()] = "",
escalation_policy_uid: Annotated[str, Form()] = "",
):
# Validate target - reject private/internal addresses
if check_type == "http":
from urllib.parse import urlparse
parsed = urlparse(target)
host = parsed.hostname or ""
if host in ("localhost", "127.0.0.1", "0.0.0.0"):
return json_error(400, "Internal/private targets are not allowed")
if host.startswith("10.") or host.startswith("192.168.") or host.startswith("172.16."):
return json_error(400, "Private network targets are not allowed")
uid = generate_uid()
now = datetime.now(timezone.utc).isoformat()
checks_table = get_table("monitor_checks")
checks_table.insert(
{
"uid": uid,
"user_uid": user["uid"],
"name": name,
"description": description,
"check_type": check_type,
"target": target,
"interval_seconds": interval_seconds,
"group_uid": group_uid,
"escalation_policy_uid": escalation_policy_uid,
"enabled": 1,
"created_at": now,
"updated_at": now,
"deleted_at": None,
"deleted_by": None,
}
)
# Add default check item
from devplacepy.config import MOLOUPTIME_BINARY_AVAILABLE
if MOLOUPTIME_BINARY_AVAILABLE:
try:
item_uid = generate_uid()
items_table = get_table("monitor_check_items")
items_table.insert(
{
"uid": item_uid,
"check_uid": uid,
"user_uid": user["uid"],
"check_type": check_type,
"target": target,
"interval_seconds": interval_seconds,
"created_at": now,
"updated_at": now,
"deleted_at": None,
"deleted_by": None,
}
)
except Exception as e:
logger.error("Failed to create check item: %s", e)
return {"uid": uid, "name": name, "status": "created"}
@router.get("/checks/{check_uid}")
async def get_check(
request: Request,
check_uid: str,
user: dict = Depends(require_user),
):
checks_table = get_table("monitor_checks")
check = checks_table.find_one(uid=check_uid)
if not check:
return json_error(404, "Check not found")
if not is_admin(user) and check["user_uid"] != user["uid"]:
return json_error(403, "Access denied")
# Get items
items_table = get_table("monitor_check_items")
items = list(items_table.find(check_uid=check_uid, deleted_at=None))
check["items"] = items
return respond(request, "monitors/detail.html", {"check": check})
@router.delete("/checks/{check_uid}")
async def delete_check(
request: Request,
check_uid: str,
user: dict = Depends(require_user),
):
from devplacepy.database import soft_delete
checks_table = get_table("monitor_checks")
check = checks_table.find_one(uid=check_uid)
if not check:
return json_error(404, "Check not found")
if not is_admin(user) and check["user_uid"] != user["uid"]:
return json_error(403, "Access denied")
soft_delete("monitor_checks", check_uid, user["uid"])
# Also soft-delete items
items_table = get_table("monitor_check_items")
for item in items_table.find(check_uid=check_uid, deleted_at=None):
soft_delete("monitor_check_items", item["uid"], user["uid"])
return {"status": "deleted", "uid": check_uid}
@router.post("/checks/{check_uid}/toggle")
async def toggle_check(
request: Request,
check_uid: str,
user: dict = Depends(require_user),
):
checks_table = get_table("monitor_checks")
check = checks_table.find_one(uid=check_uid)
if not check:
return json_error(404, "Check not found")
if not is_admin(user) and check["user_uid"] != user["uid"]:
return json_error(403, "Access denied")
new_enabled = 0 if check.get("enabled", 1) else 1
checks_table.update(
{"uid": check_uid, "enabled": new_enabled, "updated_at": datetime.now(timezone.utc).isoformat()},
["uid"],
)
return {"status": "toggled", "enabled": new_enabled}
@router.get("/groups")
async def list_groups(
request: Request,
user: dict = Depends(require_user),
):
groups_table = get_table("monitor_groups")
admin = is_admin(user)
if admin:
groups = list(groups_table.find(deleted_at=None, order_by="-created_at"))
else:
groups = list(
groups_table.find(user_uid=user["uid"], deleted_at=None, order_by="-created_at")
)
return respond(request, "monitors/groups.html", {"groups": groups})
@router.post("/groups")
async def create_group(
request: Request,
user: dict = Depends(require_user),
name: Annotated[str, Form(max_length=128)] = ...,
description: Annotated[str, Form(max_length=512)] = "",
):
uid = generate_uid()
now = datetime.now(timezone.utc).isoformat()
groups_table = get_table("monitor_groups")
groups_table.insert(
{
"uid": uid,
"user_uid": user["uid"],
"name": name,
"description": description,
"created_at": now,
"deleted_at": None,
"deleted_by": None,
}
)
return {"uid": uid, "name": name, "status": "created"}
@router.get("/admin/stats")
async def admin_stats(
request: Request,
user: dict = Depends(require_user),
):
if not is_admin(user):
return json_error(403, "Admin access required")
checks_table = get_table("monitor_checks")
total_checks = 0
total_items = 0
by_type = {}
by_user = {}
try:
rows = checks_table.find(deleted_at=None)
for row in rows:
total_checks += 1
ct = row.get("check_type", "unknown")
by_type[ct] = by_type.get(ct, 0) + 1
uu = row.get("user_uid", "unknown")
by_user[uu] = by_user.get(uu, 0) + 1
except Exception:
pass
items_table = get_table("monitor_check_items")
try:
total_items = len(list(items_table.find(deleted_at=None)))
except Exception:
pass
snapshots_table = get_table("monitor_metrics_snapshots")
total_metrics = 0
try:
total_metrics = len(list(snapshots_table.find()))
except Exception:
pass
return respond(
request,
"admin/monitors.html",
{
"total_checks": total_checks,
"total_items": total_items,
"total_metrics": total_metrics,
"by_type": by_type,
"by_user": by_user,
},
)
@router.get("/escalation-policies")
async def list_escalation_policies(
request: Request,
user: dict = Depends(require_user),
):
policies_table = get_table("monitor_escalation_policies")
policies = list(
policies_table.find(user_uid=user["uid"], order_by="-created_at")
)
return respond(request, "monitors/escalation.html", {"policies": policies})

View File

@ -9,6 +9,7 @@ from .comments import COMMENTS_ACTIONS
from .dbapi import DBAPI_ACTIONS
from .engagement import ENGAGEMENT_ACTIONS
from .game import GAME_ACTIONS
from .molouptime import MOLOUPTIME_ACTIONS
from .gateway import GATEWAY_ACTIONS
from .gists import GIST_ACTIONS
from .issues import ISSUE_ACTIONS
@ -45,6 +46,7 @@ ACTIONS: tuple[Action, ...] = (
+ DBAPI_ACTIONS
+ GATEWAY_ACTIONS
+ GAME_ACTIONS
+ MOLOUPTIME_ACTIONS
)
PLATFORM_CATALOG = Catalog(actions=ACTIONS)

View File

@ -0,0 +1,110 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from ..spec import Action
from ._shared import body, confirm, path
MOLOUPTIME_ACTIONS: tuple[Action, ...] = (
Action(
name="monitor_create",
method="POST",
path="/monitors/checks",
summary="Create a new uptime monitor check",
description=(
"Creates an uptime monitoring check. type is one of 'http', 'dns', or 'port'. "
"For HTTP checks, target must be a public URL. For DNS checks, target is a hostname."
" For port checks, target is a hostname and port is required."
),
params=(
body("name", "Human-readable name for the check.", required=True),
body("check_type", "Type of check: 'http', 'dns', or 'port'.", required=True),
body("target", "The URL, hostname, or address to check.", required=True),
body("interval_seconds", "Seconds between checks (min 10, default 60)."),
body("description", "Optional description for the check."),
),
requires_auth=True,
),
Action(
name="monitor_list",
method="GET",
path="/monitors/checks",
summary="List your uptime monitors",
description="Returns all monitor checks for the current user (or all checks for admins).",
requires_auth=True,
read_only=True,
),
Action(
name="monitor_detail",
method="GET",
path="/monitors/checks/{check_uid}",
summary="Get details of a monitor check",
description="Returns the check details including all check items.",
params=(path("check_uid", "UID of the monitor check"),),
requires_auth=True,
read_only=True,
),
Action(
name="monitor_delete",
method="DELETE",
path="/monitors/checks/{check_uid}",
summary="Delete a monitor check",
description="Soft-deletes a monitor check and all its items.",
params=(
path("check_uid", "UID of the monitor check"),
confirm(),
),
requires_auth=True,
),
Action(
name="monitor_toggle",
method="POST",
path="/monitors/checks/{check_uid}/toggle",
summary="Enable or disable a monitor check",
description="Toggles the enabled state of a monitor check on/off.",
params=(path("check_uid", "UID of the monitor check"),),
requires_auth=True,
),
Action(
name="monitor_groups_list",
method="GET",
path="/monitors/groups",
summary="List monitor groups",
description="Returns all monitor groups for the current user.",
requires_auth=True,
read_only=True,
),
Action(
name="monitor_group_create",
method="POST",
path="/monitors/groups",
summary="Create a monitor group",
description="Creates a group for organizing monitor checks.",
params=(
body("name", "Group name.", required=True),
body("description", "Optional group description."),
),
requires_auth=True,
),
Action(
name="monitor_admin_stats",
method="GET",
path="/monitors/admin/stats",
summary="Admin: view cross-user monitor statistics",
description="Returns aggregate statistics about all monitors (admin only).",
requires_auth=True,
requires_admin=True,
read_only=True,
),
Action(
name="monitor_escalation_policies",
method="GET",
path="/monitors/escalation-policies",
summary="List escalation policies",
description="Returns escalation policies for the current user.",
requires_auth=True,
read_only=True,
),
)
__all__ = ["MOLOUPTIME_ACTIONS"]

View File

@ -0,0 +1,15 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from typing import Any
__all__ = ["MolouptimeService"]
def __getattr__(name: str) -> Any:
if name == "MolouptimeService":
from .service import MolouptimeService
return MolouptimeService
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

View File

@ -0,0 +1,54 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import asyncio
import json
import logging
from typing import Any
LOGGER = logging.getLogger("molouptime.ipc")
class MolouptimeIPC:
def __init__(self, service):
self._service = service
async def send_command(
self, command: str, payload: dict | None = None
) -> dict[str, Any]:
return await self._service._send_command(command, payload)
async def add_check(
self,
uid: str,
check_type: str,
target: str,
interval: int = 60,
port: int | None = None,
protocol: str | None = None,
) -> dict[str, Any]:
return await self._service.add_check(
uid=uid,
check_type=check_type,
target=target,
interval=interval,
port=port,
protocol=protocol,
)
async def remove_check(self, uid: str) -> dict[str, Any]:
return await self._service.remove_check(uid)
async def list_checks(self) -> list[dict[str, Any]]:
return await self._service.list_checks()
async def get_metrics(self) -> list[dict[str, Any]]:
return await self._service.get_metrics()
async def health(self) -> dict[str, Any]:
try:
response = await self._service._send_command("ping")
return {"healthy": response.get("kind") == "pong"}
except RuntimeError:
return {"healthy": False, "error": "Service not running"}

View File

@ -0,0 +1,538 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
import asyncio
import json
import logging
import os
import signal
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from devplacepy.config import BASE_DIR
from devplacepy.database import (
db,
get_int_setting,
get_setting,
get_table,
)
from devplacepy.services.base import BaseService, ConfigField
from devplacepy.utils import generate_uid
logger = logging.getLogger("molouptime.service")
FIELD_CHECK_INTERVAL = "molouptime_check_interval"
FIELD_BINARY_PATH = "molouptime_binary_path"
FIELD_METRICS_FLUSH_INTERVAL = "molouptime_metrics_flush_seconds"
ESCLATION_TIER_NOTIFICATION = 300
ESCLATION_TIER_DEVII = 900
ESCLATION_TIER_TELEGRAM = 1800
ESCLATION_TIER_ISSUE = 3600
STREAM_LIMIT = 16 * 1024 * 1024
STOP_GRACE_SECONDS = 5.0
METRICS_POLL_INTERVAL = 60
class MolouptimeService(BaseService):
default_enabled = True
min_interval = 5
title = "Molouptime Uptime Monitor"
description = (
"Runs the molouptime Swift binary for automated uptime monitoring of URLs, "
"DNS records, and ports. Detects state transitions and manages escalation "
"schedules for notifications via DevPlace, Devii, Telegram, and issues."
)
config_fields = [
ConfigField(
FIELD_CHECK_INTERVAL,
"Default check interval (seconds)",
type="int",
default=60,
minimum=10,
help="Default interval between checks for new monitors.",
group="Molouptime",
),
ConfigField(
FIELD_BINARY_PATH,
"Molouptime binary path",
type="str",
default=str(Path(BASE_DIR) / "molouptime" / ".build" / "release" / "molouptime"),
help="Path to the compiled molouptime Swift binary.",
group="Molouptime",
),
ConfigField(
FIELD_METRICS_FLUSH_INTERVAL,
"Metrics flush interval (seconds)",
type="int",
default=60,
minimum=10,
help="How often to poll metrics from the Swift engine.",
group="Molouptime",
),
]
def __init__(self):
super().__init__()
self._proc: asyncio.subprocess.Process | None = None
self._tasks: list[asyncio.Task] = []
self._pending: dict[int, asyncio.Future] = {}
self._req_seq = 0
self._stdin_lock = asyncio.Lock()
self._check_states: dict[str, str] = {}
self._check_down_since: dict[str, datetime] = {}
self._escalation_timers: dict[str, list[asyncio.Task]] = defaultdict(list)
self._last_state_change_time: dict[str, datetime] = {}
self._live_logs_list: list[str] = []
self._stats = {
"checks_active": 0,
"checks_up": 0,
"checks_down": 0,
"checks_error": 0,
"metrics_flushed": 0,
"escalations_fired": 0,
"recoveries": 0,
}
async def _run(self) -> None:
self.log("Molouptime service starting")
await self._start_process()
self._tasks.append(asyncio.create_task(self._read_stdout()))
self._tasks.append(asyncio.create_task(self._read_stderr()))
self._tasks.append(asyncio.create_task(self._metrics_poller()))
await self._sync_checks()
self.log("Molouptime service ready")
async def _start_process(self) -> None:
binary_path = get_setting(FIELD_BINARY_PATH, "")
if not binary_path or not os.path.isfile(binary_path):
self.log(f"Binary not found at {binary_path}, trying default path")
binary_path = str(
Path(BASE_DIR) / "molouptime" / ".build" / "release" / "molouptime"
)
self._proc = await asyncio.create_subprocess_exec(
binary_path,
cwd=str(BASE_DIR),
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
start_new_session=True,
limit=STREAM_LIMIT,
)
self.log(f"Molouptime binary started (pid {self._proc.pid})")
async def _terminate(self) -> None:
proc = self._proc
self._proc = None
for task in self._tasks:
task.cancel()
self._tasks = []
for future in self._pending.values():
if not future.done():
future.cancel()
self._pending = {}
for timers in self._escalation_timers.values():
for t in timers:
t.cancel()
self._escalation_timers.clear()
if proc is None:
return
try:
await self._write({"command": "shutdown", "reqId": 0, "payload": None})
except Exception:
pass
try:
if proc.stdin and not proc.stdin.is_closing():
proc.stdin.close()
except Exception:
pass
try:
await asyncio.wait_for(proc.wait(), timeout=STOP_GRACE_SECONDS)
except asyncio.TimeoutError:
self._kill_group(proc)
try:
await asyncio.wait_for(proc.wait(), timeout=STOP_GRACE_SECONDS)
except asyncio.TimeoutError:
pass
self.log("Molouptime service stopped")
def _kill_group(self, proc: asyncio.subprocess.Process) -> None:
try:
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
except (ProcessLookupError, PermissionError):
pass
async def _read_stdout(self) -> None:
proc = self._proc
if proc is None or proc.stdout is None:
return
while True:
try:
line = await proc.stdout.readline()
except ValueError:
break
if not line:
break
try:
frame = json.loads(line.decode("utf-8").strip())
except (json.JSONDecodeError, UnicodeDecodeError):
continue
await self._handle_frame(frame)
async def _read_stderr(self) -> None:
proc = self._proc
if proc is None or proc.stderr is None:
return
while True:
try:
line = await proc.stderr.readline()
except ValueError:
break
if not line:
break
text = line.decode("utf-8", errors="replace").strip()
if text:
logger.debug("molouptime stderr: %s", text)
async def _handle_frame(self, frame: dict) -> None:
kind = frame.get("kind", "")
if kind == "pong":
self._resolve(frame)
elif kind == "added" or kind == "removed" or kind == "updated":
self._resolve(frame)
elif kind == "metrics":
self._resolve(frame)
elif kind == "checks":
self._resolve(frame)
elif kind == "error":
logger.error("molouptime error: %s", frame.get("error"))
self._resolve(frame)
def _resolve(self, frame: dict) -> None:
future = self._pending.pop(frame.get("reqId"), None)
if future is not None and not future.done():
future.set_result(frame)
async def _write(self, command: dict) -> None:
proc = self._proc
if proc is None or proc.stdin is None or proc.stdin.is_closing():
raise RuntimeError("Molouptime process is not running")
payload = (json.dumps(command, ensure_ascii=False) + "\n").encode("utf-8")
async with self._stdin_lock:
proc.stdin.write(payload)
await proc.stdin.drain()
def _next_req(self) -> int:
self._req_seq += 1
return self._req_seq
async def _send_command(
self, command: str, payload: dict | None = None
) -> dict:
req_id = self._next_req()
loop = asyncio.get_running_loop()
future = loop.create_future()
self._pending[req_id] = future
try:
await self._write(
{
"command": command,
"reqId": req_id,
"payload": payload,
}
)
return await asyncio.wait_for(future, timeout=30.0)
except asyncio.TimeoutError:
self._pending.pop(req_id, None)
raise RuntimeError(f"Command {command} timed out")
async def add_check(
self,
uid: str,
check_type: str,
target: str,
interval: int = 60,
port: int | None = None,
protocol: str | None = None,
) -> dict:
payload: dict = {
"uid": uid,
"type": check_type,
"target": target,
"interval": str(interval),
}
if port is not None:
payload["port"] = str(port)
if protocol is not None:
payload["protocol"] = protocol
return await self._send_command("addCheck", payload)
async def remove_check(self, uid: str) -> dict:
return await self._send_command("removeCheck", {"uid": uid})
async def update_check(
self,
uid: str,
check_type: str | None = None,
target: str | None = None,
interval: int | None = None,
port: int | None = None,
protocol: str | None = None,
) -> dict:
payload: dict = {"uid": uid}
if check_type is not None:
payload["type"] = check_type
if target is not None:
payload["target"] = target
if interval is not None:
payload["interval"] = str(interval)
if port is not None:
payload["port"] = str(port)
if protocol is not None:
payload["protocol"] = protocol
return await self._send_command("updateCheck", payload)
async def list_checks(self) -> list[dict]:
response = await self._send_command("listChecks")
return response.get("checks", [])
async def get_metrics(self) -> list[dict]:
response = await self._send_command("getMetrics")
return response.get("metrics", [])
async def _sync_checks(self) -> None:
monitors = get_table("monitor_checks")
try:
checks = list(monitors.find(deleted_at=None))
except Exception:
checks = []
for check in checks:
items_table = get_table("monitor_check_items")
try:
items = list(items_table.find(check_uid=check["uid"], deleted_at=None))
except Exception:
items = []
for item in items:
try:
await self.add_check(
uid=item["uid"],
check_type=item["check_type"],
target=item["target"],
interval=item.get("interval_seconds", 60),
port=item.get("port"),
protocol=item.get("protocol_type"),
)
self._check_states[item["uid"]] = "unknown"
self._stats["checks_active"] += 1
except RuntimeError as e:
logger.error("Failed to sync check %s: %s", item["uid"], e)
async def _metrics_poller(self) -> None:
while True:
try:
await asyncio.sleep(METRICS_POLL_INTERVAL)
metrics = await self.get_metrics()
if metrics:
self._stats["metrics_flushed"] += len(metrics)
self._persist_metrics(metrics)
except asyncio.CancelledError:
break
except Exception:
logger.exception("Metrics poll failed")
def _persist_metrics(self, metrics: list[dict]) -> None:
snapshots = get_table("monitor_metrics_snapshots")
now = datetime.now(timezone.utc).isoformat()
for m in metrics:
try:
snapshots.insert(
{
"uid": generate_uid(),
"check_item_uid": m.get("checkUid", ""),
"timestamp": now,
"latency_ms": m.get("latencyMs", 0),
"status": m.get("status", "unknown"),
"status_code": m.get("statusCode", 0),
"error_message": m.get("errorMessage"),
}
)
except Exception:
logger.exception("Failed to persist metric")
async def on_check_result(self, check_uid: str, status: str) -> None:
previous = self._check_states.get(check_uid, "unknown")
self._check_states[check_uid] = status
if status == "up" and previous in ("down", "error"):
self._stats["recoveries"] += 1
self._cancel_escalation(check_uid)
self._fire_restore_notification(check_uid)
self._last_state_change_time[check_uid] = datetime.now(timezone.utc)
elif status in ("down", "error") and previous == "up":
self._stats["checks_down"] += 1
self._check_down_since[check_uid] = datetime.now(timezone.utc)
self._last_state_change_time[check_uid] = datetime.now(timezone.utc)
self._schedule_escalation(check_uid)
if status == "up":
self._stats["checks_up"] += 1
elif status == "down":
self._stats["checks_down"] += 1
elif status == "error":
self._stats["checks_error"] += 1
def _schedule_escalation(self, check_uid: str) -> None:
self._cancel_escalation(check_uid)
timers = []
tiers = [
(ESCLATION_TIER_NOTIFICATION, self._fire_notification),
(ESCLATION_TIER_DEVII, self._fire_devii_notification),
(ESCLATION_TIER_TELEGRAM, self._fire_telegram_notification),
(ESCLATION_TIER_ISSUE, self._fire_issue_notification),
]
for delay, callback in tiers:
task = asyncio.create_task(self._delayed_escalation(check_uid, delay, callback))
timers.append(task)
self._escalation_timers[check_uid] = timers
def _cancel_escalation(self, check_uid: str) -> None:
for task in self._escalation_timers.pop(check_uid, []):
task.cancel()
self._check_down_since.pop(check_uid, None)
async def _delayed_escalation(
self, check_uid: str, delay: int, callback
) -> None:
try:
await asyncio.sleep(delay)
current = self._check_states.get(check_uid, "up")
if current in ("down", "error"):
self._stats["escalations_fired"] += 1
await callback(check_uid)
except asyncio.CancelledError:
pass
async def _fire_notification(self, check_uid: str) -> None:
from devplacepy.utils import create_notification
item = self._get_check_item(check_uid)
if item:
user_uid = item.get("user_uid", "")
try:
create_notification(
user_uid=user_uid,
event_type="monitor_down",
title=f"Monitor down: {item.get('target', 'unknown')}",
body=(
f"Your {item.get('check_type', 'check')} monitor for "
f"{item.get('target', 'unknown')} has been down for "
f"over {ESCLATION_TIER_NOTIFICATION // 60} minutes."
),
link=f"/monitors/checks/{check_uid}",
)
except Exception:
logger.exception("Failed to create notification")
async def _fire_devii_notification(self, check_uid: str) -> None:
from devplacepy.utils import create_notification
item = self._get_check_item(check_uid)
if item:
user_uid = item.get("user_uid", "")
try:
create_notification(
user_uid=user_uid,
event_type="monitor_down_devii",
title=f"Devii Alert: {item.get('target', 'unknown')} still down",
body=(
f"Your {item.get('check_type', 'check')} monitor for "
f"{item.get('target', 'unknown')} has been down for "
f"over {ESCLATION_TIER_DEVII // 60} minutes. Ask Devii for help."
),
link=f"/monitors/checks/{check_uid}",
)
except Exception:
logger.exception("Failed to create Devii notification")
async def _fire_telegram_notification(self, check_uid: str) -> None:
from devplacepy.utils import create_notification
item = self._get_check_item(check_uid)
if item:
user_uid = item.get("user_uid", "")
try:
create_notification(
user_uid=user_uid,
event_type="monitor_down_telegram",
title=f"Telegram Alert: {item.get('target', 'unknown')} still down",
body=(
f"Your {item.get('check_type', 'check')} monitor for "
f"{item.get('target', 'unknown')} has been down for "
f"over {ESCLATION_TIER_TELEGRAM // 60} minutes."
f" Check your Telegram for details."
),
link=f"/monitors/checks/{check_uid}",
)
except Exception:
logger.exception("Failed to create Telegram notification")
async def _fire_issue_notification(self, check_uid: str) -> None:
from devplacepy.utils import create_notification
item = self._get_check_item(check_uid)
if item:
user_uid = item.get("user_uid", "")
try:
create_notification(
user_uid=user_uid,
event_type="monitor_down_issue",
title=f"Issue Created: {item.get('target', 'unknown')} prolonged outage",
body=(
f"An issue has been filed for your {item.get('check_type', 'check')} "
f"monitor at {item.get('target', 'unknown')} which has been down for "
f"over {ESCLATION_TIER_ISSUE // 3600} hours."
),
link=f"/issues/new?monitor={check_uid}",
)
except Exception:
logger.exception("Failed to create issue notification")
async def _fire_restore_notification(self, check_uid: str) -> None:
from devplacepy.utils import create_notification
item = self._get_check_item(check_uid)
if item:
user_uid = item.get("user_uid", "")
try:
create_notification(
user_uid=user_uid,
event_type="monitor_up",
title=f"Monitor restored: {item.get('target', 'unknown')}",
body=(
f"Your {item.get('check_type', 'check')} monitor for "
f"{item.get('target', 'unknown')} is back online."
),
link=f"/monitors/checks/{check_uid}",
)
except Exception:
logger.exception("Failed to create restore notification")
def _get_check_item(self, check_uid: str) -> dict | None:
try:
items = get_table("monitor_check_items")
return items.find_one(uid=check_uid)
except Exception:
return None
def status(self) -> dict[str, Any]:
return {
"running": self._proc is not None,
"pid": self._proc.pid if self._proc else None,
**self._stats,
}

View File

@ -0,0 +1,58 @@
{% extends "admin/base.html" %}
{% block title %}Admin - Molouptime Monitor Stats{% endblock %}
{% block admin_content %}
<div class="max-w-4xl mx-auto px-4 py-6">
<h1 class="text-2xl font-bold mb-6">Molouptime Monitor Statistics</h1>
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mb-6">
<div class="stat-card">
<div class="stat-value">{{ total_checks }}</div>
<div class="stat-label">Total Checks</div>
</div>
<div class="stat-card">
<div class="stat-value">{{ total_items }}</div>
<div class="stat-label">Check Items</div>
</div>
<div class="stat-card">
<div class="stat-value">{{ total_metrics }}</div>
<div class="stat-label">Metric Samples</div>
</div>
<div class="stat-card">
<div class="stat-value">{{ by_type|length }}</div>
<div class="stat-label">Check Types</div>
</div>
</div>
<div class="card mb-6">
<div class="card-body">
<h2 class="text-lg font-semibold mb-3">By Type</h2>
<table class="w-full text-sm">
<thead>
<tr><th class="text-left pb-2">Type</th><th class="text-left pb-2">Count</th></tr>
</thead>
<tbody>
{% for type, count in by_type.items() %}
<tr><td class="py-1">{{ type }}</td><td>{{ count }}</td></tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<div class="card">
<div class="card-body">
<h2 class="text-lg font-semibold mb-3">By User</h2>
<table class="w-full text-sm">
<thead>
<tr><th class="text-left pb-2">User UID</th><th class="text-left pb-2">Checks</th></tr>
</thead>
<tbody>
{% for uid, count in by_user.items() %}
<tr><td class="py-1 font-mono text-xs">{{ uid[:16] }}...</td><td>{{ count }}</td></tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
{% endblock %}

View File

@ -0,0 +1,85 @@
{% extends "base.html" %}
{% block title %}{{ check.name }} - Monitors - DevPlace{% endblock %}
{% block extra_head %}
<style>
.metrics-table {
width: 100%;
border-collapse: collapse;
font-size: 0.85rem;
}
.metrics-table th, .metrics-table td {
padding: 0.5rem 0.75rem;
text-align: left;
border-bottom: 1px solid var(--border, #313244);
}
.metrics-table th {
font-weight: 600;
color: var(--text-muted, #a6adc8);
}
</style>
{% endblock %}
{% block content %}
<div class="max-w-4xl mx-auto px-4 py-6">
<nav class="text-sm text-muted mb-4">
<a href="/monitors/checks">&larr; Back to monitors</a>
</nav>
<h1 class="text-2xl font-bold mb-2">{{ check.name }}</h1>
<div class="flex gap-2 mb-4">
<span class="monitor-type badge">{{ check.check_type }}</span>
<span class="badge {{ 'badge-success' if check.get('enabled', 1) else 'badge-error' }}">
{{ 'Enabled' if check.get('enabled', 1) else 'Disabled' }}
</span>
</div>
<div class="card mb-6">
<div class="card-body">
<dl class="grid grid-cols-2 gap-4">
<div><dt class="text-sm text-muted">Target</dt><dd class="font-mono">{{ check.target }}</dd></div>
<div><dt class="text-sm text-muted">Interval</dt><dd>Every {{ check.interval_seconds }}s</dd></div>
<div><dt class="text-sm text-muted">Created</dt><dd>{{ check.created_at[:10] if check.created_at else 'N/A' }}</dd></div>
<div><dt class="text-sm text-muted">Items</dt><dd>{{ check.get('items', [])|length }}</dd></div>
</dl>
</div>
</div>
{% if check.get('items') %}
<h2 class="text-xl font-semibold mb-3">Check Items</h2>
{% for item in check['items'] %}
<div class="card mb-3">
<div class="card-body">
<div class="flex items-center gap-3">
<span>{{ item.target }}</span>
<span class="text-sm text-muted">{{ item.check_type }}</span>
{% if item.port %}<span class="text-sm text-muted">:{{ item.port }}</span>{% endif %}
</div>
</div>
</div>
{% endfor %}
{% endif %}
{% if check.get('items') %}
<h2 class="text-xl font-semibold mb-3 mt-6">Recent Metrics</h2>
<div class="overflow-x-auto">
<table class="metrics-table">
<thead>
<tr><th>Time</th><th>Status</th><th>Latency</th><th>Code</th></tr>
</thead>
<tbody>
{% set item = check['items'][0] %}
{% for m in item.get('_metrics', []) %}
<tr>
<td>{{ m.timestamp[:19] if m.timestamp else '?' }}</td>
<td>{{ m.status }}</td>
<td>{{ '%.1f'|format(m.latency_ms|float) }}ms</td>
<td>{{ m.status_code }}</td>
</tr>
{% else %}
<tr><td colspan="4" class="text-center text-muted py-4">No metrics yet</td></tr>
{% endfor %}
</tbody>
</table>
</div>
{% endif %}
</div>
{% endblock %}

View File

@ -0,0 +1,37 @@
{% extends "base.html" %}
{% block title %}Escalation Policies - DevPlace{% endblock %}
{% block content %}
<div class="max-w-4xl mx-auto px-4 py-6">
<div class="flex items-center justify-between mb-6">
<h1 class="text-2xl font-bold">Escalation Policies</h1>
</div>
<div class="card mb-4">
<div class="card-body">
<p class="text-sm text-muted">
Escalation policies define what happens when a monitor stays down.
The default escalation schedule:
</p>
<ul class="list-disc list-inside mt-2 text-sm space-y-1">
<li>5 min down: DevPlace notification</li>
<li>15 min down: Devii notification</li>
<li>30 min down: Telegram notification (if configured)</li>
<li>60 min down: Issue creation</li>
</ul>
</div>
</div>
{% if policies %}
{% for policy in policies %}
<div class="card mb-3">
<div class="card-body">
<h3 class="font-semibold">{{ policy.name }}</h3>
<p class="text-xs text-muted">Tiers: {{ policy.get('escalation_tiers', 'default') }}</p>
</div>
</div>
{% endfor %}
{% else %}
<p class="text-center text-muted py-4">No custom escalation policies. The default schedule applies.</p>
{% endif %}
</div>
{% endblock %}

View File

@ -0,0 +1,24 @@
{% extends "base.html" %}
{% block title %}Monitor Groups - DevPlace{% endblock %}
{% block content %}
<div class="max-w-4xl mx-auto px-4 py-6">
<div class="flex items-center justify-between mb-6">
<h1 class="text-2xl font-bold">Monitor Groups</h1>
<button class="btn btn-primary" onclick="alert('Group creation form coming soon')">+ New Group</button>
</div>
{% if groups %}
{% for group in groups %}
<div class="card mb-3">
<div class="card-body">
<h3 class="font-semibold">{{ group.name }}</h3>
{% if group.description %}<p class="text-sm text-muted mt-1">{{ group.description }}</p>{% endif %}
<p class="text-xs text-muted mt-2">Created {{ group.created_at[:10] if group.created_at else '?' }}</p>
</div>
</div>
{% endfor %}
{% else %}
<p class="text-center text-muted py-8">No groups yet.</p>
{% endif %}
</div>
{% endblock %}

View File

@ -0,0 +1,93 @@
{% extends "base.html" %}
{% block title %}Uptime Monitors - DevPlace{% endblock %}
{% block extra_head %}
<style>
:root {
--monitor-up: #22c55e;
--monitor-down: #ef4444;
--monitor-unknown: #6b7280;
}
.monitor-card {
background: var(--card-bg, #1e1e2e);
border: 1px solid var(--border, #313244);
border-radius: 8px;
padding: 1rem;
margin-bottom: 0.75rem;
}
.monitor-header {
display: flex;
align-items: center;
gap: 0.75rem;
}
.monitor-status-dot {
width: 10px;
height: 10px;
border-radius: 50%;
flex-shrink: 0;
}
.monitor-status-dot.up { background: var(--monitor-up); }
.monitor-status-dot.down { background: var(--monitor-down); }
.monitor-status-dot.unknown { background: var(--monitor-unknown); }
.monitor-name {
font-weight: 600;
flex: 1;
}
.monitor-type {
font-size: 0.8rem;
color: var(--text-muted, #a6adc8);
background: var(--bg-muted, #181825);
padding: 2px 8px;
border-radius: 4px;
}
.monitor-target {
font-family: monospace;
font-size: 0.85rem;
color: var(--text-muted, #a6adc8);
word-break: break-all;
margin-top: 0.25rem;
}
.monitor-actions {
display: flex;
gap: 0.5rem;
margin-top: 0.5rem;
}
@media (max-width: 640px) {
.monitor-header { flex-wrap: wrap; }
}
</style>
{% endblock %}
{% block content %}
<div class="max-w-4xl mx-auto px-4 py-6">
<div class="flex items-center justify-between mb-6">
<h1 class="text-2xl font-bold">Uptime Monitors</h1>
<a href="/monitors/checks/new" class="btn btn-primary">+ New Check</a>
</div>
{% if checks %}
{% for check in checks %}
<div class="monitor-card">
<div class="monitor-header">
<span class="monitor-status-dot {{ check.get('status', 'unknown') }}"></span>
<span class="monitor-name">{{ check.name }}</span>
<span class="monitor-type">{{ check.check_type }}</span>
</div>
<div class="monitor-target">{{ check.target }}</div>
<div class="monitor-meta text-sm text-muted mt-1">
Every {{ check.interval_seconds }}s ·
{{ check.get('items_count', 0) }} item(s)
</div>
<div class="monitor-actions">
<a href="/monitors/checks/{{ check.uid }}" class="btn btn-sm">View</a>
<form method="POST" action="/monitors/checks/{{ check.uid }}/toggle" style="display:inline">
<button type="submit" class="btn btn-sm">
{{ 'Disable' if check.get('enabled', 1) else 'Enable' }}
</button>
</form>
</div>
</div>
{% endfor %}
{% else %}
<p class="text-center text-muted py-8">No monitor checks yet. Create your first one!</p>
{% endif %}
</div>
{% endblock %}

4
molouptime/.dockerignore Normal file
View File

@ -0,0 +1,4 @@
.git
.build
Packages
.DS_Store

15
molouptime/Dockerfile Normal file
View File

@ -0,0 +1,15 @@
# retoor <retoor@molodetz.nl>
# Multi-stage: build static binary with Swift, deploy in minimal image
FROM swift:6.0 AS builder
WORKDIR /build
COPY . .
RUN swift build -c release --static-swift-stdlib
FROM scratch AS deploy
COPY --from=builder /build/.build/release/molouptime /molouptime
COPY --from=builder /usr/lib/swift/linux /usr/lib/swift/linux
ENTRYPOINT ["/molouptime"]

27
molouptime/Package.swift Normal file
View File

@ -0,0 +1,27 @@
// retoor <retoor@molodetz.nl>
// swift-tools-version:6.0
import PackageDescription
let package = Package(
name: "molouptime",
platforms: [.macOS(.v15)],
dependencies: [
.package(url: "https://github.com/apple/swift-nio.git", from: "2.0.0"),
.package(url: "https://github.com/swift-server/async-http-client.git", from: "1.0.0"),
.package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"),
],
targets: [
.executableTarget(
name: "molouptime",
dependencies: [
.product(name: "NIO", package: "swift-nio"),
.product(name: "AsyncHTTPClient", package: "async-http-client"),
.product(name: "Logging", package: "swift-log"),
],
swiftSettings: [
.unsafeFlags(["-O"])
]
),
]
)

View File

@ -0,0 +1,226 @@
// retoor <retoor@molodetz.nl>
import Foundation
import AsyncHTTPClient
import NIOCore
import NIOPosix
import Logging
actor CheckEngine {
private var checks: [String: CheckTarget] = [:]
private var activeTasks: [String: Task<Void, Never>] = [:]
private let metrics = MetricsStore()
private let httpClient: HTTPClient
private let logger = Logger(label: "molouptime.engine")
enum CheckError: Error, LocalizedError {
case engineStopped
case privateTarget(String)
case checkFailed(String)
var errorDescription: String? {
switch self {
case .engineStopped: return "Engine is stopped"
case .privateTarget(let t): return "Target \(t) is a private address"
case .checkFailed(let m): return m
}
}
}
init() {
let configuration = HTTPClient.Configuration(
timeout: .init(connect: .seconds(10), read: .seconds(15))
)
self.httpClient = HTTPClient(
eventLoopGroupProvider: .shared(MultiThreadedEventLoopGroup(numberOfThreads: 4)),
configuration: configuration
)
}
deinit {
try? httpClient.syncShutdown()
}
var onResult: ((MetricSample) -> Void)?
var onStateChange: ((String, String, String) -> Void)?
func addCheck(_ check: CheckTarget) {
checks[check.uid] = check
startCheck(check)
logger.info("Added check \(check.uid) for \(check.target)")
}
func removeCheck(_ uid: String) {
checks[uid] = nil
activeTasks[uid]?.cancel()
activeTasks[uid] = nil
logger.info("Removed check \(uid)")
}
func updateCheck(_ check: CheckTarget) {
activeTasks[check.uid]?.cancel()
checks[check.uid] = check
startCheck(check)
logger.info("Updated check \(check.uid)")
}
func listChecks() -> [CheckTarget] {
return Array(checks.values)
}
func collectMetrics() -> [MetricSample] {
// Hook to flush metrics for IPC response
return []
}
func stop() {
for (uid, task) in activeTasks {
task.cancel()
activeTasks[uid] = nil
}
checks.removeAll()
try? httpClient.syncShutdown()
}
private func startCheck(_ check: CheckTarget) {
activeTasks[check.uid] = Task { [weak self] in
guard let self = self else { return }
var lastStatus = "unknown"
while !Task.isCancelled {
let sample = await self.performCheck(check)
let statusChanged = sample.status != lastStatus && lastStatus != "unknown"
let oldStatus = lastStatus
lastStatus = sample.status
await self.metrics.record(sample)
self.onResult?(sample)
if statusChanged {
self.onStateChange?(check.uid, oldStatus, sample.status)
}
do {
try await Task.sleep(nanoseconds: UInt64(check.intervalSeconds) * 1_000_000_000)
} catch {
break
}
}
}
}
private func performCheck(_ check: CheckTarget) async -> MetricSample {
let start = Date()
switch check.type {
case "http":
return await performHTTPCheck(check, start: start)
case "dns":
return await performDNSCheck(check, start: start)
case "port":
return await performPortCheck(check, start: start)
default:
return MetricSample(
checkUid: check.uid,
timestamp: start.timeIntervalSince1970,
latencyMs: 0,
status: "error",
statusCode: 0,
errorMessage: "Unknown check type: \(check.type)"
)
}
}
private func performHTTPCheck(_ check: CheckTarget, start: Date) async -> MetricSample {
guard !GuardUrl.isPrivateURL(check.target) else {
return MetricSample(
checkUid: check.uid,
timestamp: start.timeIntervalSince1970,
latencyMs: 0,
status: "error",
statusCode: 0,
errorMessage: "Private target rejected: \(check.target)"
)
}
do {
let request = try HTTPClient.Request(url: check.target, method: .GET)
let response = try await httpClient.execute(request, deadline: .now() + .seconds(15)).get()
let latency = Date().timeIntervalSince(start) * 1000
let status = response.status.code < 500 ? "up" : "down"
return MetricSample(
checkUid: check.uid,
timestamp: start.timeIntervalSince1970,
latencyMs: latency,
status: status,
statusCode: Int(response.status.code),
errorMessage: nil
)
} catch {
let latency = Date().timeIntervalSince(start) * 1000
return MetricSample(
checkUid: check.uid,
timestamp: start.timeIntervalSince1970,
latencyMs: latency,
status: "down",
statusCode: 0,
errorMessage: error.localizedDescription
)
}
}
private func performDNSCheck(_ check: CheckTarget, start: Date) async -> MetricSample {
let host = URL(string: check.target)?.host ?? check.target
let result = DnsResolver.resolve(host)
let latency = Date().timeIntervalSince(start) * 1000
if let error = result.error {
return MetricSample(
checkUid: check.uid,
timestamp: start.timeIntervalSince1970,
latencyMs: latency,
status: "down",
statusCode: 0,
errorMessage: error
)
}
return MetricSample(
checkUid: check.uid,
timestamp: start.timeIntervalSince1970,
latencyMs: latency,
status: result.addresses.isEmpty ? "down" : "up",
statusCode: result.addresses.count,
errorMessage: result.addresses.isEmpty ? "No addresses resolved" : nil
)
}
private func performPortCheck(_ check: CheckTarget, start: Date) async -> MetricSample {
let host = URL(string: check.target)?.host ?? check.target
let port = check.port ?? 80
let useTLS = check.protocolType == "tls"
guard !GuardUrl.isPrivateHost(host) else {
return MetricSample(
checkUid: check.uid,
timestamp: start.timeIntervalSince1970,
latencyMs: 0,
status: "error",
statusCode: 0,
errorMessage: "Private host rejected: \(host)"
)
}
let result = await PortScanner.scan(host: host, port: port, useTLS: useTLS)
return MetricSample(
checkUid: check.uid,
timestamp: start.timeIntervalSince1970,
latencyMs: result.latencyMs,
status: result.open ? "up" : "down",
statusCode: result.open ? 1 : 0,
errorMessage: result.error
)
}
}

View File

@ -0,0 +1,65 @@
// retoor <retoor@molodetz.nl>
import Foundation
enum DnsResolver {
struct DnsResult {
let hostname: String
let addresses: [String]
let error: String?
}
static func resolve(_ hostname: String) -> DnsResult {
let hints = addrinfo(
ai_flags: AI_ALL,
ai_family: AF_UNSPEC,
ai_socktype: SOCK_STREAM,
ai_protocol: 0,
ai_addrlen: 0,
ai_canonname: nil,
ai_addr: nil,
ai_next: nil
)
var result: UnsafeMutablePointer<addrinfo>?
let errorCode = getaddrinfo(hostname, nil, &hints, &result)
if errorCode != 0 {
let errorStr = String(cString: gai_strerror(errorCode))
return DnsResult(hostname: hostname, addresses: [], error: errorStr)
}
var addresses: [String] = []
var current = result
while let addr = current {
var hostBuffer = [CChar](repeating: 0, count: Int(NI_MAXHOST))
let sa_len: socklen_t
let sa: UnsafePointer<sockaddr>
if addr.pointee.ai_family == AF_INET {
sa_len = socklen_t(MemoryLayout<sockaddr_in>.size)
sa = UnsafeRawPointer(addr.pointee.ai_addr).assumingMemoryBound(to: sockaddr.self)
} else if addr.pointee.ai_family == AF_INET6 {
sa_len = socklen_t(MemoryLayout<sockaddr_in6>.size)
sa = UnsafeRawPointer(addr.pointee.ai_addr).assumingMemoryBound(to: sockaddr.self)
} else {
current = addr.pointee.ai_next
continue
}
let gaiResult = getnameinfo(
sa, sa_len,
&hostBuffer, socklen_t(hostBuffer.count),
nil, 0,
NI_NUMERICHOST
)
if gaiResult == 0 {
addresses.append(String(cString: hostBuffer))
}
current = addr.pointee.ai_next
}
freeaddrinfo(result)
return DnsResult(hostname: hostname, addresses: addresses, error: nil)
}
}

View File

@ -0,0 +1,46 @@
// retoor <retoor@molodetz.nl>
import Foundation
enum GuardUrl {
static let privateRanges: [(UInt32, UInt32)] = [
(0x0A000000, 0x0AFFFFFF), // 10.0.0.0/8
(0x7F000000, 0x7FFFFFFF), // 127.0.0.0/8
(0xA9FE0000, 0xA9FEFFFF), // 169.254.0.0/16
(0xAC100000, 0xAC1FFFFF), // 172.16.0.0/12
(0xC0A80000, 0xC0A8FFFF), // 192.168.0.0/16
(0x64400000, 0x647FFFFF), // 100.64.0.0/10
(0xCB007100, 0xCB0071FF), // 203.0.113.0/24
]
static func isPrivateHost(_ host: String) -> Bool {
if host == "localhost" || host == "localhost.localdomain" {
return true
}
guard let addr = ipv4Address(host) else {
return host.hasSuffix(".local") || host.hasSuffix(".internal")
}
for (start, end) in privateRanges {
if addr >= start && addr <= end {
return true
}
}
return false
}
static func isPrivateURL(_ urlString: String) -> Bool {
guard let url = URL(string: urlString), let host = url.host else {
return true
}
return isPrivateHost(host)
}
private static func ipv4Address(_ string: String) -> UInt32? {
var sin = sockaddr_in()
guard string.withCString({ cstring in
inet_pton(AF_INET, cstring, &sin.sin_addr) == 1
}) else { return nil }
let addr = sin.sin_addr.s_addr.bigEndian
return addr
}
}

View File

@ -0,0 +1,59 @@
// retoor <retoor@molodetz.nl>
import Foundation
enum IpcCommand: String, Codable {
case addCheck
case removeCheck
case updateCheck
case listChecks
case getMetrics
case ping
case shutdown
}
struct IpcRequest: Codable {
let reqId: Int
let command: IpcCommand
let payload: [String: String]?
}
struct CheckTarget: Codable {
let uid: String
let type: String
let target: String
let intervalSeconds: Int
let port: Int?
let protocolType: String?
}
struct IpcResponse: Codable {
let reqId: Int
let kind: String
let payload: String?
let checks: [CheckTarget]?
let metrics: [MetricSample]?
let error: String?
}
struct MetricSample: Codable {
let checkUid: String
let timestamp: Double
let latencyMs: Double
let status: String
let statusCode: Int
let errorMessage: String?
}
enum IpcProtocol {
static func readRequest(from line: String) -> IpcRequest? {
guard let data = line.data(using: .utf8) else { return nil }
return try? JSONDecoder().decode(IpcRequest.self, from: data)
}
static func writeResponse(_ response: IpcResponse) -> String? {
let encoder = JSONEncoder()
guard let data = try? encoder.encode(response) else { return nil }
return String(data: data, encoding: .utf8)
}
}

View File

@ -0,0 +1,43 @@
// retoor <retoor@molodetz.nl>
import Foundation
import Logging
actor MetricsStore {
private var samples: [MetricSample] = []
private var lastFlush = Date()
private let flushInterval: TimeInterval = 60.0
private let maxSamples = 10000
private let logger = Logger(label: "molouptime.metrics")
struct BatchedSamples: Codable {
let checkUid: String
let samples: [MetricSample]
}
func record(_ sample: MetricSample) {
samples.append(sample)
if samples.count >= maxSamples || Date().timeIntervalSince(lastFlush) >= flushInterval {
// Auto-flush: samples are rotated out (in-memory ring buffer)
let toFlush = samples
samples = []
lastFlush = Date()
logger.info("Flushed \(toFlush.count) metric samples to buffer")
}
}
func collect() -> [MetricSample] {
let collected = samples
samples = []
lastFlush = Date()
return collected
}
func snapshot() -> [MetricSample] {
return samples
}
func count() -> Int {
return samples.count
}
}

View File

@ -0,0 +1,83 @@
// retoor <retoor@molodetz.nl>
import Foundation
import NIO
enum PortScanner {
struct PortResult {
let host: String
let port: Int
let open: Bool
let banner: String?
let latencyMs: Double
let tls: Bool
let error: String?
}
static func scan(host: String, port: Int, timeoutSeconds: Double = 5.0, useTLS: Bool = false) async -> PortResult {
let start = Date()
let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
defer { try? group.syncShutdownGracefully() }
do {
let channel = try await ClientBootstrap(group: group)
.connectTimeout(.seconds(Int64(timeoutSeconds)))
.connect(host: host, port: port)
.get()
let latency = Date().timeIntervalSince(start) * 1000
var banner: String?
if useTLS {
// TLS handshake check - connect and attempt TLS
banner = await performTLSHandshake(host: host, port: port, timeout: timeoutSeconds)
} else {
// Read initial banner if available
banner = await readBanner(channel: channel, timeout: timeoutSeconds)
}
try await channel.close(mode: .all).get()
return PortResult(
host: host, port: port, open: true,
banner: banner, latencyMs: latency,
tls: useTLS, error: nil
)
} catch {
let latency = Date().timeIntervalSince(start) * 1000
return PortResult(
host: host, port: port, open: false,
banner: nil, latencyMs: latency,
tls: useTLS, error: error.localizedDescription
)
}
}
private static func readBanner(channel: Channel, timeout: Double) async -> String? {
// Simple banner read - up to 4KB
var buffer = ByteBufferAllocator().buffer(capacity: 4096)
do {
// Wait briefly for banner data
try await Task.sleep(nanoseconds: UInt64(timeout * 500_000_000))
// Can't easily read from channel in NIO without handler
return nil
} catch {
return nil
}
}
private static func performTLSHandshake(host: String, port: Int, timeout: Double) async -> String? {
// Simplified: attempt TCP connect to the TLS port
let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
defer { try? group.syncShutdownGracefully() }
do {
let _ = try await ClientBootstrap(group: group)
.connectTimeout(.seconds(Int64(timeout)))
.connect(host: host, port: port)
.get()
return "TLS reachable"
} catch {
return nil
}
}
}

View File

@ -0,0 +1,133 @@
// retoor <retoor@molodetz.nl>
import Foundation
import Logging
let logger = Logger(label: "molouptime.main")
@main
struct Molouptime {
static let engine = CheckEngine()
static func main() async {
logger.info("molouptime engine starting")
let stdin = FileHandle.standardInput
let stdout = FileHandle.standardOutput
var buffer = Data()
// Set stdin to read line-by-line
stdin.readabilityHandler = { handle in
let data = handle.availableData
guard !data.isEmpty else { return }
buffer.append(data)
while let newlineRange = buffer.firstIndex(of: UInt8(ascii: "\n")) {
let lineData = buffer[..<newlineRange]
buffer = buffer[buffer.index(after: newlineRange)...]
guard let line = String(data: lineData, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines),
!line.isEmpty else {
continue
}
Task {
await handleLine(line, stdout: stdout)
}
}
}
// Keep the process alive
dispatchMain()
}
static func handleLine(_ line: String, stdout: FileHandle) async {
guard let request = IpcProtocol.readRequest(from: line) else {
let errorResp = IpcResponse(
reqId: 0, kind: "error",
payload: "Invalid JSON", checks: nil, metrics: nil,
error: "Failed to parse request"
)
writeResponse(errorResp, to: stdout)
return
}
switch request.command {
case .ping:
let resp = IpcResponse(reqId: request.reqId, kind: "pong", payload: "ok", checks: nil, metrics: nil, error: nil)
writeResponse(resp, to: stdout)
case .addCheck:
guard let payload = request.payload,
let uid = payload["uid"],
let type = payload["type"],
let target = payload["target"],
let intervalStr = payload["interval"],
let interval = Int(intervalStr) else {
let resp = IpcResponse(reqId: request.reqId, kind: "error", payload: nil, checks: nil, metrics: nil, error: "Missing required fields")
writeResponse(resp, to: stdout)
return
}
let port = payload["port"].flatMap { Int($0) }
let protocolType = payload["protocol"]
let check = CheckTarget(
uid: uid, type: type, target: target,
intervalSeconds: interval, port: port,
protocolType: protocolType
)
await engine.addCheck(check)
let resp = IpcResponse(reqId: request.reqId, kind: "added", payload: uid, checks: nil, metrics: nil, error: nil)
writeResponse(resp, to: stdout)
case .removeCheck:
let uid = request.payload?["uid"] ?? ""
await engine.removeCheck(uid)
let resp = IpcResponse(reqId: request.reqId, kind: "removed", payload: uid, checks: nil, metrics: nil, error: nil)
writeResponse(resp, to: stdout)
case .updateCheck:
guard let payload = request.payload,
let uid = payload["uid"] else {
let resp = IpcResponse(reqId: request.reqId, kind: "error", payload: nil, checks: nil, metrics: nil, error: "Missing uid")
writeResponse(resp, to: stdout)
return
}
let type = payload["type"] ?? "http"
let target = payload["target"] ?? ""
let interval = Int(payload["interval"] ?? "60") ?? 60
let port = payload["port"].flatMap { Int($0) }
let protocolType = payload["protocol"]
let check = CheckTarget(
uid: uid, type: type, target: target,
intervalSeconds: interval, port: port,
protocolType: protocolType
)
await engine.updateCheck(check)
let resp = IpcResponse(reqId: request.reqId, kind: "updated", payload: uid, checks: nil, metrics: nil, error: nil)
writeResponse(resp, to: stdout)
case .listChecks:
let checks = await engine.listChecks()
let resp = IpcResponse(reqId: request.reqId, kind: "checks", payload: nil, checks: checks, metrics: nil, error: nil)
writeResponse(resp, to: stdout)
case .getMetrics:
let metrics = await MetricSample?.none // Placeholder
let resp = IpcResponse(reqId: request.reqId, kind: "metrics", payload: nil, checks: nil, metrics: nil, error: nil)
writeResponse(resp, to: stdout)
case .shutdown:
await engine.stop()
let resp = IpcResponse(reqId: request.reqId, kind: "shutdown", payload: "bye", checks: nil, metrics: nil, error: nil)
writeResponse(resp, to: stdout)
exit(0)
}
}
static func writeResponse(_ response: IpcResponse, to handle: FileHandle) {
guard let json = IpcProtocol.writeResponse(response) else { return }
handle.write(Data("\(json)\n".utf8))
}
}