ticket #78 attempt 1
This commit is contained in:
parent
43c5a948e8
commit
c4490e783a
1
.dpc/verify_cache.json
Normal file
1
.dpc/verify_cache.json
Normal file
File diff suppressed because one or more lines are too long
@ -12,6 +12,7 @@ from devplacepy.cli.jobs import register_jobs
|
|||||||
from devplacepy.cli.backups import register_backups
|
from devplacepy.cli.backups import register_backups
|
||||||
from devplacepy.cli.containers import register_containers
|
from devplacepy.cli.containers import register_containers
|
||||||
from devplacepy.cli.migrate import register_migrate
|
from devplacepy.cli.migrate import register_migrate
|
||||||
|
from devplacepy.cli.molouptime import register_molouptime
|
||||||
|
|
||||||
|
|
||||||
def build_parser():
|
def build_parser():
|
||||||
@ -28,6 +29,7 @@ def build_parser():
|
|||||||
register_backups(sub)
|
register_backups(sub)
|
||||||
register_containers(sub)
|
register_containers(sub)
|
||||||
register_migrate(sub)
|
register_migrate(sub)
|
||||||
|
register_molouptime(sub)
|
||||||
|
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|||||||
189
devplacepy/cli/molouptime.py
Normal file
189
devplacepy/cli/molouptime.py
Normal file
@ -0,0 +1,189 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from devplacepy.database import get_table
|
||||||
|
from devplacepy.utils import generate_uid
|
||||||
|
|
||||||
|
|
||||||
|
def register_molouptime(sub):
|
||||||
|
molouptime = sub.add_parser("molouptime", help="Manage molouptime uptime monitors")
|
||||||
|
molouptime_sub = molouptime.add_subparsers(title="molouptime commands", dest="molouptime_command")
|
||||||
|
|
||||||
|
p_create = molouptime_sub.add_parser("create", help="Create a new monitor check")
|
||||||
|
p_create.add_argument("--name", required=True, help="Check name")
|
||||||
|
p_create.add_argument("--type", required=True, choices=["http", "dns", "port"], dest="check_type", help="Check type")
|
||||||
|
p_create.add_argument("--target", required=True, help="Target URL, hostname, or address")
|
||||||
|
p_create.add_argument("--interval", type=int, default=60, help="Check interval in seconds (default 60)")
|
||||||
|
p_create.add_argument("--user-uid", help="Owner user UID (defaults to primary admin)")
|
||||||
|
p_create.set_defaults(func=cmd_create)
|
||||||
|
|
||||||
|
p_list = molouptime_sub.add_parser("list", help="List monitor checks")
|
||||||
|
p_list.add_argument("--user-uid", help="Filter by user UID")
|
||||||
|
p_list.set_defaults(func=cmd_list)
|
||||||
|
|
||||||
|
p_delete = molouptime_sub.add_parser("delete", help="Delete a monitor check")
|
||||||
|
p_delete.add_argument("uid", help="Check UID to delete")
|
||||||
|
p_delete.set_defaults(func=cmd_delete)
|
||||||
|
|
||||||
|
p_metrics = molouptime_sub.add_parser("metrics", help="Show metrics for a check")
|
||||||
|
p_metrics.add_argument("uid", help="Check item UID")
|
||||||
|
p_metrics.add_argument("--limit", type=int, default=20, help="Number of metrics to show")
|
||||||
|
p_metrics.set_defaults(func=cmd_metrics)
|
||||||
|
|
||||||
|
p_group = molouptime_sub.add_parser("group", help="Manage monitor groups")
|
||||||
|
group_sub = p_group.add_subparsers(title="group commands", dest="group_command")
|
||||||
|
p_group_create = group_sub.add_parser("create", help="Create a group")
|
||||||
|
p_group_create.add_argument("--name", required=True, help="Group name")
|
||||||
|
p_group_create.add_argument("--user-uid", help="Owner user UID")
|
||||||
|
p_group_create.set_defaults(func=cmd_group_create)
|
||||||
|
p_group_list = group_sub.add_parser("list", help="List groups")
|
||||||
|
p_group_list.set_defaults(func=cmd_group_list)
|
||||||
|
|
||||||
|
p_check_health = molouptime_sub.add_parser("health", help="Check if the Swift binary responds")
|
||||||
|
p_check_health.set_defaults(func=cmd_health)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_create(args):
|
||||||
|
checks_table = get_table("monitor_checks")
|
||||||
|
user_uid = args.user_uid or _get_primary_admin_uid()
|
||||||
|
uid = generate_uid()
|
||||||
|
now = datetime.now(timezone.utc).isoformat()
|
||||||
|
checks_table.insert(
|
||||||
|
{
|
||||||
|
"uid": uid,
|
||||||
|
"user_uid": user_uid,
|
||||||
|
"name": args.name,
|
||||||
|
"description": "",
|
||||||
|
"check_type": args.check_type,
|
||||||
|
"target": args.target,
|
||||||
|
"interval_seconds": args.interval,
|
||||||
|
"group_uid": "",
|
||||||
|
"escalation_policy_uid": "",
|
||||||
|
"enabled": 1,
|
||||||
|
"created_at": now,
|
||||||
|
"updated_at": now,
|
||||||
|
"deleted_at": None,
|
||||||
|
"deleted_by": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
# Create default check item
|
||||||
|
items_table = get_table("monitor_check_items")
|
||||||
|
item_uid = generate_uid()
|
||||||
|
items_table.insert(
|
||||||
|
{
|
||||||
|
"uid": item_uid,
|
||||||
|
"check_uid": uid,
|
||||||
|
"user_uid": user_uid,
|
||||||
|
"check_type": args.check_type,
|
||||||
|
"target": args.target,
|
||||||
|
"interval_seconds": args.interval,
|
||||||
|
"created_at": now,
|
||||||
|
"updated_at": now,
|
||||||
|
"deleted_at": None,
|
||||||
|
"deleted_by": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
print(f"Created monitor check {uid} ({args.name}) targeting {args.target}")
|
||||||
|
print(f" Item UID: {item_uid}")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_list(args):
|
||||||
|
checks_table = get_table("monitor_checks")
|
||||||
|
filters = {"deleted_at": None}
|
||||||
|
if args.user_uid:
|
||||||
|
filters["user_uid"] = args.user_uid
|
||||||
|
checks = list(checks_table.find(**filters, order_by="-created_at"))
|
||||||
|
if not checks:
|
||||||
|
print("No monitor checks found")
|
||||||
|
return
|
||||||
|
for c in checks:
|
||||||
|
print(f" {c['uid'][:8]}... {c.get('name','?')} [{c.get('check_type','?')}] -> {c.get('target','?')}")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_delete(args):
|
||||||
|
from devplacepy.database import soft_delete
|
||||||
|
|
||||||
|
checks_table = get_table("monitor_checks")
|
||||||
|
check = checks_table.find_one(uid=args.uid)
|
||||||
|
if not check:
|
||||||
|
print(f"Check {args.uid} not found", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
soft_delete("monitor_checks", args.uid, "cli")
|
||||||
|
print(f"Deleted check {args.uid}")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_metrics(args):
|
||||||
|
snapshots_table = get_table("monitor_metrics_snapshots")
|
||||||
|
metrics = list(
|
||||||
|
snapshots_table.find(check_item_uid=args.uid, order_by="-timestamp", _limit=args.limit)
|
||||||
|
)
|
||||||
|
if not metrics:
|
||||||
|
print(f"No metrics found for {args.uid}")
|
||||||
|
return
|
||||||
|
for m in reversed(metrics):
|
||||||
|
ts = m.get("timestamp", "?")[:19] if m.get("timestamp") else "?"
|
||||||
|
lat = m.get("latency_ms", 0)
|
||||||
|
status = m.get("status", "?")
|
||||||
|
code = m.get("status_code", 0)
|
||||||
|
err = m.get("error_message", "")
|
||||||
|
err_suffix = f" ({err})" if err else ""
|
||||||
|
print(f" {ts} | {status:5s} | {lat:8.1f}ms | HTTP {code}{err_suffix}")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_group_create(args):
|
||||||
|
groups_table = get_table("monitor_groups")
|
||||||
|
user_uid = args.user_uid or _get_primary_admin_uid()
|
||||||
|
uid = generate_uid()
|
||||||
|
now = datetime.now(timezone.utc).isoformat()
|
||||||
|
groups_table.insert(
|
||||||
|
{
|
||||||
|
"uid": uid,
|
||||||
|
"user_uid": user_uid,
|
||||||
|
"name": args.name,
|
||||||
|
"description": "",
|
||||||
|
"created_at": now,
|
||||||
|
"deleted_at": None,
|
||||||
|
"deleted_by": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
print(f"Created group {uid} ({args.name})")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_group_list(args):
|
||||||
|
groups_table = get_table("monitor_groups")
|
||||||
|
groups = list(groups_table.find(deleted_at=None))
|
||||||
|
if not groups:
|
||||||
|
print("No groups found")
|
||||||
|
return
|
||||||
|
for g in groups:
|
||||||
|
print(f" {g['uid'][:8]}... {g.get('name','?')}")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_health(args):
|
||||||
|
from devplacepy.services.molouptime import MolouptimeService # type: ignore[attr-defined]
|
||||||
|
from devplacepy.services.manager import service_manager
|
||||||
|
|
||||||
|
service = service_manager.get("molouptime")
|
||||||
|
if service is None:
|
||||||
|
print("Molouptime service is not registered")
|
||||||
|
sys.exit(1)
|
||||||
|
status = service.status()
|
||||||
|
if status.get("running"):
|
||||||
|
print(f"Molouptime is running (pid {status.get('pid')})")
|
||||||
|
print(f" Active checks: {status.get('checks_active', 0)}")
|
||||||
|
print(f" Up: {status.get('checks_up', 0)} Down: {status.get('checks_down', 0)}")
|
||||||
|
else:
|
||||||
|
print("Molouptime service is not running")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_primary_admin_uid():
|
||||||
|
from devplacepy.database import get_primary_admin_uid
|
||||||
|
|
||||||
|
return get_primary_admin_uid()
|
||||||
@ -32,6 +32,8 @@ ISSLOP_RUNS_DIR = ISSLOP_DIR / "runs"
|
|||||||
ISSLOP_MEDIA_DIR = ISSLOP_DIR / "media"
|
ISSLOP_MEDIA_DIR = ISSLOP_DIR / "media"
|
||||||
KEYS_DIR = DATA_DIR / "keys"
|
KEYS_DIR = DATA_DIR / "keys"
|
||||||
BOT_DIR = DATA_DIR / "bot"
|
BOT_DIR = DATA_DIR / "bot"
|
||||||
|
MOLOUPTIME_BINARY_AVAILABLE = (BASE_DIR / "molouptime" / ".build" / "release" / "molouptime").exists()
|
||||||
|
MOLOUPTIME_BINARY_PATH = str(BASE_DIR / "molouptime" / ".build" / "release" / "molouptime")
|
||||||
LOCKS_DIR = DATA_DIR / "locks"
|
LOCKS_DIR = DATA_DIR / "locks"
|
||||||
|
|
||||||
DEVII_TASKS_DB = DATA_DIR / "devii_tasks.db"
|
DEVII_TASKS_DB = DATA_DIR / "devii_tasks.db"
|
||||||
|
|||||||
@ -1314,6 +1314,14 @@ def init_db():
|
|||||||
_index(db, "jobs", "idx_jobs_created_at", ["created_at"])
|
_index(db, "jobs", "idx_jobs_created_at", ["created_at"])
|
||||||
_index(db, "attachments", "idx_attachments_created_at", ["created_at"])
|
_index(db, "attachments", "idx_attachments_created_at", ["created_at"])
|
||||||
|
|
||||||
|
_index(db, "monitor_checks", "idx_monitor_checks_user", ["user_uid", "deleted_at"])
|
||||||
|
_index(db, "monitor_checks", "idx_monitor_checks_created", ["created_at"])
|
||||||
|
_index(db, "monitor_check_items", "idx_monitor_items_check", ["check_uid", "deleted_at"])
|
||||||
|
_index(db, "monitor_groups", "idx_monitor_groups_user", ["user_uid"])
|
||||||
|
_index(db, "monitor_group_members", "idx_monitor_group_members", ["group_uid", "check_uid"])
|
||||||
|
_index(db, "monitor_escalation_policies", "idx_monitor_escalation_policies", ["check_uid"])
|
||||||
|
_index(db, "monitor_metrics_snapshots", "idx_monitor_metrics_check", ["check_uid", "timestamp"])
|
||||||
|
|
||||||
_backfill_gamification()
|
_backfill_gamification()
|
||||||
backfill_api_keys()
|
backfill_api_keys()
|
||||||
migrate_ai_gateway_settings()
|
migrate_ai_gateway_settings()
|
||||||
|
|||||||
@ -85,6 +85,7 @@ from devplacepy.routers import (
|
|||||||
dbapi,
|
dbapi,
|
||||||
pubsub,
|
pubsub,
|
||||||
game,
|
game,
|
||||||
|
monitors,
|
||||||
)
|
)
|
||||||
from devplacepy.services.manager import service_manager
|
from devplacepy.services.manager import service_manager
|
||||||
from devplacepy.services.background import background
|
from devplacepy.services.background import background
|
||||||
@ -116,6 +117,7 @@ from devplacepy.services.audit import AuditService
|
|||||||
from devplacepy.services.audit import record as audit
|
from devplacepy.services.audit import record as audit
|
||||||
from devplacepy.services.telegram import TelegramService
|
from devplacepy.services.telegram import TelegramService
|
||||||
from devplacepy.services.telegram.outbox_service import TelegramOutboxService
|
from devplacepy.services.telegram.outbox_service import TelegramOutboxService
|
||||||
|
from devplacepy.services.molouptime import MolouptimeService
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
@ -276,6 +278,7 @@ async def lifespan(app: FastAPI):
|
|||||||
service_manager.register(AuditService())
|
service_manager.register(AuditService())
|
||||||
service_manager.register(TelegramService())
|
service_manager.register(TelegramService())
|
||||||
service_manager.register(TelegramOutboxService())
|
service_manager.register(TelegramOutboxService())
|
||||||
|
service_manager.register(MolouptimeService())
|
||||||
if not os.environ.get("DEVPLACE_DISABLE_SERVICES"):
|
if not os.environ.get("DEVPLACE_DISABLE_SERVICES"):
|
||||||
await background.start()
|
await background.start()
|
||||||
if acquire_service_lock():
|
if acquire_service_lock():
|
||||||
@ -463,6 +466,7 @@ app.include_router(devrant.router, prefix="/api")
|
|||||||
app.include_router(dbapi.router, prefix="/dbapi")
|
app.include_router(dbapi.router, prefix="/dbapi")
|
||||||
app.include_router(pubsub.router, prefix="/pubsub")
|
app.include_router(pubsub.router, prefix="/pubsub")
|
||||||
app.include_router(game.router, prefix="/game")
|
app.include_router(game.router, prefix="/game")
|
||||||
|
app.include_router(monitors.router, prefix="/monitors")
|
||||||
|
|
||||||
|
|
||||||
@app.middleware("http")
|
@app.middleware("http")
|
||||||
|
|||||||
281
devplacepy/routers/monitors.py
Normal file
281
devplacepy/routers/monitors.py
Normal file
@ -0,0 +1,281 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Form, Request
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
|
from devplacepy.database import get_table
|
||||||
|
from devplacepy.responses import json_error, respond
|
||||||
|
from devplacepy.utils import generate_uid, get_current_user, is_admin, require_user
|
||||||
|
|
||||||
|
logger = logging.getLogger("routers.monitors")
|
||||||
|
|
||||||
|
router = APIRouter(tags=["monitors"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/checks")
|
||||||
|
async def list_checks(
|
||||||
|
request: Request,
|
||||||
|
user: dict = Depends(require_user),
|
||||||
|
):
|
||||||
|
checks_table = get_table("monitor_checks")
|
||||||
|
admin = is_admin(user)
|
||||||
|
if admin:
|
||||||
|
checks = list(checks_table.find(deleted_at=None, order_by="-created_at"))
|
||||||
|
else:
|
||||||
|
checks = list(
|
||||||
|
checks_table.find(user_uid=user["uid"], deleted_at=None, order_by="-created_at")
|
||||||
|
)
|
||||||
|
# Enrich with items count
|
||||||
|
items_table = get_table("monitor_check_items")
|
||||||
|
for check in checks:
|
||||||
|
try:
|
||||||
|
items = list(items_table.find(check_uid=check["uid"], deleted_at=None))
|
||||||
|
check["items_count"] = len(items)
|
||||||
|
except Exception:
|
||||||
|
check["items_count"] = 0
|
||||||
|
return respond(request, "monitors/list.html", {"checks": checks})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/checks")
|
||||||
|
async def create_check(
|
||||||
|
request: Request,
|
||||||
|
user: dict = Depends(require_user),
|
||||||
|
name: Annotated[str, Form(max_length=128)] = ...,
|
||||||
|
check_type: Annotated[str, Form()] = ...,
|
||||||
|
target: Annotated[str, Form(max_length=1024)] = ...,
|
||||||
|
interval_seconds: Annotated[int, Form(ge=10, le=86400)] = 60,
|
||||||
|
description: Annotated[str, Form(max_length=512)] = "",
|
||||||
|
group_uid: Annotated[str, Form()] = "",
|
||||||
|
escalation_policy_uid: Annotated[str, Form()] = "",
|
||||||
|
):
|
||||||
|
# Validate target - reject private/internal addresses
|
||||||
|
if check_type == "http":
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
parsed = urlparse(target)
|
||||||
|
host = parsed.hostname or ""
|
||||||
|
if host in ("localhost", "127.0.0.1", "0.0.0.0"):
|
||||||
|
return json_error(400, "Internal/private targets are not allowed")
|
||||||
|
if host.startswith("10.") or host.startswith("192.168.") or host.startswith("172.16."):
|
||||||
|
return json_error(400, "Private network targets are not allowed")
|
||||||
|
|
||||||
|
uid = generate_uid()
|
||||||
|
now = datetime.now(timezone.utc).isoformat()
|
||||||
|
checks_table = get_table("monitor_checks")
|
||||||
|
checks_table.insert(
|
||||||
|
{
|
||||||
|
"uid": uid,
|
||||||
|
"user_uid": user["uid"],
|
||||||
|
"name": name,
|
||||||
|
"description": description,
|
||||||
|
"check_type": check_type,
|
||||||
|
"target": target,
|
||||||
|
"interval_seconds": interval_seconds,
|
||||||
|
"group_uid": group_uid,
|
||||||
|
"escalation_policy_uid": escalation_policy_uid,
|
||||||
|
"enabled": 1,
|
||||||
|
"created_at": now,
|
||||||
|
"updated_at": now,
|
||||||
|
"deleted_at": None,
|
||||||
|
"deleted_by": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Add default check item
|
||||||
|
from devplacepy.config import MOLOUPTIME_BINARY_AVAILABLE
|
||||||
|
|
||||||
|
if MOLOUPTIME_BINARY_AVAILABLE:
|
||||||
|
try:
|
||||||
|
item_uid = generate_uid()
|
||||||
|
items_table = get_table("monitor_check_items")
|
||||||
|
items_table.insert(
|
||||||
|
{
|
||||||
|
"uid": item_uid,
|
||||||
|
"check_uid": uid,
|
||||||
|
"user_uid": user["uid"],
|
||||||
|
"check_type": check_type,
|
||||||
|
"target": target,
|
||||||
|
"interval_seconds": interval_seconds,
|
||||||
|
"created_at": now,
|
||||||
|
"updated_at": now,
|
||||||
|
"deleted_at": None,
|
||||||
|
"deleted_by": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Failed to create check item: %s", e)
|
||||||
|
|
||||||
|
return {"uid": uid, "name": name, "status": "created"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/checks/{check_uid}")
|
||||||
|
async def get_check(
|
||||||
|
request: Request,
|
||||||
|
check_uid: str,
|
||||||
|
user: dict = Depends(require_user),
|
||||||
|
):
|
||||||
|
checks_table = get_table("monitor_checks")
|
||||||
|
check = checks_table.find_one(uid=check_uid)
|
||||||
|
if not check:
|
||||||
|
return json_error(404, "Check not found")
|
||||||
|
if not is_admin(user) and check["user_uid"] != user["uid"]:
|
||||||
|
return json_error(403, "Access denied")
|
||||||
|
# Get items
|
||||||
|
items_table = get_table("monitor_check_items")
|
||||||
|
items = list(items_table.find(check_uid=check_uid, deleted_at=None))
|
||||||
|
check["items"] = items
|
||||||
|
return respond(request, "monitors/detail.html", {"check": check})
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/checks/{check_uid}")
|
||||||
|
async def delete_check(
|
||||||
|
request: Request,
|
||||||
|
check_uid: str,
|
||||||
|
user: dict = Depends(require_user),
|
||||||
|
):
|
||||||
|
from devplacepy.database import soft_delete
|
||||||
|
|
||||||
|
checks_table = get_table("monitor_checks")
|
||||||
|
check = checks_table.find_one(uid=check_uid)
|
||||||
|
if not check:
|
||||||
|
return json_error(404, "Check not found")
|
||||||
|
if not is_admin(user) and check["user_uid"] != user["uid"]:
|
||||||
|
return json_error(403, "Access denied")
|
||||||
|
|
||||||
|
soft_delete("monitor_checks", check_uid, user["uid"])
|
||||||
|
# Also soft-delete items
|
||||||
|
items_table = get_table("monitor_check_items")
|
||||||
|
for item in items_table.find(check_uid=check_uid, deleted_at=None):
|
||||||
|
soft_delete("monitor_check_items", item["uid"], user["uid"])
|
||||||
|
|
||||||
|
return {"status": "deleted", "uid": check_uid}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/checks/{check_uid}/toggle")
|
||||||
|
async def toggle_check(
|
||||||
|
request: Request,
|
||||||
|
check_uid: str,
|
||||||
|
user: dict = Depends(require_user),
|
||||||
|
):
|
||||||
|
checks_table = get_table("monitor_checks")
|
||||||
|
check = checks_table.find_one(uid=check_uid)
|
||||||
|
if not check:
|
||||||
|
return json_error(404, "Check not found")
|
||||||
|
if not is_admin(user) and check["user_uid"] != user["uid"]:
|
||||||
|
return json_error(403, "Access denied")
|
||||||
|
|
||||||
|
new_enabled = 0 if check.get("enabled", 1) else 1
|
||||||
|
checks_table.update(
|
||||||
|
{"uid": check_uid, "enabled": new_enabled, "updated_at": datetime.now(timezone.utc).isoformat()},
|
||||||
|
["uid"],
|
||||||
|
)
|
||||||
|
return {"status": "toggled", "enabled": new_enabled}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/groups")
|
||||||
|
async def list_groups(
|
||||||
|
request: Request,
|
||||||
|
user: dict = Depends(require_user),
|
||||||
|
):
|
||||||
|
groups_table = get_table("monitor_groups")
|
||||||
|
admin = is_admin(user)
|
||||||
|
if admin:
|
||||||
|
groups = list(groups_table.find(deleted_at=None, order_by="-created_at"))
|
||||||
|
else:
|
||||||
|
groups = list(
|
||||||
|
groups_table.find(user_uid=user["uid"], deleted_at=None, order_by="-created_at")
|
||||||
|
)
|
||||||
|
return respond(request, "monitors/groups.html", {"groups": groups})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/groups")
|
||||||
|
async def create_group(
|
||||||
|
request: Request,
|
||||||
|
user: dict = Depends(require_user),
|
||||||
|
name: Annotated[str, Form(max_length=128)] = ...,
|
||||||
|
description: Annotated[str, Form(max_length=512)] = "",
|
||||||
|
):
|
||||||
|
uid = generate_uid()
|
||||||
|
now = datetime.now(timezone.utc).isoformat()
|
||||||
|
groups_table = get_table("monitor_groups")
|
||||||
|
groups_table.insert(
|
||||||
|
{
|
||||||
|
"uid": uid,
|
||||||
|
"user_uid": user["uid"],
|
||||||
|
"name": name,
|
||||||
|
"description": description,
|
||||||
|
"created_at": now,
|
||||||
|
"deleted_at": None,
|
||||||
|
"deleted_by": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return {"uid": uid, "name": name, "status": "created"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/admin/stats")
|
||||||
|
async def admin_stats(
|
||||||
|
request: Request,
|
||||||
|
user: dict = Depends(require_user),
|
||||||
|
):
|
||||||
|
if not is_admin(user):
|
||||||
|
return json_error(403, "Admin access required")
|
||||||
|
|
||||||
|
checks_table = get_table("monitor_checks")
|
||||||
|
total_checks = 0
|
||||||
|
total_items = 0
|
||||||
|
by_type = {}
|
||||||
|
by_user = {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
rows = checks_table.find(deleted_at=None)
|
||||||
|
for row in rows:
|
||||||
|
total_checks += 1
|
||||||
|
ct = row.get("check_type", "unknown")
|
||||||
|
by_type[ct] = by_type.get(ct, 0) + 1
|
||||||
|
uu = row.get("user_uid", "unknown")
|
||||||
|
by_user[uu] = by_user.get(uu, 0) + 1
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
items_table = get_table("monitor_check_items")
|
||||||
|
try:
|
||||||
|
total_items = len(list(items_table.find(deleted_at=None)))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
snapshots_table = get_table("monitor_metrics_snapshots")
|
||||||
|
total_metrics = 0
|
||||||
|
try:
|
||||||
|
total_metrics = len(list(snapshots_table.find()))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return respond(
|
||||||
|
request,
|
||||||
|
"admin/monitors.html",
|
||||||
|
{
|
||||||
|
"total_checks": total_checks,
|
||||||
|
"total_items": total_items,
|
||||||
|
"total_metrics": total_metrics,
|
||||||
|
"by_type": by_type,
|
||||||
|
"by_user": by_user,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/escalation-policies")
|
||||||
|
async def list_escalation_policies(
|
||||||
|
request: Request,
|
||||||
|
user: dict = Depends(require_user),
|
||||||
|
):
|
||||||
|
policies_table = get_table("monitor_escalation_policies")
|
||||||
|
policies = list(
|
||||||
|
policies_table.find(user_uid=user["uid"], order_by="-created_at")
|
||||||
|
)
|
||||||
|
return respond(request, "monitors/escalation.html", {"policies": policies})
|
||||||
@ -9,6 +9,7 @@ from .comments import COMMENTS_ACTIONS
|
|||||||
from .dbapi import DBAPI_ACTIONS
|
from .dbapi import DBAPI_ACTIONS
|
||||||
from .engagement import ENGAGEMENT_ACTIONS
|
from .engagement import ENGAGEMENT_ACTIONS
|
||||||
from .game import GAME_ACTIONS
|
from .game import GAME_ACTIONS
|
||||||
|
from .molouptime import MOLOUPTIME_ACTIONS
|
||||||
from .gateway import GATEWAY_ACTIONS
|
from .gateway import GATEWAY_ACTIONS
|
||||||
from .gists import GIST_ACTIONS
|
from .gists import GIST_ACTIONS
|
||||||
from .issues import ISSUE_ACTIONS
|
from .issues import ISSUE_ACTIONS
|
||||||
@ -45,6 +46,7 @@ ACTIONS: tuple[Action, ...] = (
|
|||||||
+ DBAPI_ACTIONS
|
+ DBAPI_ACTIONS
|
||||||
+ GATEWAY_ACTIONS
|
+ GATEWAY_ACTIONS
|
||||||
+ GAME_ACTIONS
|
+ GAME_ACTIONS
|
||||||
|
+ MOLOUPTIME_ACTIONS
|
||||||
)
|
)
|
||||||
|
|
||||||
PLATFORM_CATALOG = Catalog(actions=ACTIONS)
|
PLATFORM_CATALOG = Catalog(actions=ACTIONS)
|
||||||
|
|||||||
110
devplacepy/services/devii/actions/catalog/molouptime.py
Normal file
110
devplacepy/services/devii/actions/catalog/molouptime.py
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from ..spec import Action
|
||||||
|
from ._shared import body, confirm, path
|
||||||
|
|
||||||
|
MOLOUPTIME_ACTIONS: tuple[Action, ...] = (
|
||||||
|
Action(
|
||||||
|
name="monitor_create",
|
||||||
|
method="POST",
|
||||||
|
path="/monitors/checks",
|
||||||
|
summary="Create a new uptime monitor check",
|
||||||
|
description=(
|
||||||
|
"Creates an uptime monitoring check. type is one of 'http', 'dns', or 'port'. "
|
||||||
|
"For HTTP checks, target must be a public URL. For DNS checks, target is a hostname."
|
||||||
|
" For port checks, target is a hostname and port is required."
|
||||||
|
),
|
||||||
|
params=(
|
||||||
|
body("name", "Human-readable name for the check.", required=True),
|
||||||
|
body("check_type", "Type of check: 'http', 'dns', or 'port'.", required=True),
|
||||||
|
body("target", "The URL, hostname, or address to check.", required=True),
|
||||||
|
body("interval_seconds", "Seconds between checks (min 10, default 60)."),
|
||||||
|
body("description", "Optional description for the check."),
|
||||||
|
),
|
||||||
|
requires_auth=True,
|
||||||
|
),
|
||||||
|
Action(
|
||||||
|
name="monitor_list",
|
||||||
|
method="GET",
|
||||||
|
path="/monitors/checks",
|
||||||
|
summary="List your uptime monitors",
|
||||||
|
description="Returns all monitor checks for the current user (or all checks for admins).",
|
||||||
|
requires_auth=True,
|
||||||
|
read_only=True,
|
||||||
|
),
|
||||||
|
Action(
|
||||||
|
name="monitor_detail",
|
||||||
|
method="GET",
|
||||||
|
path="/monitors/checks/{check_uid}",
|
||||||
|
summary="Get details of a monitor check",
|
||||||
|
description="Returns the check details including all check items.",
|
||||||
|
params=(path("check_uid", "UID of the monitor check"),),
|
||||||
|
requires_auth=True,
|
||||||
|
read_only=True,
|
||||||
|
),
|
||||||
|
Action(
|
||||||
|
name="monitor_delete",
|
||||||
|
method="DELETE",
|
||||||
|
path="/monitors/checks/{check_uid}",
|
||||||
|
summary="Delete a monitor check",
|
||||||
|
description="Soft-deletes a monitor check and all its items.",
|
||||||
|
params=(
|
||||||
|
path("check_uid", "UID of the monitor check"),
|
||||||
|
confirm(),
|
||||||
|
),
|
||||||
|
requires_auth=True,
|
||||||
|
),
|
||||||
|
Action(
|
||||||
|
name="monitor_toggle",
|
||||||
|
method="POST",
|
||||||
|
path="/monitors/checks/{check_uid}/toggle",
|
||||||
|
summary="Enable or disable a monitor check",
|
||||||
|
description="Toggles the enabled state of a monitor check on/off.",
|
||||||
|
params=(path("check_uid", "UID of the monitor check"),),
|
||||||
|
requires_auth=True,
|
||||||
|
),
|
||||||
|
Action(
|
||||||
|
name="monitor_groups_list",
|
||||||
|
method="GET",
|
||||||
|
path="/monitors/groups",
|
||||||
|
summary="List monitor groups",
|
||||||
|
description="Returns all monitor groups for the current user.",
|
||||||
|
requires_auth=True,
|
||||||
|
read_only=True,
|
||||||
|
),
|
||||||
|
Action(
|
||||||
|
name="monitor_group_create",
|
||||||
|
method="POST",
|
||||||
|
path="/monitors/groups",
|
||||||
|
summary="Create a monitor group",
|
||||||
|
description="Creates a group for organizing monitor checks.",
|
||||||
|
params=(
|
||||||
|
body("name", "Group name.", required=True),
|
||||||
|
body("description", "Optional group description."),
|
||||||
|
),
|
||||||
|
requires_auth=True,
|
||||||
|
),
|
||||||
|
Action(
|
||||||
|
name="monitor_admin_stats",
|
||||||
|
method="GET",
|
||||||
|
path="/monitors/admin/stats",
|
||||||
|
summary="Admin: view cross-user monitor statistics",
|
||||||
|
description="Returns aggregate statistics about all monitors (admin only).",
|
||||||
|
requires_auth=True,
|
||||||
|
requires_admin=True,
|
||||||
|
read_only=True,
|
||||||
|
),
|
||||||
|
Action(
|
||||||
|
name="monitor_escalation_policies",
|
||||||
|
method="GET",
|
||||||
|
path="/monitors/escalation-policies",
|
||||||
|
summary="List escalation policies",
|
||||||
|
description="Returns escalation policies for the current user.",
|
||||||
|
requires_auth=True,
|
||||||
|
read_only=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = ["MOLOUPTIME_ACTIONS"]
|
||||||
15
devplacepy/services/molouptime/__init__.py
Normal file
15
devplacepy/services/molouptime/__init__.py
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
__all__ = ["MolouptimeService"]
|
||||||
|
|
||||||
|
|
||||||
|
def __getattr__(name: str) -> Any:
|
||||||
|
if name == "MolouptimeService":
|
||||||
|
from .service import MolouptimeService
|
||||||
|
|
||||||
|
return MolouptimeService
|
||||||
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||||
54
devplacepy/services/molouptime/ipc.py
Normal file
54
devplacepy/services/molouptime/ipc.py
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
LOGGER = logging.getLogger("molouptime.ipc")
|
||||||
|
|
||||||
|
|
||||||
|
class MolouptimeIPC:
|
||||||
|
def __init__(self, service):
|
||||||
|
self._service = service
|
||||||
|
|
||||||
|
async def send_command(
|
||||||
|
self, command: str, payload: dict | None = None
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return await self._service._send_command(command, payload)
|
||||||
|
|
||||||
|
async def add_check(
|
||||||
|
self,
|
||||||
|
uid: str,
|
||||||
|
check_type: str,
|
||||||
|
target: str,
|
||||||
|
interval: int = 60,
|
||||||
|
port: int | None = None,
|
||||||
|
protocol: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return await self._service.add_check(
|
||||||
|
uid=uid,
|
||||||
|
check_type=check_type,
|
||||||
|
target=target,
|
||||||
|
interval=interval,
|
||||||
|
port=port,
|
||||||
|
protocol=protocol,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def remove_check(self, uid: str) -> dict[str, Any]:
|
||||||
|
return await self._service.remove_check(uid)
|
||||||
|
|
||||||
|
async def list_checks(self) -> list[dict[str, Any]]:
|
||||||
|
return await self._service.list_checks()
|
||||||
|
|
||||||
|
async def get_metrics(self) -> list[dict[str, Any]]:
|
||||||
|
return await self._service.get_metrics()
|
||||||
|
|
||||||
|
async def health(self) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
response = await self._service._send_command("ping")
|
||||||
|
return {"healthy": response.get("kind") == "pong"}
|
||||||
|
except RuntimeError:
|
||||||
|
return {"healthy": False, "error": "Service not running"}
|
||||||
538
devplacepy/services/molouptime/service.py
Normal file
538
devplacepy/services/molouptime/service.py
Normal file
@ -0,0 +1,538 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
from collections import defaultdict
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from devplacepy.config import BASE_DIR
|
||||||
|
from devplacepy.database import (
|
||||||
|
db,
|
||||||
|
get_int_setting,
|
||||||
|
get_setting,
|
||||||
|
get_table,
|
||||||
|
)
|
||||||
|
from devplacepy.services.base import BaseService, ConfigField
|
||||||
|
from devplacepy.utils import generate_uid
|
||||||
|
|
||||||
|
logger = logging.getLogger("molouptime.service")
|
||||||
|
|
||||||
|
FIELD_CHECK_INTERVAL = "molouptime_check_interval"
|
||||||
|
FIELD_BINARY_PATH = "molouptime_binary_path"
|
||||||
|
FIELD_METRICS_FLUSH_INTERVAL = "molouptime_metrics_flush_seconds"
|
||||||
|
|
||||||
|
ESCLATION_TIER_NOTIFICATION = 300
|
||||||
|
ESCLATION_TIER_DEVII = 900
|
||||||
|
ESCLATION_TIER_TELEGRAM = 1800
|
||||||
|
ESCLATION_TIER_ISSUE = 3600
|
||||||
|
|
||||||
|
STREAM_LIMIT = 16 * 1024 * 1024
|
||||||
|
STOP_GRACE_SECONDS = 5.0
|
||||||
|
METRICS_POLL_INTERVAL = 60
|
||||||
|
|
||||||
|
|
||||||
|
class MolouptimeService(BaseService):
|
||||||
|
default_enabled = True
|
||||||
|
min_interval = 5
|
||||||
|
title = "Molouptime Uptime Monitor"
|
||||||
|
description = (
|
||||||
|
"Runs the molouptime Swift binary for automated uptime monitoring of URLs, "
|
||||||
|
"DNS records, and ports. Detects state transitions and manages escalation "
|
||||||
|
"schedules for notifications via DevPlace, Devii, Telegram, and issues."
|
||||||
|
)
|
||||||
|
config_fields = [
|
||||||
|
ConfigField(
|
||||||
|
FIELD_CHECK_INTERVAL,
|
||||||
|
"Default check interval (seconds)",
|
||||||
|
type="int",
|
||||||
|
default=60,
|
||||||
|
minimum=10,
|
||||||
|
help="Default interval between checks for new monitors.",
|
||||||
|
group="Molouptime",
|
||||||
|
),
|
||||||
|
ConfigField(
|
||||||
|
FIELD_BINARY_PATH,
|
||||||
|
"Molouptime binary path",
|
||||||
|
type="str",
|
||||||
|
default=str(Path(BASE_DIR) / "molouptime" / ".build" / "release" / "molouptime"),
|
||||||
|
help="Path to the compiled molouptime Swift binary.",
|
||||||
|
group="Molouptime",
|
||||||
|
),
|
||||||
|
ConfigField(
|
||||||
|
FIELD_METRICS_FLUSH_INTERVAL,
|
||||||
|
"Metrics flush interval (seconds)",
|
||||||
|
type="int",
|
||||||
|
default=60,
|
||||||
|
minimum=10,
|
||||||
|
help="How often to poll metrics from the Swift engine.",
|
||||||
|
group="Molouptime",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self._proc: asyncio.subprocess.Process | None = None
|
||||||
|
self._tasks: list[asyncio.Task] = []
|
||||||
|
self._pending: dict[int, asyncio.Future] = {}
|
||||||
|
self._req_seq = 0
|
||||||
|
self._stdin_lock = asyncio.Lock()
|
||||||
|
self._check_states: dict[str, str] = {}
|
||||||
|
self._check_down_since: dict[str, datetime] = {}
|
||||||
|
self._escalation_timers: dict[str, list[asyncio.Task]] = defaultdict(list)
|
||||||
|
self._last_state_change_time: dict[str, datetime] = {}
|
||||||
|
self._live_logs_list: list[str] = []
|
||||||
|
self._stats = {
|
||||||
|
"checks_active": 0,
|
||||||
|
"checks_up": 0,
|
||||||
|
"checks_down": 0,
|
||||||
|
"checks_error": 0,
|
||||||
|
"metrics_flushed": 0,
|
||||||
|
"escalations_fired": 0,
|
||||||
|
"recoveries": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _run(self) -> None:
|
||||||
|
self.log("Molouptime service starting")
|
||||||
|
await self._start_process()
|
||||||
|
self._tasks.append(asyncio.create_task(self._read_stdout()))
|
||||||
|
self._tasks.append(asyncio.create_task(self._read_stderr()))
|
||||||
|
self._tasks.append(asyncio.create_task(self._metrics_poller()))
|
||||||
|
await self._sync_checks()
|
||||||
|
self.log("Molouptime service ready")
|
||||||
|
|
||||||
|
async def _start_process(self) -> None:
|
||||||
|
binary_path = get_setting(FIELD_BINARY_PATH, "")
|
||||||
|
if not binary_path or not os.path.isfile(binary_path):
|
||||||
|
self.log(f"Binary not found at {binary_path}, trying default path")
|
||||||
|
binary_path = str(
|
||||||
|
Path(BASE_DIR) / "molouptime" / ".build" / "release" / "molouptime"
|
||||||
|
)
|
||||||
|
|
||||||
|
self._proc = await asyncio.create_subprocess_exec(
|
||||||
|
binary_path,
|
||||||
|
cwd=str(BASE_DIR),
|
||||||
|
stdin=asyncio.subprocess.PIPE,
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.PIPE,
|
||||||
|
start_new_session=True,
|
||||||
|
limit=STREAM_LIMIT,
|
||||||
|
)
|
||||||
|
self.log(f"Molouptime binary started (pid {self._proc.pid})")
|
||||||
|
|
||||||
|
async def _terminate(self) -> None:
|
||||||
|
proc = self._proc
|
||||||
|
self._proc = None
|
||||||
|
for task in self._tasks:
|
||||||
|
task.cancel()
|
||||||
|
self._tasks = []
|
||||||
|
for future in self._pending.values():
|
||||||
|
if not future.done():
|
||||||
|
future.cancel()
|
||||||
|
self._pending = {}
|
||||||
|
for timers in self._escalation_timers.values():
|
||||||
|
for t in timers:
|
||||||
|
t.cancel()
|
||||||
|
self._escalation_timers.clear()
|
||||||
|
if proc is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
await self._write({"command": "shutdown", "reqId": 0, "payload": None})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
if proc.stdin and not proc.stdin.is_closing():
|
||||||
|
proc.stdin.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(proc.wait(), timeout=STOP_GRACE_SECONDS)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
self._kill_group(proc)
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(proc.wait(), timeout=STOP_GRACE_SECONDS)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass
|
||||||
|
self.log("Molouptime service stopped")
|
||||||
|
|
||||||
|
def _kill_group(self, proc: asyncio.subprocess.Process) -> None:
|
||||||
|
try:
|
||||||
|
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
|
||||||
|
except (ProcessLookupError, PermissionError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def _read_stdout(self) -> None:
|
||||||
|
proc = self._proc
|
||||||
|
if proc is None or proc.stdout is None:
|
||||||
|
return
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
line = await proc.stdout.readline()
|
||||||
|
except ValueError:
|
||||||
|
break
|
||||||
|
if not line:
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
frame = json.loads(line.decode("utf-8").strip())
|
||||||
|
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||||
|
continue
|
||||||
|
await self._handle_frame(frame)
|
||||||
|
|
||||||
|
async def _read_stderr(self) -> None:
|
||||||
|
proc = self._proc
|
||||||
|
if proc is None or proc.stderr is None:
|
||||||
|
return
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
line = await proc.stderr.readline()
|
||||||
|
except ValueError:
|
||||||
|
break
|
||||||
|
if not line:
|
||||||
|
break
|
||||||
|
text = line.decode("utf-8", errors="replace").strip()
|
||||||
|
if text:
|
||||||
|
logger.debug("molouptime stderr: %s", text)
|
||||||
|
|
||||||
|
async def _handle_frame(self, frame: dict) -> None:
|
||||||
|
kind = frame.get("kind", "")
|
||||||
|
if kind == "pong":
|
||||||
|
self._resolve(frame)
|
||||||
|
elif kind == "added" or kind == "removed" or kind == "updated":
|
||||||
|
self._resolve(frame)
|
||||||
|
elif kind == "metrics":
|
||||||
|
self._resolve(frame)
|
||||||
|
elif kind == "checks":
|
||||||
|
self._resolve(frame)
|
||||||
|
elif kind == "error":
|
||||||
|
logger.error("molouptime error: %s", frame.get("error"))
|
||||||
|
self._resolve(frame)
|
||||||
|
|
||||||
|
def _resolve(self, frame: dict) -> None:
|
||||||
|
future = self._pending.pop(frame.get("reqId"), None)
|
||||||
|
if future is not None and not future.done():
|
||||||
|
future.set_result(frame)
|
||||||
|
|
||||||
|
async def _write(self, command: dict) -> None:
|
||||||
|
proc = self._proc
|
||||||
|
if proc is None or proc.stdin is None or proc.stdin.is_closing():
|
||||||
|
raise RuntimeError("Molouptime process is not running")
|
||||||
|
payload = (json.dumps(command, ensure_ascii=False) + "\n").encode("utf-8")
|
||||||
|
async with self._stdin_lock:
|
||||||
|
proc.stdin.write(payload)
|
||||||
|
await proc.stdin.drain()
|
||||||
|
|
||||||
|
def _next_req(self) -> int:
|
||||||
|
self._req_seq += 1
|
||||||
|
return self._req_seq
|
||||||
|
|
||||||
|
async def _send_command(
|
||||||
|
self, command: str, payload: dict | None = None
|
||||||
|
) -> dict:
|
||||||
|
req_id = self._next_req()
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
future = loop.create_future()
|
||||||
|
self._pending[req_id] = future
|
||||||
|
try:
|
||||||
|
await self._write(
|
||||||
|
{
|
||||||
|
"command": command,
|
||||||
|
"reqId": req_id,
|
||||||
|
"payload": payload,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return await asyncio.wait_for(future, timeout=30.0)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
self._pending.pop(req_id, None)
|
||||||
|
raise RuntimeError(f"Command {command} timed out")
|
||||||
|
|
||||||
|
async def add_check(
|
||||||
|
self,
|
||||||
|
uid: str,
|
||||||
|
check_type: str,
|
||||||
|
target: str,
|
||||||
|
interval: int = 60,
|
||||||
|
port: int | None = None,
|
||||||
|
protocol: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
payload: dict = {
|
||||||
|
"uid": uid,
|
||||||
|
"type": check_type,
|
||||||
|
"target": target,
|
||||||
|
"interval": str(interval),
|
||||||
|
}
|
||||||
|
if port is not None:
|
||||||
|
payload["port"] = str(port)
|
||||||
|
if protocol is not None:
|
||||||
|
payload["protocol"] = protocol
|
||||||
|
return await self._send_command("addCheck", payload)
|
||||||
|
|
||||||
|
async def remove_check(self, uid: str) -> dict:
|
||||||
|
return await self._send_command("removeCheck", {"uid": uid})
|
||||||
|
|
||||||
|
async def update_check(
|
||||||
|
self,
|
||||||
|
uid: str,
|
||||||
|
check_type: str | None = None,
|
||||||
|
target: str | None = None,
|
||||||
|
interval: int | None = None,
|
||||||
|
port: int | None = None,
|
||||||
|
protocol: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
payload: dict = {"uid": uid}
|
||||||
|
if check_type is not None:
|
||||||
|
payload["type"] = check_type
|
||||||
|
if target is not None:
|
||||||
|
payload["target"] = target
|
||||||
|
if interval is not None:
|
||||||
|
payload["interval"] = str(interval)
|
||||||
|
if port is not None:
|
||||||
|
payload["port"] = str(port)
|
||||||
|
if protocol is not None:
|
||||||
|
payload["protocol"] = protocol
|
||||||
|
return await self._send_command("updateCheck", payload)
|
||||||
|
|
||||||
|
async def list_checks(self) -> list[dict]:
|
||||||
|
response = await self._send_command("listChecks")
|
||||||
|
return response.get("checks", [])
|
||||||
|
|
||||||
|
async def get_metrics(self) -> list[dict]:
|
||||||
|
response = await self._send_command("getMetrics")
|
||||||
|
return response.get("metrics", [])
|
||||||
|
|
||||||
|
async def _sync_checks(self) -> None:
|
||||||
|
monitors = get_table("monitor_checks")
|
||||||
|
try:
|
||||||
|
checks = list(monitors.find(deleted_at=None))
|
||||||
|
except Exception:
|
||||||
|
checks = []
|
||||||
|
for check in checks:
|
||||||
|
items_table = get_table("monitor_check_items")
|
||||||
|
try:
|
||||||
|
items = list(items_table.find(check_uid=check["uid"], deleted_at=None))
|
||||||
|
except Exception:
|
||||||
|
items = []
|
||||||
|
for item in items:
|
||||||
|
try:
|
||||||
|
await self.add_check(
|
||||||
|
uid=item["uid"],
|
||||||
|
check_type=item["check_type"],
|
||||||
|
target=item["target"],
|
||||||
|
interval=item.get("interval_seconds", 60),
|
||||||
|
port=item.get("port"),
|
||||||
|
protocol=item.get("protocol_type"),
|
||||||
|
)
|
||||||
|
self._check_states[item["uid"]] = "unknown"
|
||||||
|
self._stats["checks_active"] += 1
|
||||||
|
except RuntimeError as e:
|
||||||
|
logger.error("Failed to sync check %s: %s", item["uid"], e)
|
||||||
|
|
||||||
|
async def _metrics_poller(self) -> None:
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
await asyncio.sleep(METRICS_POLL_INTERVAL)
|
||||||
|
metrics = await self.get_metrics()
|
||||||
|
if metrics:
|
||||||
|
self._stats["metrics_flushed"] += len(metrics)
|
||||||
|
self._persist_metrics(metrics)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Metrics poll failed")
|
||||||
|
|
||||||
|
def _persist_metrics(self, metrics: list[dict]) -> None:
|
||||||
|
snapshots = get_table("monitor_metrics_snapshots")
|
||||||
|
now = datetime.now(timezone.utc).isoformat()
|
||||||
|
for m in metrics:
|
||||||
|
try:
|
||||||
|
snapshots.insert(
|
||||||
|
{
|
||||||
|
"uid": generate_uid(),
|
||||||
|
"check_item_uid": m.get("checkUid", ""),
|
||||||
|
"timestamp": now,
|
||||||
|
"latency_ms": m.get("latencyMs", 0),
|
||||||
|
"status": m.get("status", "unknown"),
|
||||||
|
"status_code": m.get("statusCode", 0),
|
||||||
|
"error_message": m.get("errorMessage"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to persist metric")
|
||||||
|
|
||||||
|
async def on_check_result(self, check_uid: str, status: str) -> None:
|
||||||
|
previous = self._check_states.get(check_uid, "unknown")
|
||||||
|
self._check_states[check_uid] = status
|
||||||
|
|
||||||
|
if status == "up" and previous in ("down", "error"):
|
||||||
|
self._stats["recoveries"] += 1
|
||||||
|
self._cancel_escalation(check_uid)
|
||||||
|
self._fire_restore_notification(check_uid)
|
||||||
|
self._last_state_change_time[check_uid] = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
elif status in ("down", "error") and previous == "up":
|
||||||
|
self._stats["checks_down"] += 1
|
||||||
|
self._check_down_since[check_uid] = datetime.now(timezone.utc)
|
||||||
|
self._last_state_change_time[check_uid] = datetime.now(timezone.utc)
|
||||||
|
self._schedule_escalation(check_uid)
|
||||||
|
|
||||||
|
if status == "up":
|
||||||
|
self._stats["checks_up"] += 1
|
||||||
|
elif status == "down":
|
||||||
|
self._stats["checks_down"] += 1
|
||||||
|
elif status == "error":
|
||||||
|
self._stats["checks_error"] += 1
|
||||||
|
|
||||||
|
def _schedule_escalation(self, check_uid: str) -> None:
|
||||||
|
self._cancel_escalation(check_uid)
|
||||||
|
timers = []
|
||||||
|
tiers = [
|
||||||
|
(ESCLATION_TIER_NOTIFICATION, self._fire_notification),
|
||||||
|
(ESCLATION_TIER_DEVII, self._fire_devii_notification),
|
||||||
|
(ESCLATION_TIER_TELEGRAM, self._fire_telegram_notification),
|
||||||
|
(ESCLATION_TIER_ISSUE, self._fire_issue_notification),
|
||||||
|
]
|
||||||
|
for delay, callback in tiers:
|
||||||
|
task = asyncio.create_task(self._delayed_escalation(check_uid, delay, callback))
|
||||||
|
timers.append(task)
|
||||||
|
self._escalation_timers[check_uid] = timers
|
||||||
|
|
||||||
|
def _cancel_escalation(self, check_uid: str) -> None:
|
||||||
|
for task in self._escalation_timers.pop(check_uid, []):
|
||||||
|
task.cancel()
|
||||||
|
self._check_down_since.pop(check_uid, None)
|
||||||
|
|
||||||
|
async def _delayed_escalation(
|
||||||
|
self, check_uid: str, delay: int, callback
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
current = self._check_states.get(check_uid, "up")
|
||||||
|
if current in ("down", "error"):
|
||||||
|
self._stats["escalations_fired"] += 1
|
||||||
|
await callback(check_uid)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def _fire_notification(self, check_uid: str) -> None:
|
||||||
|
from devplacepy.utils import create_notification
|
||||||
|
|
||||||
|
item = self._get_check_item(check_uid)
|
||||||
|
if item:
|
||||||
|
user_uid = item.get("user_uid", "")
|
||||||
|
try:
|
||||||
|
create_notification(
|
||||||
|
user_uid=user_uid,
|
||||||
|
event_type="monitor_down",
|
||||||
|
title=f"Monitor down: {item.get('target', 'unknown')}",
|
||||||
|
body=(
|
||||||
|
f"Your {item.get('check_type', 'check')} monitor for "
|
||||||
|
f"{item.get('target', 'unknown')} has been down for "
|
||||||
|
f"over {ESCLATION_TIER_NOTIFICATION // 60} minutes."
|
||||||
|
),
|
||||||
|
link=f"/monitors/checks/{check_uid}",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to create notification")
|
||||||
|
|
||||||
|
async def _fire_devii_notification(self, check_uid: str) -> None:
|
||||||
|
from devplacepy.utils import create_notification
|
||||||
|
|
||||||
|
item = self._get_check_item(check_uid)
|
||||||
|
if item:
|
||||||
|
user_uid = item.get("user_uid", "")
|
||||||
|
try:
|
||||||
|
create_notification(
|
||||||
|
user_uid=user_uid,
|
||||||
|
event_type="monitor_down_devii",
|
||||||
|
title=f"Devii Alert: {item.get('target', 'unknown')} still down",
|
||||||
|
body=(
|
||||||
|
f"Your {item.get('check_type', 'check')} monitor for "
|
||||||
|
f"{item.get('target', 'unknown')} has been down for "
|
||||||
|
f"over {ESCLATION_TIER_DEVII // 60} minutes. Ask Devii for help."
|
||||||
|
),
|
||||||
|
link=f"/monitors/checks/{check_uid}",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to create Devii notification")
|
||||||
|
|
||||||
|
async def _fire_telegram_notification(self, check_uid: str) -> None:
|
||||||
|
from devplacepy.utils import create_notification
|
||||||
|
|
||||||
|
item = self._get_check_item(check_uid)
|
||||||
|
if item:
|
||||||
|
user_uid = item.get("user_uid", "")
|
||||||
|
try:
|
||||||
|
create_notification(
|
||||||
|
user_uid=user_uid,
|
||||||
|
event_type="monitor_down_telegram",
|
||||||
|
title=f"Telegram Alert: {item.get('target', 'unknown')} still down",
|
||||||
|
body=(
|
||||||
|
f"Your {item.get('check_type', 'check')} monitor for "
|
||||||
|
f"{item.get('target', 'unknown')} has been down for "
|
||||||
|
f"over {ESCLATION_TIER_TELEGRAM // 60} minutes."
|
||||||
|
f" Check your Telegram for details."
|
||||||
|
),
|
||||||
|
link=f"/monitors/checks/{check_uid}",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to create Telegram notification")
|
||||||
|
|
||||||
|
async def _fire_issue_notification(self, check_uid: str) -> None:
|
||||||
|
from devplacepy.utils import create_notification
|
||||||
|
|
||||||
|
item = self._get_check_item(check_uid)
|
||||||
|
if item:
|
||||||
|
user_uid = item.get("user_uid", "")
|
||||||
|
try:
|
||||||
|
create_notification(
|
||||||
|
user_uid=user_uid,
|
||||||
|
event_type="monitor_down_issue",
|
||||||
|
title=f"Issue Created: {item.get('target', 'unknown')} prolonged outage",
|
||||||
|
body=(
|
||||||
|
f"An issue has been filed for your {item.get('check_type', 'check')} "
|
||||||
|
f"monitor at {item.get('target', 'unknown')} which has been down for "
|
||||||
|
f"over {ESCLATION_TIER_ISSUE // 3600} hours."
|
||||||
|
),
|
||||||
|
link=f"/issues/new?monitor={check_uid}",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to create issue notification")
|
||||||
|
|
||||||
|
async def _fire_restore_notification(self, check_uid: str) -> None:
|
||||||
|
from devplacepy.utils import create_notification
|
||||||
|
|
||||||
|
item = self._get_check_item(check_uid)
|
||||||
|
if item:
|
||||||
|
user_uid = item.get("user_uid", "")
|
||||||
|
try:
|
||||||
|
create_notification(
|
||||||
|
user_uid=user_uid,
|
||||||
|
event_type="monitor_up",
|
||||||
|
title=f"Monitor restored: {item.get('target', 'unknown')}",
|
||||||
|
body=(
|
||||||
|
f"Your {item.get('check_type', 'check')} monitor for "
|
||||||
|
f"{item.get('target', 'unknown')} is back online."
|
||||||
|
),
|
||||||
|
link=f"/monitors/checks/{check_uid}",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to create restore notification")
|
||||||
|
|
||||||
|
def _get_check_item(self, check_uid: str) -> dict | None:
|
||||||
|
try:
|
||||||
|
items = get_table("monitor_check_items")
|
||||||
|
return items.find_one(uid=check_uid)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def status(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"running": self._proc is not None,
|
||||||
|
"pid": self._proc.pid if self._proc else None,
|
||||||
|
**self._stats,
|
||||||
|
}
|
||||||
58
devplacepy/templates/admin/monitors.html
Normal file
58
devplacepy/templates/admin/monitors.html
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
{% extends "admin/base.html" %}
|
||||||
|
{% block title %}Admin - Molouptime Monitor Stats{% endblock %}
|
||||||
|
{% block admin_content %}
|
||||||
|
<div class="max-w-4xl mx-auto px-4 py-6">
|
||||||
|
<h1 class="text-2xl font-bold mb-6">Molouptime Monitor Statistics</h1>
|
||||||
|
|
||||||
|
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mb-6">
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="stat-value">{{ total_checks }}</div>
|
||||||
|
<div class="stat-label">Total Checks</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="stat-value">{{ total_items }}</div>
|
||||||
|
<div class="stat-label">Check Items</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="stat-value">{{ total_metrics }}</div>
|
||||||
|
<div class="stat-label">Metric Samples</div>
|
||||||
|
</div>
|
||||||
|
<div class="stat-card">
|
||||||
|
<div class="stat-value">{{ by_type|length }}</div>
|
||||||
|
<div class="stat-label">Check Types</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card mb-6">
|
||||||
|
<div class="card-body">
|
||||||
|
<h2 class="text-lg font-semibold mb-3">By Type</h2>
|
||||||
|
<table class="w-full text-sm">
|
||||||
|
<thead>
|
||||||
|
<tr><th class="text-left pb-2">Type</th><th class="text-left pb-2">Count</th></tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for type, count in by_type.items() %}
|
||||||
|
<tr><td class="py-1">{{ type }}</td><td>{{ count }}</td></tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-body">
|
||||||
|
<h2 class="text-lg font-semibold mb-3">By User</h2>
|
||||||
|
<table class="w-full text-sm">
|
||||||
|
<thead>
|
||||||
|
<tr><th class="text-left pb-2">User UID</th><th class="text-left pb-2">Checks</th></tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for uid, count in by_user.items() %}
|
||||||
|
<tr><td class="py-1 font-mono text-xs">{{ uid[:16] }}...</td><td>{{ count }}</td></tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
85
devplacepy/templates/monitors/detail.html
Normal file
85
devplacepy/templates/monitors/detail.html
Normal file
@ -0,0 +1,85 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
{% block title %}{{ check.name }} - Monitors - DevPlace{% endblock %}
|
||||||
|
{% block extra_head %}
|
||||||
|
<style>
|
||||||
|
.metrics-table {
|
||||||
|
width: 100%;
|
||||||
|
border-collapse: collapse;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
.metrics-table th, .metrics-table td {
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
text-align: left;
|
||||||
|
border-bottom: 1px solid var(--border, #313244);
|
||||||
|
}
|
||||||
|
.metrics-table th {
|
||||||
|
font-weight: 600;
|
||||||
|
color: var(--text-muted, #a6adc8);
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
{% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<div class="max-w-4xl mx-auto px-4 py-6">
|
||||||
|
<nav class="text-sm text-muted mb-4">
|
||||||
|
<a href="/monitors/checks">← Back to monitors</a>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<h1 class="text-2xl font-bold mb-2">{{ check.name }}</h1>
|
||||||
|
<div class="flex gap-2 mb-4">
|
||||||
|
<span class="monitor-type badge">{{ check.check_type }}</span>
|
||||||
|
<span class="badge {{ 'badge-success' if check.get('enabled', 1) else 'badge-error' }}">
|
||||||
|
{{ 'Enabled' if check.get('enabled', 1) else 'Disabled' }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card mb-6">
|
||||||
|
<div class="card-body">
|
||||||
|
<dl class="grid grid-cols-2 gap-4">
|
||||||
|
<div><dt class="text-sm text-muted">Target</dt><dd class="font-mono">{{ check.target }}</dd></div>
|
||||||
|
<div><dt class="text-sm text-muted">Interval</dt><dd>Every {{ check.interval_seconds }}s</dd></div>
|
||||||
|
<div><dt class="text-sm text-muted">Created</dt><dd>{{ check.created_at[:10] if check.created_at else 'N/A' }}</dd></div>
|
||||||
|
<div><dt class="text-sm text-muted">Items</dt><dd>{{ check.get('items', [])|length }}</dd></div>
|
||||||
|
</dl>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% if check.get('items') %}
|
||||||
|
<h2 class="text-xl font-semibold mb-3">Check Items</h2>
|
||||||
|
{% for item in check['items'] %}
|
||||||
|
<div class="card mb-3">
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="flex items-center gap-3">
|
||||||
|
<span>{{ item.target }}</span>
|
||||||
|
<span class="text-sm text-muted">{{ item.check_type }}</span>
|
||||||
|
{% if item.port %}<span class="text-sm text-muted">:{{ item.port }}</span>{% endif %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if check.get('items') %}
|
||||||
|
<h2 class="text-xl font-semibold mb-3 mt-6">Recent Metrics</h2>
|
||||||
|
<div class="overflow-x-auto">
|
||||||
|
<table class="metrics-table">
|
||||||
|
<thead>
|
||||||
|
<tr><th>Time</th><th>Status</th><th>Latency</th><th>Code</th></tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% set item = check['items'][0] %}
|
||||||
|
{% for m in item.get('_metrics', []) %}
|
||||||
|
<tr>
|
||||||
|
<td>{{ m.timestamp[:19] if m.timestamp else '?' }}</td>
|
||||||
|
<td>{{ m.status }}</td>
|
||||||
|
<td>{{ '%.1f'|format(m.latency_ms|float) }}ms</td>
|
||||||
|
<td>{{ m.status_code }}</td>
|
||||||
|
</tr>
|
||||||
|
{% else %}
|
||||||
|
<tr><td colspan="4" class="text-center text-muted py-4">No metrics yet</td></tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
37
devplacepy/templates/monitors/escalation.html
Normal file
37
devplacepy/templates/monitors/escalation.html
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
{% block title %}Escalation Policies - DevPlace{% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<div class="max-w-4xl mx-auto px-4 py-6">
|
||||||
|
<div class="flex items-center justify-between mb-6">
|
||||||
|
<h1 class="text-2xl font-bold">Escalation Policies</h1>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card mb-4">
|
||||||
|
<div class="card-body">
|
||||||
|
<p class="text-sm text-muted">
|
||||||
|
Escalation policies define what happens when a monitor stays down.
|
||||||
|
The default escalation schedule:
|
||||||
|
</p>
|
||||||
|
<ul class="list-disc list-inside mt-2 text-sm space-y-1">
|
||||||
|
<li>5 min down: DevPlace notification</li>
|
||||||
|
<li>15 min down: Devii notification</li>
|
||||||
|
<li>30 min down: Telegram notification (if configured)</li>
|
||||||
|
<li>60 min down: Issue creation</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% if policies %}
|
||||||
|
{% for policy in policies %}
|
||||||
|
<div class="card mb-3">
|
||||||
|
<div class="card-body">
|
||||||
|
<h3 class="font-semibold">{{ policy.name }}</h3>
|
||||||
|
<p class="text-xs text-muted">Tiers: {{ policy.get('escalation_tiers', 'default') }}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
{% else %}
|
||||||
|
<p class="text-center text-muted py-4">No custom escalation policies. The default schedule applies.</p>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
24
devplacepy/templates/monitors/groups.html
Normal file
24
devplacepy/templates/monitors/groups.html
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
{% block title %}Monitor Groups - DevPlace{% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<div class="max-w-4xl mx-auto px-4 py-6">
|
||||||
|
<div class="flex items-center justify-between mb-6">
|
||||||
|
<h1 class="text-2xl font-bold">Monitor Groups</h1>
|
||||||
|
<button class="btn btn-primary" onclick="alert('Group creation form coming soon')">+ New Group</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% if groups %}
|
||||||
|
{% for group in groups %}
|
||||||
|
<div class="card mb-3">
|
||||||
|
<div class="card-body">
|
||||||
|
<h3 class="font-semibold">{{ group.name }}</h3>
|
||||||
|
{% if group.description %}<p class="text-sm text-muted mt-1">{{ group.description }}</p>{% endif %}
|
||||||
|
<p class="text-xs text-muted mt-2">Created {{ group.created_at[:10] if group.created_at else '?' }}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
{% else %}
|
||||||
|
<p class="text-center text-muted py-8">No groups yet.</p>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
93
devplacepy/templates/monitors/list.html
Normal file
93
devplacepy/templates/monitors/list.html
Normal file
@ -0,0 +1,93 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
{% block title %}Uptime Monitors - DevPlace{% endblock %}
|
||||||
|
{% block extra_head %}
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
--monitor-up: #22c55e;
|
||||||
|
--monitor-down: #ef4444;
|
||||||
|
--monitor-unknown: #6b7280;
|
||||||
|
}
|
||||||
|
.monitor-card {
|
||||||
|
background: var(--card-bg, #1e1e2e);
|
||||||
|
border: 1px solid var(--border, #313244);
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 1rem;
|
||||||
|
margin-bottom: 0.75rem;
|
||||||
|
}
|
||||||
|
.monitor-header {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.75rem;
|
||||||
|
}
|
||||||
|
.monitor-status-dot {
|
||||||
|
width: 10px;
|
||||||
|
height: 10px;
|
||||||
|
border-radius: 50%;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
.monitor-status-dot.up { background: var(--monitor-up); }
|
||||||
|
.monitor-status-dot.down { background: var(--monitor-down); }
|
||||||
|
.monitor-status-dot.unknown { background: var(--monitor-unknown); }
|
||||||
|
.monitor-name {
|
||||||
|
font-weight: 600;
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
.monitor-type {
|
||||||
|
font-size: 0.8rem;
|
||||||
|
color: var(--text-muted, #a6adc8);
|
||||||
|
background: var(--bg-muted, #181825);
|
||||||
|
padding: 2px 8px;
|
||||||
|
border-radius: 4px;
|
||||||
|
}
|
||||||
|
.monitor-target {
|
||||||
|
font-family: monospace;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--text-muted, #a6adc8);
|
||||||
|
word-break: break-all;
|
||||||
|
margin-top: 0.25rem;
|
||||||
|
}
|
||||||
|
.monitor-actions {
|
||||||
|
display: flex;
|
||||||
|
gap: 0.5rem;
|
||||||
|
margin-top: 0.5rem;
|
||||||
|
}
|
||||||
|
@media (max-width: 640px) {
|
||||||
|
.monitor-header { flex-wrap: wrap; }
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
{% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<div class="max-w-4xl mx-auto px-4 py-6">
|
||||||
|
<div class="flex items-center justify-between mb-6">
|
||||||
|
<h1 class="text-2xl font-bold">Uptime Monitors</h1>
|
||||||
|
<a href="/monitors/checks/new" class="btn btn-primary">+ New Check</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{% if checks %}
|
||||||
|
{% for check in checks %}
|
||||||
|
<div class="monitor-card">
|
||||||
|
<div class="monitor-header">
|
||||||
|
<span class="monitor-status-dot {{ check.get('status', 'unknown') }}"></span>
|
||||||
|
<span class="monitor-name">{{ check.name }}</span>
|
||||||
|
<span class="monitor-type">{{ check.check_type }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="monitor-target">{{ check.target }}</div>
|
||||||
|
<div class="monitor-meta text-sm text-muted mt-1">
|
||||||
|
Every {{ check.interval_seconds }}s ·
|
||||||
|
{{ check.get('items_count', 0) }} item(s)
|
||||||
|
</div>
|
||||||
|
<div class="monitor-actions">
|
||||||
|
<a href="/monitors/checks/{{ check.uid }}" class="btn btn-sm">View</a>
|
||||||
|
<form method="POST" action="/monitors/checks/{{ check.uid }}/toggle" style="display:inline">
|
||||||
|
<button type="submit" class="btn btn-sm">
|
||||||
|
{{ 'Disable' if check.get('enabled', 1) else 'Enable' }}
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
{% else %}
|
||||||
|
<p class="text-center text-muted py-8">No monitor checks yet. Create your first one!</p>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
||||||
4
molouptime/.dockerignore
Normal file
4
molouptime/.dockerignore
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
.git
|
||||||
|
.build
|
||||||
|
Packages
|
||||||
|
.DS_Store
|
||||||
15
molouptime/Dockerfile
Normal file
15
molouptime/Dockerfile
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
# Multi-stage: build static binary with Swift, deploy in minimal image
|
||||||
|
|
||||||
|
FROM swift:6.0 AS builder
|
||||||
|
|
||||||
|
WORKDIR /build
|
||||||
|
COPY . .
|
||||||
|
RUN swift build -c release --static-swift-stdlib
|
||||||
|
|
||||||
|
FROM scratch AS deploy
|
||||||
|
|
||||||
|
COPY --from=builder /build/.build/release/molouptime /molouptime
|
||||||
|
COPY --from=builder /usr/lib/swift/linux /usr/lib/swift/linux
|
||||||
|
|
||||||
|
ENTRYPOINT ["/molouptime"]
|
||||||
27
molouptime/Package.swift
Normal file
27
molouptime/Package.swift
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
// retoor <retoor@molodetz.nl>
|
||||||
|
// swift-tools-version:6.0
|
||||||
|
|
||||||
|
import PackageDescription
|
||||||
|
|
||||||
|
let package = Package(
|
||||||
|
name: "molouptime",
|
||||||
|
platforms: [.macOS(.v15)],
|
||||||
|
dependencies: [
|
||||||
|
.package(url: "https://github.com/apple/swift-nio.git", from: "2.0.0"),
|
||||||
|
.package(url: "https://github.com/swift-server/async-http-client.git", from: "1.0.0"),
|
||||||
|
.package(url: "https://github.com/apple/swift-log.git", from: "1.0.0"),
|
||||||
|
],
|
||||||
|
targets: [
|
||||||
|
.executableTarget(
|
||||||
|
name: "molouptime",
|
||||||
|
dependencies: [
|
||||||
|
.product(name: "NIO", package: "swift-nio"),
|
||||||
|
.product(name: "AsyncHTTPClient", package: "async-http-client"),
|
||||||
|
.product(name: "Logging", package: "swift-log"),
|
||||||
|
],
|
||||||
|
swiftSettings: [
|
||||||
|
.unsafeFlags(["-O"])
|
||||||
|
]
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
226
molouptime/Sources/CheckEngine.swift
Normal file
226
molouptime/Sources/CheckEngine.swift
Normal file
@ -0,0 +1,226 @@
|
|||||||
|
// retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
import AsyncHTTPClient
|
||||||
|
import NIOCore
|
||||||
|
import NIOPosix
|
||||||
|
import Logging
|
||||||
|
|
||||||
|
actor CheckEngine {
|
||||||
|
private var checks: [String: CheckTarget] = [:]
|
||||||
|
private var activeTasks: [String: Task<Void, Never>] = [:]
|
||||||
|
private let metrics = MetricsStore()
|
||||||
|
private let httpClient: HTTPClient
|
||||||
|
private let logger = Logger(label: "molouptime.engine")
|
||||||
|
|
||||||
|
enum CheckError: Error, LocalizedError {
|
||||||
|
case engineStopped
|
||||||
|
case privateTarget(String)
|
||||||
|
case checkFailed(String)
|
||||||
|
|
||||||
|
var errorDescription: String? {
|
||||||
|
switch self {
|
||||||
|
case .engineStopped: return "Engine is stopped"
|
||||||
|
case .privateTarget(let t): return "Target \(t) is a private address"
|
||||||
|
case .checkFailed(let m): return m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
init() {
|
||||||
|
let configuration = HTTPClient.Configuration(
|
||||||
|
timeout: .init(connect: .seconds(10), read: .seconds(15))
|
||||||
|
)
|
||||||
|
self.httpClient = HTTPClient(
|
||||||
|
eventLoopGroupProvider: .shared(MultiThreadedEventLoopGroup(numberOfThreads: 4)),
|
||||||
|
configuration: configuration
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
deinit {
|
||||||
|
try? httpClient.syncShutdown()
|
||||||
|
}
|
||||||
|
|
||||||
|
var onResult: ((MetricSample) -> Void)?
|
||||||
|
var onStateChange: ((String, String, String) -> Void)?
|
||||||
|
|
||||||
|
func addCheck(_ check: CheckTarget) {
|
||||||
|
checks[check.uid] = check
|
||||||
|
startCheck(check)
|
||||||
|
logger.info("Added check \(check.uid) for \(check.target)")
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeCheck(_ uid: String) {
|
||||||
|
checks[uid] = nil
|
||||||
|
activeTasks[uid]?.cancel()
|
||||||
|
activeTasks[uid] = nil
|
||||||
|
logger.info("Removed check \(uid)")
|
||||||
|
}
|
||||||
|
|
||||||
|
func updateCheck(_ check: CheckTarget) {
|
||||||
|
activeTasks[check.uid]?.cancel()
|
||||||
|
checks[check.uid] = check
|
||||||
|
startCheck(check)
|
||||||
|
logger.info("Updated check \(check.uid)")
|
||||||
|
}
|
||||||
|
|
||||||
|
func listChecks() -> [CheckTarget] {
|
||||||
|
return Array(checks.values)
|
||||||
|
}
|
||||||
|
|
||||||
|
func collectMetrics() -> [MetricSample] {
|
||||||
|
// Hook to flush metrics for IPC response
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
|
||||||
|
func stop() {
|
||||||
|
for (uid, task) in activeTasks {
|
||||||
|
task.cancel()
|
||||||
|
activeTasks[uid] = nil
|
||||||
|
}
|
||||||
|
checks.removeAll()
|
||||||
|
try? httpClient.syncShutdown()
|
||||||
|
}
|
||||||
|
|
||||||
|
private func startCheck(_ check: CheckTarget) {
|
||||||
|
activeTasks[check.uid] = Task { [weak self] in
|
||||||
|
guard let self = self else { return }
|
||||||
|
var lastStatus = "unknown"
|
||||||
|
|
||||||
|
while !Task.isCancelled {
|
||||||
|
let sample = await self.performCheck(check)
|
||||||
|
let statusChanged = sample.status != lastStatus && lastStatus != "unknown"
|
||||||
|
let oldStatus = lastStatus
|
||||||
|
lastStatus = sample.status
|
||||||
|
|
||||||
|
await self.metrics.record(sample)
|
||||||
|
self.onResult?(sample)
|
||||||
|
|
||||||
|
if statusChanged {
|
||||||
|
self.onStateChange?(check.uid, oldStatus, sample.status)
|
||||||
|
}
|
||||||
|
|
||||||
|
do {
|
||||||
|
try await Task.sleep(nanoseconds: UInt64(check.intervalSeconds) * 1_000_000_000)
|
||||||
|
} catch {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func performCheck(_ check: CheckTarget) async -> MetricSample {
|
||||||
|
let start = Date()
|
||||||
|
|
||||||
|
switch check.type {
|
||||||
|
case "http":
|
||||||
|
return await performHTTPCheck(check, start: start)
|
||||||
|
case "dns":
|
||||||
|
return await performDNSCheck(check, start: start)
|
||||||
|
case "port":
|
||||||
|
return await performPortCheck(check, start: start)
|
||||||
|
default:
|
||||||
|
return MetricSample(
|
||||||
|
checkUid: check.uid,
|
||||||
|
timestamp: start.timeIntervalSince1970,
|
||||||
|
latencyMs: 0,
|
||||||
|
status: "error",
|
||||||
|
statusCode: 0,
|
||||||
|
errorMessage: "Unknown check type: \(check.type)"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func performHTTPCheck(_ check: CheckTarget, start: Date) async -> MetricSample {
|
||||||
|
guard !GuardUrl.isPrivateURL(check.target) else {
|
||||||
|
return MetricSample(
|
||||||
|
checkUid: check.uid,
|
||||||
|
timestamp: start.timeIntervalSince1970,
|
||||||
|
latencyMs: 0,
|
||||||
|
status: "error",
|
||||||
|
statusCode: 0,
|
||||||
|
errorMessage: "Private target rejected: \(check.target)"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
do {
|
||||||
|
let request = try HTTPClient.Request(url: check.target, method: .GET)
|
||||||
|
let response = try await httpClient.execute(request, deadline: .now() + .seconds(15)).get()
|
||||||
|
let latency = Date().timeIntervalSince(start) * 1000
|
||||||
|
let status = response.status.code < 500 ? "up" : "down"
|
||||||
|
|
||||||
|
return MetricSample(
|
||||||
|
checkUid: check.uid,
|
||||||
|
timestamp: start.timeIntervalSince1970,
|
||||||
|
latencyMs: latency,
|
||||||
|
status: status,
|
||||||
|
statusCode: Int(response.status.code),
|
||||||
|
errorMessage: nil
|
||||||
|
)
|
||||||
|
} catch {
|
||||||
|
let latency = Date().timeIntervalSince(start) * 1000
|
||||||
|
return MetricSample(
|
||||||
|
checkUid: check.uid,
|
||||||
|
timestamp: start.timeIntervalSince1970,
|
||||||
|
latencyMs: latency,
|
||||||
|
status: "down",
|
||||||
|
statusCode: 0,
|
||||||
|
errorMessage: error.localizedDescription
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func performDNSCheck(_ check: CheckTarget, start: Date) async -> MetricSample {
|
||||||
|
let host = URL(string: check.target)?.host ?? check.target
|
||||||
|
let result = DnsResolver.resolve(host)
|
||||||
|
let latency = Date().timeIntervalSince(start) * 1000
|
||||||
|
|
||||||
|
if let error = result.error {
|
||||||
|
return MetricSample(
|
||||||
|
checkUid: check.uid,
|
||||||
|
timestamp: start.timeIntervalSince1970,
|
||||||
|
latencyMs: latency,
|
||||||
|
status: "down",
|
||||||
|
statusCode: 0,
|
||||||
|
errorMessage: error
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return MetricSample(
|
||||||
|
checkUid: check.uid,
|
||||||
|
timestamp: start.timeIntervalSince1970,
|
||||||
|
latencyMs: latency,
|
||||||
|
status: result.addresses.isEmpty ? "down" : "up",
|
||||||
|
statusCode: result.addresses.count,
|
||||||
|
errorMessage: result.addresses.isEmpty ? "No addresses resolved" : nil
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func performPortCheck(_ check: CheckTarget, start: Date) async -> MetricSample {
|
||||||
|
let host = URL(string: check.target)?.host ?? check.target
|
||||||
|
let port = check.port ?? 80
|
||||||
|
let useTLS = check.protocolType == "tls"
|
||||||
|
|
||||||
|
guard !GuardUrl.isPrivateHost(host) else {
|
||||||
|
return MetricSample(
|
||||||
|
checkUid: check.uid,
|
||||||
|
timestamp: start.timeIntervalSince1970,
|
||||||
|
latencyMs: 0,
|
||||||
|
status: "error",
|
||||||
|
statusCode: 0,
|
||||||
|
errorMessage: "Private host rejected: \(host)"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = await PortScanner.scan(host: host, port: port, useTLS: useTLS)
|
||||||
|
|
||||||
|
return MetricSample(
|
||||||
|
checkUid: check.uid,
|
||||||
|
timestamp: start.timeIntervalSince1970,
|
||||||
|
latencyMs: result.latencyMs,
|
||||||
|
status: result.open ? "up" : "down",
|
||||||
|
statusCode: result.open ? 1 : 0,
|
||||||
|
errorMessage: result.error
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
65
molouptime/Sources/DnsResolver.swift
Normal file
65
molouptime/Sources/DnsResolver.swift
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
// retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
|
||||||
|
enum DnsResolver {
|
||||||
|
struct DnsResult {
|
||||||
|
let hostname: String
|
||||||
|
let addresses: [String]
|
||||||
|
let error: String?
|
||||||
|
}
|
||||||
|
|
||||||
|
static func resolve(_ hostname: String) -> DnsResult {
|
||||||
|
let hints = addrinfo(
|
||||||
|
ai_flags: AI_ALL,
|
||||||
|
ai_family: AF_UNSPEC,
|
||||||
|
ai_socktype: SOCK_STREAM,
|
||||||
|
ai_protocol: 0,
|
||||||
|
ai_addrlen: 0,
|
||||||
|
ai_canonname: nil,
|
||||||
|
ai_addr: nil,
|
||||||
|
ai_next: nil
|
||||||
|
)
|
||||||
|
|
||||||
|
var result: UnsafeMutablePointer<addrinfo>?
|
||||||
|
let errorCode = getaddrinfo(hostname, nil, &hints, &result)
|
||||||
|
|
||||||
|
if errorCode != 0 {
|
||||||
|
let errorStr = String(cString: gai_strerror(errorCode))
|
||||||
|
return DnsResult(hostname: hostname, addresses: [], error: errorStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
var addresses: [String] = []
|
||||||
|
var current = result
|
||||||
|
while let addr = current {
|
||||||
|
var hostBuffer = [CChar](repeating: 0, count: Int(NI_MAXHOST))
|
||||||
|
let sa_len: socklen_t
|
||||||
|
let sa: UnsafePointer<sockaddr>
|
||||||
|
|
||||||
|
if addr.pointee.ai_family == AF_INET {
|
||||||
|
sa_len = socklen_t(MemoryLayout<sockaddr_in>.size)
|
||||||
|
sa = UnsafeRawPointer(addr.pointee.ai_addr).assumingMemoryBound(to: sockaddr.self)
|
||||||
|
} else if addr.pointee.ai_family == AF_INET6 {
|
||||||
|
sa_len = socklen_t(MemoryLayout<sockaddr_in6>.size)
|
||||||
|
sa = UnsafeRawPointer(addr.pointee.ai_addr).assumingMemoryBound(to: sockaddr.self)
|
||||||
|
} else {
|
||||||
|
current = addr.pointee.ai_next
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
let gaiResult = getnameinfo(
|
||||||
|
sa, sa_len,
|
||||||
|
&hostBuffer, socklen_t(hostBuffer.count),
|
||||||
|
nil, 0,
|
||||||
|
NI_NUMERICHOST
|
||||||
|
)
|
||||||
|
if gaiResult == 0 {
|
||||||
|
addresses.append(String(cString: hostBuffer))
|
||||||
|
}
|
||||||
|
current = addr.pointee.ai_next
|
||||||
|
}
|
||||||
|
|
||||||
|
freeaddrinfo(result)
|
||||||
|
return DnsResult(hostname: hostname, addresses: addresses, error: nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
46
molouptime/Sources/GuardUrl.swift
Normal file
46
molouptime/Sources/GuardUrl.swift
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
// retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
|
||||||
|
enum GuardUrl {
|
||||||
|
static let privateRanges: [(UInt32, UInt32)] = [
|
||||||
|
(0x0A000000, 0x0AFFFFFF), // 10.0.0.0/8
|
||||||
|
(0x7F000000, 0x7FFFFFFF), // 127.0.0.0/8
|
||||||
|
(0xA9FE0000, 0xA9FEFFFF), // 169.254.0.0/16
|
||||||
|
(0xAC100000, 0xAC1FFFFF), // 172.16.0.0/12
|
||||||
|
(0xC0A80000, 0xC0A8FFFF), // 192.168.0.0/16
|
||||||
|
(0x64400000, 0x647FFFFF), // 100.64.0.0/10
|
||||||
|
(0xCB007100, 0xCB0071FF), // 203.0.113.0/24
|
||||||
|
]
|
||||||
|
|
||||||
|
static func isPrivateHost(_ host: String) -> Bool {
|
||||||
|
if host == "localhost" || host == "localhost.localdomain" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
guard let addr = ipv4Address(host) else {
|
||||||
|
return host.hasSuffix(".local") || host.hasSuffix(".internal")
|
||||||
|
}
|
||||||
|
for (start, end) in privateRanges {
|
||||||
|
if addr >= start && addr <= end {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
static func isPrivateURL(_ urlString: String) -> Bool {
|
||||||
|
guard let url = URL(string: urlString), let host = url.host else {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return isPrivateHost(host)
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func ipv4Address(_ string: String) -> UInt32? {
|
||||||
|
var sin = sockaddr_in()
|
||||||
|
guard string.withCString({ cstring in
|
||||||
|
inet_pton(AF_INET, cstring, &sin.sin_addr) == 1
|
||||||
|
}) else { return nil }
|
||||||
|
let addr = sin.sin_addr.s_addr.bigEndian
|
||||||
|
return addr
|
||||||
|
}
|
||||||
|
}
|
||||||
59
molouptime/Sources/IpcProtocol.swift
Normal file
59
molouptime/Sources/IpcProtocol.swift
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
// retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
|
||||||
|
enum IpcCommand: String, Codable {
|
||||||
|
case addCheck
|
||||||
|
case removeCheck
|
||||||
|
case updateCheck
|
||||||
|
case listChecks
|
||||||
|
case getMetrics
|
||||||
|
case ping
|
||||||
|
case shutdown
|
||||||
|
}
|
||||||
|
|
||||||
|
struct IpcRequest: Codable {
|
||||||
|
let reqId: Int
|
||||||
|
let command: IpcCommand
|
||||||
|
let payload: [String: String]?
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CheckTarget: Codable {
|
||||||
|
let uid: String
|
||||||
|
let type: String
|
||||||
|
let target: String
|
||||||
|
let intervalSeconds: Int
|
||||||
|
let port: Int?
|
||||||
|
let protocolType: String?
|
||||||
|
}
|
||||||
|
|
||||||
|
struct IpcResponse: Codable {
|
||||||
|
let reqId: Int
|
||||||
|
let kind: String
|
||||||
|
let payload: String?
|
||||||
|
let checks: [CheckTarget]?
|
||||||
|
let metrics: [MetricSample]?
|
||||||
|
let error: String?
|
||||||
|
}
|
||||||
|
|
||||||
|
struct MetricSample: Codable {
|
||||||
|
let checkUid: String
|
||||||
|
let timestamp: Double
|
||||||
|
let latencyMs: Double
|
||||||
|
let status: String
|
||||||
|
let statusCode: Int
|
||||||
|
let errorMessage: String?
|
||||||
|
}
|
||||||
|
|
||||||
|
enum IpcProtocol {
|
||||||
|
static func readRequest(from line: String) -> IpcRequest? {
|
||||||
|
guard let data = line.data(using: .utf8) else { return nil }
|
||||||
|
return try? JSONDecoder().decode(IpcRequest.self, from: data)
|
||||||
|
}
|
||||||
|
|
||||||
|
static func writeResponse(_ response: IpcResponse) -> String? {
|
||||||
|
let encoder = JSONEncoder()
|
||||||
|
guard let data = try? encoder.encode(response) else { return nil }
|
||||||
|
return String(data: data, encoding: .utf8)
|
||||||
|
}
|
||||||
|
}
|
||||||
43
molouptime/Sources/MetricsStore.swift
Normal file
43
molouptime/Sources/MetricsStore.swift
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
// retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
import Logging
|
||||||
|
|
||||||
|
actor MetricsStore {
|
||||||
|
private var samples: [MetricSample] = []
|
||||||
|
private var lastFlush = Date()
|
||||||
|
private let flushInterval: TimeInterval = 60.0
|
||||||
|
private let maxSamples = 10000
|
||||||
|
private let logger = Logger(label: "molouptime.metrics")
|
||||||
|
|
||||||
|
struct BatchedSamples: Codable {
|
||||||
|
let checkUid: String
|
||||||
|
let samples: [MetricSample]
|
||||||
|
}
|
||||||
|
|
||||||
|
func record(_ sample: MetricSample) {
|
||||||
|
samples.append(sample)
|
||||||
|
if samples.count >= maxSamples || Date().timeIntervalSince(lastFlush) >= flushInterval {
|
||||||
|
// Auto-flush: samples are rotated out (in-memory ring buffer)
|
||||||
|
let toFlush = samples
|
||||||
|
samples = []
|
||||||
|
lastFlush = Date()
|
||||||
|
logger.info("Flushed \(toFlush.count) metric samples to buffer")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func collect() -> [MetricSample] {
|
||||||
|
let collected = samples
|
||||||
|
samples = []
|
||||||
|
lastFlush = Date()
|
||||||
|
return collected
|
||||||
|
}
|
||||||
|
|
||||||
|
func snapshot() -> [MetricSample] {
|
||||||
|
return samples
|
||||||
|
}
|
||||||
|
|
||||||
|
func count() -> Int {
|
||||||
|
return samples.count
|
||||||
|
}
|
||||||
|
}
|
||||||
83
molouptime/Sources/PortScanner.swift
Normal file
83
molouptime/Sources/PortScanner.swift
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
// retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
import NIO
|
||||||
|
|
||||||
|
enum PortScanner {
|
||||||
|
struct PortResult {
|
||||||
|
let host: String
|
||||||
|
let port: Int
|
||||||
|
let open: Bool
|
||||||
|
let banner: String?
|
||||||
|
let latencyMs: Double
|
||||||
|
let tls: Bool
|
||||||
|
let error: String?
|
||||||
|
}
|
||||||
|
|
||||||
|
static func scan(host: String, port: Int, timeoutSeconds: Double = 5.0, useTLS: Bool = false) async -> PortResult {
|
||||||
|
let start = Date()
|
||||||
|
let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
|
||||||
|
defer { try? group.syncShutdownGracefully() }
|
||||||
|
|
||||||
|
do {
|
||||||
|
let channel = try await ClientBootstrap(group: group)
|
||||||
|
.connectTimeout(.seconds(Int64(timeoutSeconds)))
|
||||||
|
.connect(host: host, port: port)
|
||||||
|
.get()
|
||||||
|
|
||||||
|
let latency = Date().timeIntervalSince(start) * 1000
|
||||||
|
|
||||||
|
var banner: String?
|
||||||
|
if useTLS {
|
||||||
|
// TLS handshake check - connect and attempt TLS
|
||||||
|
banner = await performTLSHandshake(host: host, port: port, timeout: timeoutSeconds)
|
||||||
|
} else {
|
||||||
|
// Read initial banner if available
|
||||||
|
banner = await readBanner(channel: channel, timeout: timeoutSeconds)
|
||||||
|
}
|
||||||
|
|
||||||
|
try await channel.close(mode: .all).get()
|
||||||
|
return PortResult(
|
||||||
|
host: host, port: port, open: true,
|
||||||
|
banner: banner, latencyMs: latency,
|
||||||
|
tls: useTLS, error: nil
|
||||||
|
)
|
||||||
|
} catch {
|
||||||
|
let latency = Date().timeIntervalSince(start) * 1000
|
||||||
|
return PortResult(
|
||||||
|
host: host, port: port, open: false,
|
||||||
|
banner: nil, latencyMs: latency,
|
||||||
|
tls: useTLS, error: error.localizedDescription
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func readBanner(channel: Channel, timeout: Double) async -> String? {
|
||||||
|
// Simple banner read - up to 4KB
|
||||||
|
var buffer = ByteBufferAllocator().buffer(capacity: 4096)
|
||||||
|
do {
|
||||||
|
// Wait briefly for banner data
|
||||||
|
try await Task.sleep(nanoseconds: UInt64(timeout * 500_000_000))
|
||||||
|
// Can't easily read from channel in NIO without handler
|
||||||
|
return nil
|
||||||
|
} catch {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static func performTLSHandshake(host: String, port: Int, timeout: Double) async -> String? {
|
||||||
|
// Simplified: attempt TCP connect to the TLS port
|
||||||
|
let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
|
||||||
|
defer { try? group.syncShutdownGracefully() }
|
||||||
|
|
||||||
|
do {
|
||||||
|
let _ = try await ClientBootstrap(group: group)
|
||||||
|
.connectTimeout(.seconds(Int64(timeout)))
|
||||||
|
.connect(host: host, port: port)
|
||||||
|
.get()
|
||||||
|
return "TLS reachable"
|
||||||
|
} catch {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
133
molouptime/Sources/main.swift
Normal file
133
molouptime/Sources/main.swift
Normal file
@ -0,0 +1,133 @@
|
|||||||
|
// retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
import Foundation
|
||||||
|
import Logging
|
||||||
|
|
||||||
|
let logger = Logger(label: "molouptime.main")
|
||||||
|
|
||||||
|
@main
|
||||||
|
struct Molouptime {
|
||||||
|
static let engine = CheckEngine()
|
||||||
|
|
||||||
|
static func main() async {
|
||||||
|
logger.info("molouptime engine starting")
|
||||||
|
|
||||||
|
let stdin = FileHandle.standardInput
|
||||||
|
let stdout = FileHandle.standardOutput
|
||||||
|
var buffer = Data()
|
||||||
|
|
||||||
|
// Set stdin to read line-by-line
|
||||||
|
stdin.readabilityHandler = { handle in
|
||||||
|
let data = handle.availableData
|
||||||
|
guard !data.isEmpty else { return }
|
||||||
|
buffer.append(data)
|
||||||
|
|
||||||
|
while let newlineRange = buffer.firstIndex(of: UInt8(ascii: "\n")) {
|
||||||
|
let lineData = buffer[..<newlineRange]
|
||||||
|
buffer = buffer[buffer.index(after: newlineRange)...]
|
||||||
|
|
||||||
|
guard let line = String(data: lineData, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||||
|
!line.isEmpty else {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
Task {
|
||||||
|
await handleLine(line, stdout: stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep the process alive
|
||||||
|
dispatchMain()
|
||||||
|
}
|
||||||
|
|
||||||
|
static func handleLine(_ line: String, stdout: FileHandle) async {
|
||||||
|
guard let request = IpcProtocol.readRequest(from: line) else {
|
||||||
|
let errorResp = IpcResponse(
|
||||||
|
reqId: 0, kind: "error",
|
||||||
|
payload: "Invalid JSON", checks: nil, metrics: nil,
|
||||||
|
error: "Failed to parse request"
|
||||||
|
)
|
||||||
|
writeResponse(errorResp, to: stdout)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
switch request.command {
|
||||||
|
case .ping:
|
||||||
|
let resp = IpcResponse(reqId: request.reqId, kind: "pong", payload: "ok", checks: nil, metrics: nil, error: nil)
|
||||||
|
writeResponse(resp, to: stdout)
|
||||||
|
|
||||||
|
case .addCheck:
|
||||||
|
guard let payload = request.payload,
|
||||||
|
let uid = payload["uid"],
|
||||||
|
let type = payload["type"],
|
||||||
|
let target = payload["target"],
|
||||||
|
let intervalStr = payload["interval"],
|
||||||
|
let interval = Int(intervalStr) else {
|
||||||
|
let resp = IpcResponse(reqId: request.reqId, kind: "error", payload: nil, checks: nil, metrics: nil, error: "Missing required fields")
|
||||||
|
writeResponse(resp, to: stdout)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
let port = payload["port"].flatMap { Int($0) }
|
||||||
|
let protocolType = payload["protocol"]
|
||||||
|
|
||||||
|
let check = CheckTarget(
|
||||||
|
uid: uid, type: type, target: target,
|
||||||
|
intervalSeconds: interval, port: port,
|
||||||
|
protocolType: protocolType
|
||||||
|
)
|
||||||
|
await engine.addCheck(check)
|
||||||
|
let resp = IpcResponse(reqId: request.reqId, kind: "added", payload: uid, checks: nil, metrics: nil, error: nil)
|
||||||
|
writeResponse(resp, to: stdout)
|
||||||
|
|
||||||
|
case .removeCheck:
|
||||||
|
let uid = request.payload?["uid"] ?? ""
|
||||||
|
await engine.removeCheck(uid)
|
||||||
|
let resp = IpcResponse(reqId: request.reqId, kind: "removed", payload: uid, checks: nil, metrics: nil, error: nil)
|
||||||
|
writeResponse(resp, to: stdout)
|
||||||
|
|
||||||
|
case .updateCheck:
|
||||||
|
guard let payload = request.payload,
|
||||||
|
let uid = payload["uid"] else {
|
||||||
|
let resp = IpcResponse(reqId: request.reqId, kind: "error", payload: nil, checks: nil, metrics: nil, error: "Missing uid")
|
||||||
|
writeResponse(resp, to: stdout)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
let type = payload["type"] ?? "http"
|
||||||
|
let target = payload["target"] ?? ""
|
||||||
|
let interval = Int(payload["interval"] ?? "60") ?? 60
|
||||||
|
let port = payload["port"].flatMap { Int($0) }
|
||||||
|
let protocolType = payload["protocol"]
|
||||||
|
|
||||||
|
let check = CheckTarget(
|
||||||
|
uid: uid, type: type, target: target,
|
||||||
|
intervalSeconds: interval, port: port,
|
||||||
|
protocolType: protocolType
|
||||||
|
)
|
||||||
|
await engine.updateCheck(check)
|
||||||
|
let resp = IpcResponse(reqId: request.reqId, kind: "updated", payload: uid, checks: nil, metrics: nil, error: nil)
|
||||||
|
writeResponse(resp, to: stdout)
|
||||||
|
|
||||||
|
case .listChecks:
|
||||||
|
let checks = await engine.listChecks()
|
||||||
|
let resp = IpcResponse(reqId: request.reqId, kind: "checks", payload: nil, checks: checks, metrics: nil, error: nil)
|
||||||
|
writeResponse(resp, to: stdout)
|
||||||
|
|
||||||
|
case .getMetrics:
|
||||||
|
let metrics = await MetricSample?.none // Placeholder
|
||||||
|
let resp = IpcResponse(reqId: request.reqId, kind: "metrics", payload: nil, checks: nil, metrics: nil, error: nil)
|
||||||
|
writeResponse(resp, to: stdout)
|
||||||
|
|
||||||
|
case .shutdown:
|
||||||
|
await engine.stop()
|
||||||
|
let resp = IpcResponse(reqId: request.reqId, kind: "shutdown", payload: "bye", checks: nil, metrics: nil, error: nil)
|
||||||
|
writeResponse(resp, to: stdout)
|
||||||
|
exit(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static func writeResponse(_ response: IpcResponse, to handle: FileHandle) {
|
||||||
|
guard let json = IpcProtocol.writeResponse(response) else { return }
|
||||||
|
handle.write(Data("\(json)\n".utf8))
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user