diff --git a/.env.example b/.env.example index 6d4402e8..50e56fcb 100644 --- a/.env.example +++ b/.env.example @@ -42,3 +42,22 @@ NGINX_CACHE_MAX_SIZE=1g # Run the app container as this host user so shared files keep dev ownership. DEVPLACE_UID=1000 DEVPLACE_GID=1000 + +# DevTunnel (SSH reverse tunneling) +# SSH server port for *.tunnel.devplace.net +# DEVPLACE_TUNNEL_SSH_PORT=4242 +# Path to SSH host key (auto-generated if missing) +# DEVPLACE_TUNNEL_SSH_HOST_KEY= +# Auth API URL that the SSH server calls to validate credentials +# DEVPLACE_TUNNEL_AUTH_API_URL=http://app:10500/api/tunnel/auth-check +# Maximum concurrent tunnels per user +# DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER=5 +# Auth cache TTL in seconds +# DEVPLACE_TUNNEL_AUTH_CACHE_TTL=300 +# Remote port range for tunnel allocation +# DEVPLACE_TUNNEL_MIN_PORT=40000 +# DEVPLACE_TUNNEL_MAX_PORT=49999 +# Tunnel domain +# DEVPLACE_TUNNEL_DOMAIN=tunnel.devplace.net +# Directory for nginx tunnel snippets (mounted volume between nginx and ssh-server) +# DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR= diff --git a/Dockerfile.ssh b/Dockerfile.ssh new file mode 100644 index 00000000..09c5b346 --- /dev/null +++ b/Dockerfile.ssh @@ -0,0 +1,23 @@ +FROM python:3.13-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates \ + openssh-client \ + && rm -rf /var/lib/apt/lists/* + +COPY pyproject.toml . +COPY devplacepy/ devplacepy/ + +RUN pip install --no-cache-dir ".[dev]" + +EXPOSE 4242 + +ENV DEVPLACE_TUNNEL_SSH_PORT=4242 +ENV DEVPLACE_TEMPLATE_AUTO_RELOAD=0 + +HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=10s \ + CMD curl -f http://localhost:10500/ || exit 1 + +CMD ["python", "-m", "devplacepy.ssh_server_main"] diff --git a/devplacepy/cli/__init__.py b/devplacepy/cli/__init__.py index aa0570e1..b6db9a81 100644 --- a/devplacepy/cli/__init__.py +++ b/devplacepy/cli/__init__.py @@ -42,6 +42,11 @@ from devplacepy.cli.containers import ( cmd_containers_prune_builds, cmd_containers_gc_workspaces, ) +from devplacepy.cli.tunnel import ( + cmd_tunnel_list, + cmd_tunnel_kill, + cmd_tunnel_metrics, +) from devplacepy.cli.migrate import cmd_emoji_sync, cmd_migrate_data __all__ = [ @@ -84,6 +89,9 @@ __all__ = [ "cmd_containers_prune", "cmd_containers_prune_builds", "cmd_containers_gc_workspaces", + "cmd_tunnel_list", + "cmd_tunnel_kill", + "cmd_tunnel_metrics", "cmd_emoji_sync", "cmd_migrate_data", ] diff --git a/devplacepy/cli/main.py b/devplacepy/cli/main.py index 9861271c..66c9a101 100644 --- a/devplacepy/cli/main.py +++ b/devplacepy/cli/main.py @@ -11,6 +11,7 @@ from devplacepy.cli.devii import register_devii from devplacepy.cli.jobs import register_jobs from devplacepy.cli.backups import register_backups from devplacepy.cli.containers import register_containers +from devplacepy.cli.tunnel import register_tunnel from devplacepy.cli.migrate import register_migrate @@ -27,6 +28,7 @@ def build_parser(): register_jobs(sub) register_backups(sub) register_containers(sub) + register_tunnel(sub) register_migrate(sub) return parser diff --git a/devplacepy/cli/tunnel.py b/devplacepy/cli/tunnel.py new file mode 100644 index 00000000..166a61a7 --- /dev/null +++ b/devplacepy/cli/tunnel.py @@ -0,0 +1,89 @@ +# retoor + +from devplacepy.cli._shared import _audit_cli + + +def cmd_tunnel_list(args): + from devplacepy.services.tunnel.session_manager import get_manager + + manager = get_manager() + sessions = manager.list_by_user(args.username) if args.username else manager.list_active() + + if not sessions: + print("No active tunnels") + return + + for s in sessions: + sub = s.subdomain or "pending" + status = "active" if s.ended_at is None else "ended" + print( + f" {s.session_id[:8]} {sub:<20} port {s.local_port:>5} " + f"-> remote {s.remote_port:>5} {s.username:<15} {status}" + ) + + _audit_cli("cli.tunnel.list", "CLI listed active tunnels", metadata={"count": len(sessions)}) + + +def cmd_tunnel_kill(args): + from devplacepy.services.tunnel.session_manager import get_manager + from devplacepy.services.tunnel.subdomain_registry import get_registry + from devplacepy.services.tunnel.nginx_updater import remove_subdomain + + manager = get_manager() + registry = get_registry() + + session = manager.get(args.session_id) + if session is None: + print(f"Session '{args.session_id}' not found") + return + + manager.end_session(args.session_id) + released = registry.release(args.session_id) + if released: + remove_subdomain(released) + + print(f"Session '{args.session_id}' terminated") + _audit_cli( + "cli.tunnel.kill", + f"CLI killed tunnel session {args.session_id}", + metadata={"session_id": args.session_id, "username": session.username}, + ) + + +def cmd_tunnel_metrics(args): + from devplacepy.services.tunnel.metrics import get_metrics + + metrics = get_metrics() + print(f"Active tunnels: {metrics['active_tunnels']}") + print(f"Total bytes in: {metrics['total_bytes_in']}") + print(f"Total bytes out: {metrics['total_bytes_out']}") + print(f"Total requests: {metrics['total_requests']}") + print(f"Uptime (s): {metrics['uptime_seconds']:.0f}") + + if metrics["per_user"]: + print("\nPer user:") + for username, data in metrics["per_user"].items(): + print( + f" {username:<15} {data['active_tunnels']} tunnels " + f"{data['total_bytes_in']}B in {data['total_bytes_out']}B out" + ) + + _audit_cli("cli.tunnel.metrics", "CLI displayed tunnel metrics", metadata=metrics) + + +def register_tunnel(subparsers): + tunnel = subparsers.add_parser("tunnel", help="DevTunnel management") + tunnel_sub = tunnel.add_subparsers(title="action", dest="action") + + list_parser = tunnel_sub.add_parser("list", help="List active tunnels") + list_parser.add_argument( + "--username", "-u", help="Filter by username", default=None + ) + list_parser.set_defaults(func=cmd_tunnel_list) + + kill_parser = tunnel_sub.add_parser("kill", help="Kill a tunnel session") + kill_parser.add_argument("session_id", help="Session ID to terminate") + kill_parser.set_defaults(func=cmd_tunnel_kill) + + metrics_parser = tunnel_sub.add_parser("metrics", help="Show tunnel metrics") + metrics_parser.set_defaults(func=cmd_tunnel_metrics) diff --git a/devplacepy/config.py b/devplacepy/config.py index 57d68293..0d856016 100644 --- a/devplacepy/config.py +++ b/devplacepy/config.py @@ -94,6 +94,18 @@ INIT_LOCK_FILE = LOCKS_DIR / "devplace-init.lock" CONTAINER_IMAGE = environ.get("DEVPLACE_CONTAINER_IMAGE", "ppy:latest") CONTAINER_PROXY_HOST = environ.get("DEVPLACE_CONTAINER_PROXY_HOST", "").strip() +TUNNEL_SSH_PORT: int = int(environ.get("DEVPLACE_TUNNEL_SSH_PORT", "4242")) +TUNNEL_SSH_HOST_KEY: str = environ.get("DEVPLACE_TUNNEL_SSH_HOST_KEY", str(KEYS_DIR / "tunnel_host_key")) +TUNNEL_MIN_PORT: int = int(environ.get("DEVPLACE_TUNNEL_MIN_PORT", "40000")) +TUNNEL_MAX_PORT: int = int(environ.get("DEVPLACE_TUNNEL_MAX_PORT", "49999")) +TUNNEL_DOMAIN: str = environ.get("DEVPLACE_TUNNEL_DOMAIN", "tunnel.devplace.net") +TUNNEL_AUTH_CACHE_TTL: int = int(environ.get("DEVPLACE_TUNNEL_AUTH_CACHE_TTL", "300")) +TUNNEL_MAX_BANDWIDTH_PER_CLIENT: int = int(environ.get("DEVPLACE_TUNNEL_MAX_BANDWIDTH_PER_CLIENT", "10485760")) +TUNNEL_RATE_LIMIT_PER_USER: int = int(environ.get("DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER", "5")) +TUNNEL_SUBDOMAIN_REGISTRY_PATH: Path = KEYS_DIR / "tunnel_subdomains.json" +TUNNEL_AUTH_API_URL: str = environ.get("DEVPLACE_TUNNEL_AUTH_API_URL", "http://localhost:10500/api/tunnel/auth-check") +TUNNEL_NGINX_TUNNEL_DIR: Path = Path(environ.get("DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR", str(BASE_DIR / "nginx" / "tunnel.d"))) + VAPID_PRIVATE_KEY_FILE = KEYS_DIR / "notification-private.pem" VAPID_PRIVATE_KEY_PKCS8_FILE = KEYS_DIR / "notification-private.pkcs8.pem" VAPID_PUBLIC_KEY_FILE = KEYS_DIR / "notification-public.pem" @@ -122,6 +134,7 @@ DATA_PATHS: dict[str, Path] = { "keys": KEYS_DIR, "bot": BOT_DIR, "locks": LOCKS_DIR, + "tunnel_nginx": TUNNEL_NGINX_TUNNEL_DIR, } diff --git a/devplacepy/database/schema.py b/devplacepy/database/schema.py index c5845de8..50c1f507 100644 --- a/devplacepy/database/schema.py +++ b/devplacepy/database/schema.py @@ -1314,6 +1314,23 @@ def init_db(): _index(db, "jobs", "idx_jobs_created_at", ["created_at"]) _index(db, "attachments", "idx_attachments_created_at", ["created_at"]) + tunnel_sessions = get_table("tunnel_sessions") + for column, example in ( + ("uid", ""), + ("user_uid", ""), + ("subdomain", ""), + ("local_port", 0), + ("remote_port", 0), + ("username", ""), + ("started_at", ""), + ("ended_at", None), + ): + if not tunnel_sessions.has_column(column): + tunnel_sessions.create_column_by_example(column, example) + _index(db, "tunnel_sessions", "idx_tunnel_sessions_user", ["user_uid"]) + _index(db, "tunnel_sessions", "idx_tunnel_sessions_subdomain", ["subdomain"]) + _index(db, "tunnel_sessions", "idx_tunnel_sessions_status", ["ended_at"]) + _backfill_gamification() backfill_api_keys() migrate_ai_gateway_settings() diff --git a/devplacepy/main.py b/devplacepy/main.py index dcaa7787..c006876d 100644 --- a/devplacepy/main.py +++ b/devplacepy/main.py @@ -85,6 +85,7 @@ from devplacepy.routers import ( dbapi, pubsub, game, + tunnel, ) from devplacepy.services.manager import service_manager from devplacepy.services.background import background @@ -463,6 +464,7 @@ app.include_router(devrant.router, prefix="/api") app.include_router(dbapi.router, prefix="/dbapi") app.include_router(pubsub.router, prefix="/pubsub") app.include_router(game.router, prefix="/game") +app.include_router(tunnel.router) @app.middleware("http") diff --git a/devplacepy/routers/tunnel.py b/devplacepy/routers/tunnel.py new file mode 100644 index 00000000..1577a7e6 --- /dev/null +++ b/devplacepy/routers/tunnel.py @@ -0,0 +1,159 @@ +# retoor + +import logging +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field +from devplacepy.utils import require_user, require_admin + +logger = logging.getLogger(__name__) +router = APIRouter() + + +class AuthCheckIn(BaseModel): + username: str = Field(..., min_length=1, max_length=64) + password: str = Field(..., min_length=1, max_length=256) + + +class SubdomainClaimIn(BaseModel): + subdomain: str = Field(..., min_length=3, max_length=32, pattern=r"^[a-z0-9-]+$") + + +@router.post("/api/tunnel/auth-check") +async def tunnel_auth_check(data: AuthCheckIn) -> JSONResponse: + from devplacepy.database import get_table + + table = get_table("users") + user = table.find_one(username=data.username) + if user is None: + return JSONResponse( + content={"error": "Invalid credentials"}, status_code=401 + ) + + from passlib.hash import pbkdf2_sha256 + + if not pbkdf2_sha256.verify(data.password, user.get("password", "")): + return JSONResponse( + content={"error": "Invalid credentials"}, status_code=401 + ) + + return JSONResponse( + content={ + "username": user["username"], + "role": user.get("role", "Member"), + "uid": user["uid"], + } + ) + + +@router.get("/api/tunnel/sessions") +async def list_sessions(request: Request) -> JSONResponse: + require_admin(request) + from devplacepy.services.tunnel.session_manager import get_manager + + manager = get_manager() + sessions = manager.list_active() + return JSONResponse( + content={ + "sessions": [ + { + "session_id": s.session_id, + "username": s.username, + "subdomain": s.subdomain, + "local_port": s.local_port, + "remote_port": s.remote_port, + "started_at": s.started_at, + "bytes_in": s.bytes_in, + "bytes_out": s.bytes_out, + "requests_count": s.requests_count, + } + for s in sessions + ] + } + ) + + +@router.get("/api/tunnel/sessions/{session_id}") +async def get_session(request: Request, session_id: str) -> JSONResponse: + require_admin(request) + from devplacepy.services.tunnel.session_manager import get_manager + + manager = get_manager() + session = manager.get(session_id) + if session is None: + raise HTTPException(status_code=404, detail="Session not found") + + return JSONResponse( + content={ + "session_id": session.session_id, + "username": session.username, + "subdomain": session.subdomain, + "local_port": session.local_port, + "remote_port": session.remote_port, + "started_at": session.started_at, + "ended_at": session.ended_at, + "bytes_in": session.bytes_in, + "bytes_out": session.bytes_out, + "requests_count": session.requests_count, + } + ) + + +@router.delete("/api/tunnel/sessions/{session_id}") +async def kill_session(request: Request, session_id: str) -> JSONResponse: + require_admin(request) + from devplacepy.services.tunnel.session_manager import get_manager + from devplacepy.services.tunnel.subdomain_registry import get_registry + from devplacepy.services.tunnel.nginx_updater import remove_subdomain + + manager = get_manager() + registry = get_registry() + + session = manager.get(session_id) + if session is None: + raise HTTPException(status_code=404, detail="Session not found") + + manager.end_session(session_id) + released = registry.release(session_id) + if released: + remove_subdomain(released) + + return JSONResponse(content={"detail": "Session terminated"}) + + +@router.get("/api/tunnel/metrics") +async def tunnel_metrics(request: Request) -> JSONResponse: + require_admin(request) + from devplacepy.services.tunnel.metrics import get_metrics + + metrics = get_metrics() + return JSONResponse(content=metrics) + + +@router.get("/api/tunnel/metrics/{username}") +async def tunnel_user_metrics( + request: Request, username: str +) -> JSONResponse: + require_admin(request) + from devplacepy.services.tunnel.metrics import get_user_metrics + + metrics = get_user_metrics(username) + if metrics is None: + raise HTTPException(status_code=404, detail="No active sessions for user") + return JSONResponse(content=metrics) + + +@router.post("/api/tunnel/subdomains") +async def claim_subdomain( + request: Request, data: SubdomainClaimIn +) -> JSONResponse: + require_user(request) + from devplacepy.services.tunnel.subdomain_registry import get_registry + + registry = get_registry() + if registry.resolve(data.subdomain) is not None: + raise HTTPException(status_code=409, detail="Subdomain already claimed") + + return JSONResponse( + content={"subdomain": data.subdomain, "status": "available"} + ) diff --git a/devplacepy/services/tunnel/__init__.py b/devplacepy/services/tunnel/__init__.py new file mode 100644 index 00000000..95ee7c3e --- /dev/null +++ b/devplacepy/services/tunnel/__init__.py @@ -0,0 +1 @@ +# retoor diff --git a/devplacepy/services/tunnel/config.py b/devplacepy/services/tunnel/config.py new file mode 100644 index 00000000..6d2be30b --- /dev/null +++ b/devplacepy/services/tunnel/config.py @@ -0,0 +1,29 @@ +# retoor + +from devplacepy.config import ( + TUNNEL_SSH_PORT, + TUNNEL_SSH_HOST_KEY, + TUNNEL_MIN_PORT, + TUNNEL_MAX_PORT, + TUNNEL_DOMAIN, + TUNNEL_AUTH_CACHE_TTL, + TUNNEL_MAX_BANDWIDTH_PER_CLIENT, + TUNNEL_RATE_LIMIT_PER_USER, + TUNNEL_SUBDOMAIN_REGISTRY_PATH, + TUNNEL_AUTH_API_URL, + TUNNEL_NGINX_TUNNEL_DIR, +) + +__all__ = [ + "TUNNEL_SSH_PORT", + "TUNNEL_SSH_HOST_KEY", + "TUNNEL_MIN_PORT", + "TUNNEL_MAX_PORT", + "TUNNEL_DOMAIN", + "TUNNEL_AUTH_CACHE_TTL", + "TUNNEL_MAX_BANDWIDTH_PER_CLIENT", + "TUNNEL_RATE_LIMIT_PER_USER", + "TUNNEL_SUBDOMAIN_REGISTRY_PATH", + "TUNNEL_AUTH_API_URL", + "TUNNEL_NGINX_TUNNEL_DIR", +] diff --git a/devplacepy/services/tunnel/metrics.py b/devplacepy/services/tunnel/metrics.py new file mode 100644 index 00000000..5a079f27 --- /dev/null +++ b/devplacepy/services/tunnel/metrics.py @@ -0,0 +1,79 @@ +# retoor + +import time +from collections import defaultdict +from typing import Optional +from devplacepy.services.tunnel.session_manager import get_manager + + +def get_metrics() -> dict: + manager = get_manager() + active = manager.list_active() + now = time.time() + + per_user: dict[str, dict] = defaultdict( + lambda: { + "active_tunnels": 0, + "total_bytes_in": 0, + "total_bytes_out": 0, + "total_requests": 0, + } + ) + + total_bytes_in = 0 + total_bytes_out = 0 + total_requests = 0 + + for session in active: + username = session.username + per_user[username]["active_tunnels"] += 1 + per_user[username]["total_bytes_in"] += session.bytes_in + per_user[username]["total_bytes_out"] += session.bytes_out + per_user[username]["total_requests"] += session.requests_count + total_bytes_in += session.bytes_in + total_bytes_out += session.bytes_out + total_requests += session.requests_count + + return { + "active_tunnels": len(active), + "total_bytes_in": total_bytes_in, + "total_bytes_out": total_bytes_out, + "total_requests": total_requests, + "uptime_seconds": now - (min(s.started_at for s in active) if active else now), + "per_user": dict(per_user), + } + + +def get_user_metrics(username: str) -> Optional[dict]: + manager = get_manager() + active = manager.list_by_user(username) + now = time.time() + + if not active: + return None + + total_bytes_in = sum(s.bytes_in for s in active) + total_bytes_out = sum(s.bytes_out for s in active) + total_requests = sum(s.requests_count for s in active) + + return { + "username": username, + "active_tunnels": len(active), + "total_bytes_in": total_bytes_in, + "total_bytes_out": total_bytes_out, + "total_requests": total_requests, + "uptime_seconds": now - min(s.started_at for s in active), + "tunnels": [ + { + "session_id": s.session_id, + "subdomain": s.subdomain, + "local_port": s.local_port, + "remote_port": s.remote_port, + "started_at": s.started_at, + "bytes_in": s.bytes_in, + "bytes_out": s.bytes_out, + "requests_count": s.requests_count, + } + for s in active + ], + } diff --git a/devplacepy/services/tunnel/nginx_updater.py b/devplacepy/services/tunnel/nginx_updater.py new file mode 100644 index 00000000..6f40724f --- /dev/null +++ b/devplacepy/services/tunnel/nginx_updater.py @@ -0,0 +1,83 @@ +# retoor + +import logging +import subprocess +import threading +from pathlib import Path +from devplacepy.services.tunnel.config import TUNNEL_NGINX_TUNNEL_DIR + +logger = logging.getLogger(__name__) +_lock = threading.Lock() + +_TEMPLATE = """location / { + proxy_pass http://127.0.0.1:{remote_port}; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; +} +""" + + +def _tunnel_dir() -> Path: + return TUNNEL_NGINX_TUNNEL_DIR + + +def _ensure_tunnel_dir() -> None: + _tunnel_dir().mkdir(parents=True, exist_ok=True) + + +def add_subdomain(subdomain: str, remote_port: int) -> bool: + with _lock: + try: + _ensure_tunnel_dir() + snippet = _TEMPLATE.format(remote_port=remote_port) + config_path = _tunnel_dir() / f"{subdomain}.conf" + config_path.write_text(snippet) + logger.info( + "Wrote nginx tunnel snippet for %s -> port %d", + subdomain, + remote_port, + ) + _reload_nginx() + return True + except OSError: + logger.exception("Failed to write nginx config for %s", subdomain) + return False + + +def remove_subdomain(subdomain: str) -> bool: + with _lock: + config_path = _tunnel_dir() / f"{subdomain}.conf" + try: + if config_path.exists(): + config_path.unlink() + logger.info("Removed nginx tunnel snippet for %s", subdomain) + _reload_nginx() + return True + except OSError: + logger.exception("Failed to remove nginx config for %s", subdomain) + return False + + +def _reload_nginx() -> None: + try: + result = subprocess.run( + ["nginx", "-s", "reload"], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode != 0: + logger.error("nginx reload failed: %s", result.stderr.strip()) + else: + logger.info("nginx reloaded successfully") + except FileNotFoundError: + logger.warning("nginx command not found - skipping reload") + except (subprocess.TimeoutExpired, OSError): + logger.exception("nginx reload command failed") diff --git a/devplacepy/services/tunnel/server.py b/devplacepy/services/tunnel/server.py new file mode 100644 index 00000000..626a2f71 --- /dev/null +++ b/devplacepy/services/tunnel/server.py @@ -0,0 +1,212 @@ +# retoor + +import asyncio +import logging +import secrets +import string +from pathlib import Path +from typing import Optional + +import asyncssh + +from devplacepy.services.tunnel.config import ( + TUNNEL_SSH_PORT, + TUNNEL_SSH_HOST_KEY, + TUNNEL_DOMAIN, + TUNNEL_RATE_LIMIT_PER_USER, +) +from devplacepy.services.tunnel.ssh_auth import authenticate_user +from devplacepy.services.tunnel.session_manager import get_manager, TunnelSession +from devplacepy.services.tunnel.subdomain_registry import get_registry +from devplacepy.services.tunnel.nginx_updater import add_subdomain, remove_subdomain + +logger = logging.getLogger(__name__) + + +def _generate_subdomain() -> str: + return "t" + "".join(secrets.choice(string.ascii_lowercase) for _ in range(8)) + + +def _format_session_info(session: TunnelSession) -> str: + return ( + f" Public URL: https://{session.subdomain}.{TUNNEL_DOMAIN}/\n" + f" Local port: {session.local_port}\n" + f" Remote port: {session.remote_port}\n" + f" Session ID: {session.session_id}" + ) + + +async def _handle_interactive_session( + username: str, + stdin: asyncio.StreamReader, + stdout: asyncio.StreamWriter, + stderr: asyncio.StreamWriter, +) -> None: + writer = stdout + writer.write( + "\nWelcome to DevTunnel - expose your local services securely!\n" + "Authentication successful.\n\n" + ) + + local_port: Optional[int] = None + subdomain: Optional[str] = None + session: Optional[TunnelSession] = None + + while local_port is None: + writer.write("Enter the local port you want to forward: ") + line = await stdin.readline() + line = line.strip() + if not line: + continue + try: + port = int(line) + if port < 1 or port > 65535: + writer.write("Invalid port. Must be between 1 and 65535.\n") + continue + local_port = port + except ValueError: + writer.write("Invalid input. Please enter a numeric port.\n") + + manager = get_manager() + active_count = manager.user_active_count(username) + if active_count >= TUNNEL_RATE_LIMIT_PER_USER: + writer.write( + f"Error: you already have {active_count} active tunnels " + f"(limit {TUNNEL_RATE_LIMIT_PER_USER}).\n" + ) + return + + registry = get_registry() + + while subdomain is None: + writer.write("Enter a subdomain (or press Enter for auto-generated): ") + line = await stdin.readline() + line = line.strip() + + if line: + candidate = line.strip().lower() + if not candidate.replace("-", "").isalnum(): + writer.write( + "Invalid subdomain. Use only letters, numbers, and hyphens.\n" + ) + continue + if registry.resolve(candidate) is not None: + writer.write(f"Subdomain '{candidate}' is already taken.\n") + continue + subdomain = candidate + else: + subdomain = _generate_subdomain() + while registry.resolve(subdomain) is not None: + subdomain = _generate_subdomain() + + session = manager.create_session( + username=username, + local_port=local_port, + subdomain=subdomain, + ) + if session is None: + writer.write("Error: could not allocate remote port. All ports in use.\n") + return + + registry.claim(subdomain, session.session_id) + add_subdomain(subdomain, session.remote_port) + + final_cmd = ( + f"ssh -R {session.remote_port}:localhost:{local_port} " + f"-o ServerAliveInterval=30 " + f"-o ServerAliveCountMax=3 " + f"{username}@{TUNNEL_DOMAIN} -p {TUNNEL_SSH_PORT}" + ) + + writer.write("\n" + "=" * 60 + "\n") + writer.write("Tunnel ready!\n") + writer.write(_format_session_info(session) + "\n") + writer.write("=" * 60 + "\n") + writer.write("\n") + writer.write("Run this command in another terminal to establish the tunnel:\n") + writer.write("\n") + writer.write(f" {final_cmd}\n") + writer.write("\n") + writer.write( + f"After connecting, your service will be available at:\n" + f" https://{subdomain}.{TUNNEL_DOMAIN}/\n" + ) + writer.write("\n") + writer.write("Press Enter to disconnect... ") + + await stdin.readline() + + writer.write("\nDisconnecting.\n") + + if session: + manager.end_session(session.session_id) + released = registry.release(session.session_id) + if released: + remove_subdomain(released) + + +async def handle_connection(process: asyncssh.SSHServerProcess) -> None: + username = process.get_extra_info("username", "") + if not username: + process.exit(1) + return + + stdin = process.stdin if process.stdin is not None else None + stdout = process.stdout if process.stdout is not None else None + stderr = process.stderr if process.stderr is not None else None + + if stdin is None or stdout is None: + process.exit(1) + return + + try: + await _handle_interactive_session(username, stdin, stdout, stderr) + except (asyncio.CancelledError, asyncssh.BreakReceived): + pass + finally: + process.exit(0) + + +async def password_auth_handler(username: str, password: str) -> bool: + return await authenticate_user(username, password) + + +def _ensure_host_key() -> None: + import subprocess as _sp + + key_file = str(TUNNEL_SSH_HOST_KEY) + if not Path(key_file).exists(): + logger.info("Generating SSH host key at %s", key_file) + Path(key_file).parent.mkdir(parents=True, exist_ok=True) + _sp.run( + ["ssh-keygen", "-t", "ed25519", "-f", key_file, "-N", ""], + capture_output=True, + check=True, + timeout=30, + ) + + +async def start_ssh_server() -> None: + _ensure_host_key() + + key_file = str(TUNNEL_SSH_HOST_KEY) + + logger.info("Starting DevTunnel SSH server on port %d", TUNNEL_SSH_PORT) + + await asyncssh.create_server( + lambda: None, + None, + TUNNEL_SSH_PORT, + server_host_keys=[key_file], + authorization_errors=1, + password_auth=password_auth_handler, + process_factory=handle_connection, + keep_alive_interval=30, + keep_alive_count_max=3, + login_timeout=30, + max_auth_tries=3, + ) + + logger.info("DevTunnel SSH server running on 0.0.0.0:%d", TUNNEL_SSH_PORT) + + await asyncio.Event().wait() diff --git a/devplacepy/services/tunnel/session_manager.py b/devplacepy/services/tunnel/session_manager.py new file mode 100644 index 00000000..2b79e2b4 --- /dev/null +++ b/devplacepy/services/tunnel/session_manager.py @@ -0,0 +1,174 @@ +# retoor + +import json +import logging +import threading +import time +from dataclasses import dataclass, asdict +from pathlib import Path +from typing import Optional +from uuid_utils import uuid7 + +logger = logging.getLogger(__name__) +_lock = threading.Lock() + + +@dataclass +class TunnelSession: + session_id: str + username: str + subdomain: Optional[str] + local_port: int + remote_port: int + started_at: float + ended_at: Optional[float] = None + bytes_in: int = 0 + bytes_out: int = 0 + requests_count: int = 0 + + +class SessionManager: + def __init__(self, persist_path: Optional[Path] = None) -> None: + self._sessions: dict[str, TunnelSession] = {} + self._used_ports: set[int] = set() + self._port_range: tuple[int, int] = (40000, 49999) + self._persist_path: Optional[Path] = persist_path + if persist_path: + persist_path.parent.mkdir(parents=True, exist_ok=True) + self._load() + + def allocate_port(self) -> Optional[int]: + lo, hi = self._port_range + available = [p for p in range(lo, hi + 1) if p not in self._used_ports] + if not available: + return None + port = available[0] + self._used_ports.add(port) + return port + + def release_port(self, port: int) -> None: + self._used_ports.discard(port) + + def create_session( + self, + username: str, + local_port: int, + subdomain: Optional[str] = None, + ) -> Optional[TunnelSession]: + remote_port = self.allocate_port() + if remote_port is None: + return None + session_id = str(uuid7()) + session = TunnelSession( + session_id=session_id, + username=username, + subdomain=subdomain, + local_port=local_port, + remote_port=remote_port, + started_at=time.time(), + ) + with _lock: + self._sessions[session_id] = session + self._save() + return session + + def get(self, session_id: str) -> Optional[TunnelSession]: + with _lock: + return self._sessions.get(session_id) + + def get_by_remote_port(self, remote_port: int) -> Optional[TunnelSession]: + with _lock: + for s in self._sessions.values(): + if s.remote_port == remote_port: + return s + return None + + def get_by_subdomain(self, subdomain: str) -> Optional[TunnelSession]: + with _lock: + for s in self._sessions.values(): + if s.subdomain == subdomain: + return s + return None + + def list_active(self) -> list[TunnelSession]: + with _lock: + return [s for s in self._sessions.values() if s.ended_at is None] + + def list_by_user(self, username: str) -> list[TunnelSession]: + with _lock: + return [s for s in self._sessions.values() if s.username == username] + + def user_active_count(self, username: str) -> int: + with _lock: + return sum( + 1 for s in self._sessions.values() + if s.username == username and s.ended_at is None + ) + + def end_session(self, session_id: str) -> Optional[TunnelSession]: + with _lock: + session = self._sessions.get(session_id) + if session and session.ended_at is None: + session.ended_at = time.time() + self.release_port(session.remote_port) + self._save() + return session + + def record_traffic( + self, + session_id: str, + bytes_in: int = 0, + bytes_out: int = 0, + requests_count: int = 0, + ) -> None: + with _lock: + session = self._sessions.get(session_id) + if session: + session.bytes_in += bytes_in + session.bytes_out += bytes_out + session.requests_count += requests_count + + def _load(self) -> None: + path = self._persist_path + if not path or not path.exists(): + return + try: + raw = path.read_text() + data = json.loads(raw) + with _lock: + for item in data.get("sessions", []): + session = TunnelSession(**item) + self._sessions[session.session_id] = session + self._used_ports.add(session.remote_port) + except (json.JSONDecodeError, OSError, TypeError): + logger.exception("Failed to load session state") + + def _save(self) -> None: + path = self._persist_path + if not path: + return + try: + path.parent.mkdir(parents=True, exist_ok=True) + with _lock: + sessions = [asdict(s) for s in self._sessions.values()] + data = {"sessions": sessions} + path.write_text(json.dumps(data, indent=2)) + except OSError: + logger.exception("Failed to save session state") + + +_manager: Optional[SessionManager] = None + + +def get_manager() -> SessionManager: + global _manager + if _manager is None: + from devplacepy.config import DATA_DIR + path = DATA_DIR / "tunnel_sessions.json" + _manager = SessionManager(persist_path=path) + return _manager + + +def reset_manager() -> None: + global _manager + _manager = None diff --git a/devplacepy/services/tunnel/ssh_auth.py b/devplacepy/services/tunnel/ssh_auth.py new file mode 100644 index 00000000..688c4e99 --- /dev/null +++ b/devplacepy/services/tunnel/ssh_auth.py @@ -0,0 +1,87 @@ +# retoor + +import asyncio +import json +import logging +import time +from pathlib import Path +from typing import Optional +from devplacepy.services.tunnel.config import TUNNEL_AUTH_CACHE_TTL, TUNNEL_AUTH_API_URL + +logger = logging.getLogger(__name__) +_cache: dict[str, tuple[float, bool]] = {} +_cache_path: Optional[Path] = None + + +def _cache_file() -> Path: + global _cache_path + if _cache_path is None: + from devplacepy.config import KEYS_DIR + _cache_path = KEYS_DIR / "tunnel_auth_cache.json" + return _cache_path + + +def _load_cache() -> None: + global _cache + path = _cache_file() + if not path.exists(): + _cache = {} + return + try: + raw = path.read_text() + data = json.loads(raw) + now = time.time() + _cache = { + k: (v["expires_at"], v["valid"]) + for k, v in data.items() + if v.get("expires_at", 0) > now + } + except (json.JSONDecodeError, OSError): + _cache = {} + + +def _save_cache() -> None: + path = _cache_file() + try: + path.parent.mkdir(parents=True, exist_ok=True) + data = { + k: {"expires_at": exp, "valid": valid} + for k, (exp, valid) in _cache.items() + } + path.write_text(json.dumps(data, indent=2)) + except OSError: + logger.exception("Failed to write auth cache") + + +async def authenticate_user(username: str, password: str) -> bool: + cache_key = f"{username}:{password}" + now = time.time() + + if cache_key in _cache: + expires_at, valid = _cache[cache_key] + if expires_at > now: + if valid: + return True + del _cache[cache_key] + _save_cache() + + try: + import httpx + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.post( + TUNNEL_AUTH_API_URL, + json={"username": username, "password": password}, + ) + valid = resp.status_code == 200 + except httpx.RequestError: + logger.exception("Auth API request failed for %s", username) + await asyncio.sleep(3) + return False + + if valid: + _cache[cache_key] = (now + TUNNEL_AUTH_CACHE_TTL, True) + _save_cache() + return True + + await asyncio.sleep(3) + return False diff --git a/devplacepy/services/tunnel/subdomain_registry.py b/devplacepy/services/tunnel/subdomain_registry.py new file mode 100644 index 00000000..3f98ef82 --- /dev/null +++ b/devplacepy/services/tunnel/subdomain_registry.py @@ -0,0 +1,92 @@ +# retoor + +import json +import logging +import threading +from typing import Optional +from devplacepy.services.tunnel.config import TUNNEL_SUBDOMAIN_REGISTRY_PATH + +logger = logging.getLogger(__name__) +_lock = threading.Lock() + + +class SubdomainRegistry: + def __init__(self) -> None: + self._sub_to_session: dict[str, str] = {} + self._session_to_sub: dict[str, str] = {} + self._load() + + def _load(self) -> None: + path = TUNNEL_SUBDOMAIN_REGISTRY_PATH + if not path.exists(): + return + try: + raw = path.read_text() + data = json.loads(raw) + with _lock: + self._sub_to_session = data.get("sub_to_session", {}) + self._session_to_sub = data.get("session_to_sub", {}) + except (json.JSONDecodeError, OSError): + logger.exception("Failed to load subdomain registry") + + def _save(self) -> None: + path = TUNNEL_SUBDOMAIN_REGISTRY_PATH + try: + path.parent.mkdir(parents=True, exist_ok=True) + data = { + "sub_to_session": self._sub_to_session, + "session_to_sub": self._session_to_sub, + } + path.write_text(json.dumps(data, indent=2)) + except OSError: + logger.exception("Failed to save subdomain registry") + + def claim(self, subdomain: str, session_id: str) -> bool: + with _lock: + if subdomain in self._sub_to_session: + return False + self._sub_to_session[subdomain] = session_id + self._session_to_sub[session_id] = subdomain + self._save() + return True + + def release(self, session_id: str) -> Optional[str]: + with _lock: + sub = self._session_to_sub.pop(session_id, None) + if sub: + self._sub_to_session.pop(sub, None) + self._save() + return sub + + def resolve(self, subdomain: str) -> Optional[str]: + with _lock: + return self._sub_to_session.get(subdomain) + + def session_subdomain(self, session_id: str) -> Optional[str]: + with _lock: + return self._session_to_sub.get(session_id) + + def all_subdomains(self) -> dict[str, str]: + with _lock: + return dict(self._sub_to_session) + + def clear(self) -> None: + with _lock: + self._sub_to_session.clear() + self._session_to_sub.clear() + self._save() + + +_registry: Optional[SubdomainRegistry] = None + + +def get_registry() -> SubdomainRegistry: + global _registry + if _registry is None: + _registry = SubdomainRegistry() + return _registry + + +def reset_registry() -> None: + global _registry + _registry = None diff --git a/devplacepy/ssh_server_main.py b/devplacepy/ssh_server_main.py new file mode 100644 index 00000000..621a45b3 --- /dev/null +++ b/devplacepy/ssh_server_main.py @@ -0,0 +1,19 @@ +# retoor + +import asyncio +import logging + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", +) + + +async def main() -> None: + from devplacepy.services.tunnel.server import start_ssh_server + + await start_ssh_server() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docker-compose.yml b/docker-compose.yml index aa2d33b1..cd5afc91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,29 @@ services: volumes: - ./devplacepy/static:/app/static:ro - ${DEVPLACE_DATA_DIR:-./data}/uploads:/data/uploads:ro + - ./nginx/tunnel.d:/etc/nginx/tunnel.d + depends_on: + app: + condition: service_healthy + networks: + - appnet + + ssh-server: + build: + context: . + dockerfile: Dockerfile.ssh + restart: unless-stopped + env_file: + - .env + environment: + DEVPLACE_TUNNEL_SSH_PORT: "4242" + DEVPLACE_TUNNEL_AUTH_API_URL: "http://app:10500/api/tunnel/auth-check" + DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR: /etc/nginx/tunnel.d + ports: + - "4242:4242" + volumes: + - .:/app + - ./nginx/tunnel.d:/etc/nginx/tunnel.d depends_on: app: condition: service_healthy diff --git a/nginx/Dockerfile b/nginx/Dockerfile index d728e889..2cb85f53 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -2,6 +2,8 @@ FROM nginx:alpine RUN apk add --no-cache gettext +RUN mkdir -p /etc/nginx/tunnel.d + COPY nginx/nginx.conf.template /etc/nginx/templates/default.conf.template COPY nginx/start.sh /start.sh COPY devplacepy/static /app/static diff --git a/nginx/nginx.conf.template b/nginx/nginx.conf.template index e5337598..cfa41328 100644 --- a/nginx/nginx.conf.template +++ b/nginx/nginx.conf.template @@ -209,3 +209,25 @@ server { ${NGINX_CACHE_CONFIG} } } + +# DevTunnel reverse proxy - routes *.tunnel.devplace.net through SSH tunnels. +server { + listen 80; + listen [::]:80; + server_name ~^(?[^.]+)\.tunnel\.devplace\.net$; + + client_max_body_size ${NGINX_MAX_BODY_SIZE}; + + gzip on; + gzip_types text/plain text/css text/javascript application/javascript application/json image/svg+xml; + gzip_min_length 1000; + gzip_vary on; + gzip_proxied any; + + include /etc/nginx/tunnel.d/*.conf; + + location / { + return 502; + add_header Content-Type text/plain; + } +} diff --git a/pyproject.toml b/pyproject.toml index ccc102c5..7225cce7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "curl_cffi", "faker", "defusedxml", + "asyncssh", ] [project.scripts]