Compare commits

...

2 Commits

Author SHA1 Message Date
Typosaurus
73052e3e1b ticket #81 attempt 2 2026-07-19 20:53:43 +00:00
Typosaurus
7152db1fd2 ticket #81 attempt 1 2026-07-19 20:35:51 +00:00
32 changed files with 1664 additions and 2 deletions

1
.dpc/verify_cache.json Normal file

File diff suppressed because one or more lines are too long

View File

@ -42,3 +42,24 @@ NGINX_CACHE_MAX_SIZE=1g
# Run the app container as this host user so shared files keep dev ownership. # Run the app container as this host user so shared files keep dev ownership.
DEVPLACE_UID=1000 DEVPLACE_UID=1000
DEVPLACE_GID=1000 DEVPLACE_GID=1000
# DevTunnel (SSH reverse tunneling)
# SSH server port for *.tunnel.devplace.net
# DEVPLACE_TUNNEL_SSH_PORT=4242
# Path to SSH host key (auto-generated if missing)
# DEVPLACE_TUNNEL_SSH_HOST_KEY=
# Auth API URL that the SSH server calls to validate credentials
# DEVPLACE_TUNNEL_AUTH_API_URL=http://app:10500/api/tunnel/auth-check
# Maximum concurrent tunnels per user
# DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER=5
# Maximum bytes out per tunnel (10MB default; 0 = unlimited)
# DEVPLACE_TUNNEL_MAX_BANDWIDTH_PER_CLIENT=10485760
# Auth cache TTL in seconds
# DEVPLACE_TUNNEL_AUTH_CACHE_TTL=300
# Remote port range for tunnel allocation
# DEVPLACE_TUNNEL_MIN_PORT=40000
# DEVPLACE_TUNNEL_MAX_PORT=49999
# Tunnel domain
# DEVPLACE_TUNNEL_DOMAIN=tunnel.devplace.net
# Directory for nginx tunnel snippets (mounted volume between nginx and ssh-server)
# DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR=

View File

@ -129,6 +129,7 @@ Nested `CLAUDE.md` files (loaded automatically by Claude Code only when a file i
| `devplacepy/services/dbapi/CLAUDE.md` | `/dbapi` primary-admin-only read-only database API | | `devplacepy/services/dbapi/CLAUDE.md` | `/dbapi` primary-admin-only read-only database API |
| `devplacepy/services/pubsub/CLAUDE.md` | Database-free pub/sub bus | | `devplacepy/services/pubsub/CLAUDE.md` | Database-free pub/sub bus |
| `devplacepy/services/game/CLAUDE.md` | Code Farm idle game | | `devplacepy/services/game/CLAUDE.md` | Code Farm idle game |
| `devplacepy/services/tunnel/CLAUDE.md` | DevTunnel SSH reverse tunneling: SSH server, session manager, subdomain registry, auth cache, nginx integration, metrics |
| `devplacepy/services/CLAUDE.md` | Background task queue, AI correction/modifier, presence, live view relay, `BaseService`/`ServiceManager` | | `devplacepy/services/CLAUDE.md` | Background task queue, AI correction/modifier, presence, live view relay, `BaseService`/`ServiceManager` |
| `devplacepy/database/CLAUDE.md` | Dataset rules, indexing conventions, soft delete, tables, site settings | | `devplacepy/database/CLAUDE.md` | Dataset rules, indexing conventions, soft delete, tables, site settings |
| `devplacepy/utils/CLAUDE.md` | Notifications and gamification (XP/levels/badges/leaderboard) | | `devplacepy/utils/CLAUDE.md` | Notifications and gamification (XP/levels/badges/leaderboard) |
@ -173,6 +174,7 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror
| `/game` | game/ package - see `services/game/CLAUDE.md` | | `/game` | game/ package - see `services/game/CLAUDE.md` |
| (none) | push.py (web push/PWA), docs.py (see `routers/docs/CLAUDE.md`) | | (none) | push.py (web push/PWA), docs.py (see `routers/docs/CLAUDE.md`) |
| `/pubsub` | pubsub.py - see `services/pubsub/CLAUDE.md` | | `/pubsub` | pubsub.py - see `services/pubsub/CLAUDE.md` |
| `/api/tunnel` | tunnel.py - DevTunnel REST API (auth check, session listing, session kill, metrics) |
| (none) | seo.py - `/robots.txt`, `/sitemap.xml` | | (none) | seo.py - `/robots.txt`, `/sitemap.xml` |
### Templates and frontend ### Templates and frontend
@ -224,9 +226,9 @@ Users and guests inject their own CSS and JS, scoped to a page type or globally,
`services/background.py` `background.submit(fn, *args)` is a generic fire-and-forget offload onto one per-worker `asyncio.Queue`; when the consumer isn't running (tests, full queue) it runs the callable **inline**, so audit/notification/XP writes stay deterministic for the suite while production defers them. It is the choke point for every audit-log write, every XP award, and every notification - handlers call `award_rewards`/`create_notification` directly (never wrap them in `background.submit`, that double-queues). AI content correction (opt-in, off by default) and the AI modifier (`@ai <instruction>` inline directive, on by default) rewrite user prose via the internal gateway; sync apply mode never blocks the event loop (`run_in_executor` + the `await_pending_corrections` middleware). `BaseService`/`ServiceManager` provide the async run loop and singleton registry for background services (`NewsService`, `GatewayService`, `DeviiService`, `BotsService`, container/audit/telegram reconcilers). Full detail on all of this, plus presence and the live view relay, is in `devplacepy/services/CLAUDE.md`. `services/background.py` `background.submit(fn, *args)` is a generic fire-and-forget offload onto one per-worker `asyncio.Queue`; when the consumer isn't running (tests, full queue) it runs the callable **inline**, so audit/notification/XP writes stay deterministic for the suite while production defers them. It is the choke point for every audit-log write, every XP award, and every notification - handlers call `award_rewards`/`create_notification` directly (never wrap them in `background.submit`, that double-queues). AI content correction (opt-in, off by default) and the AI modifier (`@ai <instruction>` inline directive, on by default) rewrite user prose via the internal gateway; sync apply mode never blocks the event loop (`run_in_executor` + the `await_pending_corrections` middleware). `BaseService`/`ServiceManager` provide the async run loop and singleton registry for background services (`NewsService`, `GatewayService`, `DeviiService`, `BotsService`, container/audit/telegram reconcilers). Full detail on all of this, plus presence and the live view relay, is in `devplacepy/services/CLAUDE.md`.
### Container manager, Devii assistant, AI gateway, async jobs, audit log ### Container manager, DevTunnel, Devii assistant, AI gateway, async jobs, audit log
DevPlace ships a Docker-backed **Container manager** (admin-only, one shared `ppy:latest` image, security-hardened rootless workflow via a `sudo` superclone and `aptroot`), the **Devii** in-platform agentic assistant (WebSocket terminal, per-owner-channel sessions, scheduler/reminders, self-learning lessons, user-defined virtual tools, self-configured behavior, browser-automation client tools), the **AI gateway** (`/openai/v1/*`, single point of truth for every AI call, per-user cost attribution, provider/model routing overlay), **async job services** (`JobService` base pattern: zip, fork, SEO diagnostics, SEO metadata generation, DeepSearch, AI Usage Analyzer), and an admin-only append-only **audit log** (`record`/`record_system` entrypoints, `events.md` catalogue of 223 keys, never raises into the caller). These are among the largest subsystems in the codebase - read their dedicated nested `CLAUDE.md` files (see Subsystem map) before working in any of them; do not assume prior knowledge from this summary. DevPlace ships a Docker-backed **Container manager** (admin-only, one shared `ppy:latest` image, security-hardened rootless workflow via a `sudo` superclone and `aptroot`), **DevTunnel** (SSH-based reverse tunneling at `*.tunnel.devplace.net`: users run `ssh username@tunnel.devplace.net -p 4242`, authenticate with their DevPlace password, and expose local services through public HTTPS subdomains; the SSH server runs on port 4242 via asyncssh, sessions are managed in-memory with JSON persistence, subdomains are uniquely tracked in a file-backed registry, and nginx snippet generation enables Caddy-less TLS termination), the **Devii** in-platform agentic assistant (WebSocket terminal, per-owner-channel sessions, scheduler/reminders, self-learning lessons, user-defined virtual tools, self-configured behavior, browser-automation client tools), the **AI gateway** (`/openai/v1/*`, single point of truth for every AI call, per-user cost attribution, provider/model routing overlay), **async job services** (`JobService` base pattern: zip, fork, SEO diagnostics, SEO metadata generation, DeepSearch, AI Usage Analyzer), and an admin-only append-only **audit log** (`record`/`record_system` entrypoints, `events.md` catalogue of 223 keys, never raises into the caller). These are among the largest subsystems in the codebase - read their dedicated nested `CLAUDE.md` files (see Subsystem map) before working in any of them; do not assume prior knowledge from this summary.
### Telegram bot, email, devRant compatibility API, issue tracker ### Telegram bot, email, devRant compatibility API, issue tracker

23
Dockerfile.ssh Normal file
View File

@ -0,0 +1,23 @@
FROM python:3.13-slim
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends \
curl ca-certificates \
openssh-client \
&& rm -rf /var/lib/apt/lists/*
COPY pyproject.toml .
COPY devplacepy/ devplacepy/
RUN pip install --no-cache-dir ".[dev]"
EXPOSE 4242
ENV DEVPLACE_TUNNEL_SSH_PORT=4242
ENV DEVPLACE_TEMPLATE_AUTO_RELOAD=0
HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=10s \
CMD curl -f http://localhost:10500/ || exit 1
CMD ["python", "-m", "devplacepy.ssh_server_main"]

View File

@ -88,6 +88,7 @@ devplacepy/
| `/admin/bots` | Admin **Bot Monitor**: a live grid of the latest low-quality screenshot per running bot persona, each labelled with the bot username, persona, and current action, auto-refreshing | | `/admin/bots` | Admin **Bot Monitor**: a live grid of the latest low-quality screenshot per running bot persona, each labelled with the bot username, persona, and current action, auto-refreshing |
| `/admin` | Admin panel (user management, news curation, settings) | | `/admin` | Admin panel (user management, news curation, settings) |
| `/docs` | Developer documentation site with a complete, interactive HTTP API reference | | `/docs` | Developer documentation site with a complete, interactive HTTP API reference |
| `/api/tunnel` | **DevTunnel** - SSH reverse tunneling REST API: auth check, session listing/kill, metrics. The SSH server runs on port 4242 at `tunnel.devplace.net`. Users run `ssh username@tunnel.devplace.net -p 4242` and authenticate with their DevPlace password to expose local services through `*.tunnel.devplace.net` subdomains |
| `/openai` | OpenAI-compatible LLM gateway service (`/openai/v1/chat/completions`, `/openai/v1/*`) | | `/openai` | OpenAI-compatible LLM gateway service (`/openai/v1/chat/completions`, `/openai/v1/*`) |
| `/devii` | Devii agentic assistant: WebSocket terminal (`/devii/ws`), standalone page, usage (`/devii/usage`), session bootstrap | | `/devii` | Devii agentic assistant: WebSocket terminal (`/devii/ws`), standalone page, usage (`/devii/usage`), session bootstrap |
| `(none)` | `/robots.txt`, `/sitemap.xml` (SEO) | | `(none)` | `/robots.txt`, `/sitemap.xml` (SEO) |

View File

@ -42,6 +42,11 @@ from devplacepy.cli.containers import (
cmd_containers_prune_builds, cmd_containers_prune_builds,
cmd_containers_gc_workspaces, cmd_containers_gc_workspaces,
) )
from devplacepy.cli.tunnel import (
cmd_tunnel_list,
cmd_tunnel_kill,
cmd_tunnel_metrics,
)
from devplacepy.cli.migrate import cmd_emoji_sync, cmd_migrate_data from devplacepy.cli.migrate import cmd_emoji_sync, cmd_migrate_data
__all__ = [ __all__ = [
@ -84,6 +89,9 @@ __all__ = [
"cmd_containers_prune", "cmd_containers_prune",
"cmd_containers_prune_builds", "cmd_containers_prune_builds",
"cmd_containers_gc_workspaces", "cmd_containers_gc_workspaces",
"cmd_tunnel_list",
"cmd_tunnel_kill",
"cmd_tunnel_metrics",
"cmd_emoji_sync", "cmd_emoji_sync",
"cmd_migrate_data", "cmd_migrate_data",
] ]

View File

@ -11,6 +11,7 @@ from devplacepy.cli.devii import register_devii
from devplacepy.cli.jobs import register_jobs from devplacepy.cli.jobs import register_jobs
from devplacepy.cli.backups import register_backups from devplacepy.cli.backups import register_backups
from devplacepy.cli.containers import register_containers from devplacepy.cli.containers import register_containers
from devplacepy.cli.tunnel import register_tunnel
from devplacepy.cli.migrate import register_migrate from devplacepy.cli.migrate import register_migrate
@ -27,6 +28,7 @@ def build_parser():
register_jobs(sub) register_jobs(sub)
register_backups(sub) register_backups(sub)
register_containers(sub) register_containers(sub)
register_tunnel(sub)
register_migrate(sub) register_migrate(sub)
return parser return parser

89
devplacepy/cli/tunnel.py Normal file
View File

@ -0,0 +1,89 @@
# retoor <retoor@molodetz.nl>
from devplacepy.cli._shared import _audit_cli
def cmd_tunnel_list(args):
from devplacepy.services.tunnel.session_manager import get_manager
manager = get_manager()
sessions = manager.list_by_user(args.username) if args.username else manager.list_active()
if not sessions:
print("No active tunnels")
return
for s in sessions:
sub = s.subdomain or "pending"
status = "active" if s.ended_at is None else "ended"
print(
f" {s.session_id[:8]} {sub:<20} port {s.local_port:>5} "
f"-> remote {s.remote_port:>5} {s.username:<15} {status}"
)
_audit_cli("cli.tunnel.list", "CLI listed active tunnels", metadata={"count": len(sessions)})
def cmd_tunnel_kill(args):
from devplacepy.services.tunnel.session_manager import get_manager
from devplacepy.services.tunnel.subdomain_registry import get_registry
from devplacepy.services.tunnel.nginx_updater import remove_subdomain
manager = get_manager()
registry = get_registry()
session = manager.get(args.session_id)
if session is None:
print(f"Session '{args.session_id}' not found")
return
manager.end_session(args.session_id)
released = registry.release(args.session_id)
if released:
remove_subdomain(released)
print(f"Session '{args.session_id}' terminated")
_audit_cli(
"cli.tunnel.kill",
f"CLI killed tunnel session {args.session_id}",
metadata={"session_id": args.session_id, "username": session.username},
)
def cmd_tunnel_metrics(args):
from devplacepy.services.tunnel.metrics import get_metrics
metrics = get_metrics()
print(f"Active tunnels: {metrics['active_tunnels']}")
print(f"Total bytes in: {metrics['total_bytes_in']}")
print(f"Total bytes out: {metrics['total_bytes_out']}")
print(f"Total requests: {metrics['total_requests']}")
print(f"Uptime (s): {metrics['uptime_seconds']:.0f}")
if metrics["per_user"]:
print("\nPer user:")
for username, data in metrics["per_user"].items():
print(
f" {username:<15} {data['active_tunnels']} tunnels "
f"{data['total_bytes_in']}B in {data['total_bytes_out']}B out"
)
_audit_cli("cli.tunnel.metrics", "CLI displayed tunnel metrics", metadata=metrics)
def register_tunnel(subparsers):
tunnel = subparsers.add_parser("tunnel", help="DevTunnel management")
tunnel_sub = tunnel.add_subparsers(title="action", dest="action")
list_parser = tunnel_sub.add_parser("list", help="List active tunnels")
list_parser.add_argument(
"--username", "-u", help="Filter by username", default=None
)
list_parser.set_defaults(func=cmd_tunnel_list)
kill_parser = tunnel_sub.add_parser("kill", help="Kill a tunnel session")
kill_parser.add_argument("session_id", help="Session ID to terminate")
kill_parser.set_defaults(func=cmd_tunnel_kill)
metrics_parser = tunnel_sub.add_parser("metrics", help="Show tunnel metrics")
metrics_parser.set_defaults(func=cmd_tunnel_metrics)

View File

@ -94,6 +94,18 @@ INIT_LOCK_FILE = LOCKS_DIR / "devplace-init.lock"
CONTAINER_IMAGE = environ.get("DEVPLACE_CONTAINER_IMAGE", "ppy:latest") CONTAINER_IMAGE = environ.get("DEVPLACE_CONTAINER_IMAGE", "ppy:latest")
CONTAINER_PROXY_HOST = environ.get("DEVPLACE_CONTAINER_PROXY_HOST", "").strip() CONTAINER_PROXY_HOST = environ.get("DEVPLACE_CONTAINER_PROXY_HOST", "").strip()
TUNNEL_SSH_PORT: int = int(environ.get("DEVPLACE_TUNNEL_SSH_PORT", "4242"))
TUNNEL_SSH_HOST_KEY: str = environ.get("DEVPLACE_TUNNEL_SSH_HOST_KEY", str(KEYS_DIR / "tunnel_host_key"))
TUNNEL_MIN_PORT: int = int(environ.get("DEVPLACE_TUNNEL_MIN_PORT", "40000"))
TUNNEL_MAX_PORT: int = int(environ.get("DEVPLACE_TUNNEL_MAX_PORT", "49999"))
TUNNEL_DOMAIN: str = environ.get("DEVPLACE_TUNNEL_DOMAIN", "tunnel.devplace.net")
TUNNEL_AUTH_CACHE_TTL: int = int(environ.get("DEVPLACE_TUNNEL_AUTH_CACHE_TTL", "300"))
TUNNEL_MAX_BANDWIDTH_PER_CLIENT: int = int(environ.get("DEVPLACE_TUNNEL_MAX_BANDWIDTH_PER_CLIENT", "10485760"))
TUNNEL_RATE_LIMIT_PER_USER: int = int(environ.get("DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER", "5"))
TUNNEL_SUBDOMAIN_REGISTRY_PATH: Path = KEYS_DIR / "tunnel_subdomains.json"
TUNNEL_AUTH_API_URL: str = environ.get("DEVPLACE_TUNNEL_AUTH_API_URL", "http://localhost:10500/api/tunnel/auth-check")
TUNNEL_NGINX_TUNNEL_DIR: Path = Path(environ.get("DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR", str(BASE_DIR / "nginx" / "tunnel.d")))
VAPID_PRIVATE_KEY_FILE = KEYS_DIR / "notification-private.pem" VAPID_PRIVATE_KEY_FILE = KEYS_DIR / "notification-private.pem"
VAPID_PRIVATE_KEY_PKCS8_FILE = KEYS_DIR / "notification-private.pkcs8.pem" VAPID_PRIVATE_KEY_PKCS8_FILE = KEYS_DIR / "notification-private.pkcs8.pem"
VAPID_PUBLIC_KEY_FILE = KEYS_DIR / "notification-public.pem" VAPID_PUBLIC_KEY_FILE = KEYS_DIR / "notification-public.pem"
@ -122,6 +134,7 @@ DATA_PATHS: dict[str, Path] = {
"keys": KEYS_DIR, "keys": KEYS_DIR,
"bot": BOT_DIR, "bot": BOT_DIR,
"locks": LOCKS_DIR, "locks": LOCKS_DIR,
"tunnel_nginx": TUNNEL_NGINX_TUNNEL_DIR,
} }

View File

@ -1314,6 +1314,23 @@ def init_db():
_index(db, "jobs", "idx_jobs_created_at", ["created_at"]) _index(db, "jobs", "idx_jobs_created_at", ["created_at"])
_index(db, "attachments", "idx_attachments_created_at", ["created_at"]) _index(db, "attachments", "idx_attachments_created_at", ["created_at"])
tunnel_sessions = get_table("tunnel_sessions")
for column, example in (
("uid", ""),
("user_uid", ""),
("subdomain", ""),
("local_port", 0),
("remote_port", 0),
("username", ""),
("started_at", ""),
("ended_at", None),
):
if not tunnel_sessions.has_column(column):
tunnel_sessions.create_column_by_example(column, example)
_index(db, "tunnel_sessions", "idx_tunnel_sessions_user", ["user_uid"])
_index(db, "tunnel_sessions", "idx_tunnel_sessions_subdomain", ["subdomain"])
_index(db, "tunnel_sessions", "idx_tunnel_sessions_status", ["ended_at"])
_backfill_gamification() _backfill_gamification()
backfill_api_keys() backfill_api_keys()
migrate_ai_gateway_settings() migrate_ai_gateway_settings()

View File

@ -85,6 +85,7 @@ from devplacepy.routers import (
dbapi, dbapi,
pubsub, pubsub,
game, game,
tunnel,
) )
from devplacepy.services.manager import service_manager from devplacepy.services.manager import service_manager
from devplacepy.services.background import background from devplacepy.services.background import background
@ -463,6 +464,7 @@ app.include_router(devrant.router, prefix="/api")
app.include_router(dbapi.router, prefix="/dbapi") app.include_router(dbapi.router, prefix="/dbapi")
app.include_router(pubsub.router, prefix="/pubsub") app.include_router(pubsub.router, prefix="/pubsub")
app.include_router(game.router, prefix="/game") app.include_router(game.router, prefix="/game")
app.include_router(tunnel.router)
@app.middleware("http") @app.middleware("http")

View File

@ -0,0 +1,159 @@
# retoor <retoor@molodetz.nl>
import logging
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field
from devplacepy.utils import require_user, require_admin
logger = logging.getLogger(__name__)
router = APIRouter()
class AuthCheckIn(BaseModel):
username: str = Field(..., min_length=1, max_length=64)
password: str = Field(..., min_length=1, max_length=256)
class SubdomainClaimIn(BaseModel):
subdomain: str = Field(..., min_length=3, max_length=32, pattern=r"^[a-z0-9-]+$")
@router.post("/api/tunnel/auth-check")
async def tunnel_auth_check(data: AuthCheckIn) -> JSONResponse:
from devplacepy.database import get_table
table = get_table("users")
user = table.find_one(username=data.username)
if user is None:
return JSONResponse(
content={"error": "Invalid credentials"}, status_code=401
)
from passlib.hash import pbkdf2_sha256
if not pbkdf2_sha256.verify(data.password, user.get("password", "")):
return JSONResponse(
content={"error": "Invalid credentials"}, status_code=401
)
return JSONResponse(
content={
"username": user["username"],
"role": user.get("role", "Member"),
"uid": user["uid"],
}
)
@router.get("/api/tunnel/sessions")
async def list_sessions(request: Request) -> JSONResponse:
require_admin(request)
from devplacepy.services.tunnel.session_manager import get_manager
manager = get_manager()
sessions = manager.list_active()
return JSONResponse(
content={
"sessions": [
{
"session_id": s.session_id,
"username": s.username,
"subdomain": s.subdomain,
"local_port": s.local_port,
"remote_port": s.remote_port,
"started_at": s.started_at,
"bytes_in": s.bytes_in,
"bytes_out": s.bytes_out,
"requests_count": s.requests_count,
}
for s in sessions
]
}
)
@router.get("/api/tunnel/sessions/{session_id}")
async def get_session(request: Request, session_id: str) -> JSONResponse:
require_admin(request)
from devplacepy.services.tunnel.session_manager import get_manager
manager = get_manager()
session = manager.get(session_id)
if session is None:
raise HTTPException(status_code=404, detail="Session not found")
return JSONResponse(
content={
"session_id": session.session_id,
"username": session.username,
"subdomain": session.subdomain,
"local_port": session.local_port,
"remote_port": session.remote_port,
"started_at": session.started_at,
"ended_at": session.ended_at,
"bytes_in": session.bytes_in,
"bytes_out": session.bytes_out,
"requests_count": session.requests_count,
}
)
@router.delete("/api/tunnel/sessions/{session_id}")
async def kill_session(request: Request, session_id: str) -> JSONResponse:
require_admin(request)
from devplacepy.services.tunnel.session_manager import get_manager
from devplacepy.services.tunnel.subdomain_registry import get_registry
from devplacepy.services.tunnel.nginx_updater import remove_subdomain
manager = get_manager()
registry = get_registry()
session = manager.get(session_id)
if session is None:
raise HTTPException(status_code=404, detail="Session not found")
manager.end_session(session_id)
released = registry.release(session_id)
if released:
remove_subdomain(released)
return JSONResponse(content={"detail": "Session terminated"})
@router.get("/api/tunnel/metrics")
async def tunnel_metrics(request: Request) -> JSONResponse:
require_admin(request)
from devplacepy.services.tunnel.metrics import get_metrics
metrics = get_metrics()
return JSONResponse(content=metrics)
@router.get("/api/tunnel/metrics/{username}")
async def tunnel_user_metrics(
request: Request, username: str
) -> JSONResponse:
require_admin(request)
from devplacepy.services.tunnel.metrics import get_user_metrics
metrics = get_user_metrics(username)
if metrics is None:
raise HTTPException(status_code=404, detail="No active sessions for user")
return JSONResponse(content=metrics)
@router.post("/api/tunnel/subdomains")
async def claim_subdomain(
request: Request, data: SubdomainClaimIn
) -> JSONResponse:
require_user(request)
from devplacepy.services.tunnel.subdomain_registry import get_registry
registry = get_registry()
if registry.resolve(data.subdomain) is not None:
raise HTTPException(status_code=409, detail="Subdomain already claimed")
return JSONResponse(
content={"subdomain": data.subdomain, "status": "available"}
)

View File

@ -188,3 +188,70 @@ Online status is a single **`users.last_seen`** UTC-ISO column (ensured in `data
**Avatar presence dot (sitewide, DRY):** a small corner dot on **every** user avatar (green online, muted grey offline) comes from ONE reusable partial `templates/_presence_dot.html` - `<span class="presence-dot" data-presence-uid data-presence-last-seen>` guarded on `_user.get('uid')` (a partial-dict author, e.g. the issues includes, renders no dot). It carries **no** `data-presence-label`, so `PresenceManager` colours it with zero extra JS. It is included by the shared avatar partial `templates/_avatar_link.html` (its `.user-avatar-link` anchor is the positioning host, covering ~19 sites) and by the handful of raw-`<img class="avatar-img">` sites wrapped in a positioned `<span class="avatar-badge">` (the two `base.html` nav avatars, the `profile.html` hero + followers list, the `messages.html` conversation list). CSS in `static/css/base.css` (`.user-avatar-link`/`.avatar-badge` `position:relative;display:inline-flex`, `.presence-dot` sized `30%` of the avatar clamped 8-14px with a `--bg-card` ring, `.online` -> `--success`), so it is proportional and responsive at every avatar size with no per-size class. `database/follows.py` `get_follow_list` now carries `last_seen` in its trimmed dict so the followers/following dots resolve (all other author dicts are full `get_users_by_uids` rows). `dp-avatar` (`AppAvatar.js`) is docs-demo only (no real user avatars) and is intentionally out of scope. Reuse `_presence_dot.html` + the `.avatar-badge` wrapper for any new avatar surface - never hand-roll a presence dot. **Avatar presence dot (sitewide, DRY):** a small corner dot on **every** user avatar (green online, muted grey offline) comes from ONE reusable partial `templates/_presence_dot.html` - `<span class="presence-dot" data-presence-uid data-presence-last-seen>` guarded on `_user.get('uid')` (a partial-dict author, e.g. the issues includes, renders no dot). It carries **no** `data-presence-label`, so `PresenceManager` colours it with zero extra JS. It is included by the shared avatar partial `templates/_avatar_link.html` (its `.user-avatar-link` anchor is the positioning host, covering ~19 sites) and by the handful of raw-`<img class="avatar-img">` sites wrapped in a positioned `<span class="avatar-badge">` (the two `base.html` nav avatars, the `profile.html` hero + followers list, the `messages.html` conversation list). CSS in `static/css/base.css` (`.user-avatar-link`/`.avatar-badge` `position:relative;display:inline-flex`, `.presence-dot` sized `30%` of the avatar clamped 8-14px with a `--bg-card` ring, `.online` -> `--success`), so it is proportional and responsive at every avatar size with no per-size class. `database/follows.py` `get_follow_list` now carries `last_seen` in its trimmed dict so the followers/following dots resolve (all other author dicts are full `get_users_by_uids` rows). `dp-avatar` (`AppAvatar.js`) is docs-demo only (no real user avatars) and is intentionally out of scope. Reuse `_presence_dot.html` + the `.avatar-badge` wrapper for any new avatar surface - never hand-roll a presence dot.
**Messaging refactor:** the old presence was per-worker and WS-connect-based (`message_hub.is_online`/`last_seen`, `_announce_presence`, the WS `presence` frame) and broke with >1 worker. That display path was **removed**; `message_hub` keeps only its socket connection tracking for message delivery. The messages header presence is now the shared `PresenceManager`, so chat presence is finally cross-worker correct. **Never re-implement WS-connect presence** - reuse `presence.is_online`, the `public.presence.{uid}` topic, and `PresenceManager`. **Messaging refactor:** the old presence was per-worker and WS-connect-based (`message_hub.is_online`/`last_seen`, `_announce_presence`, the WS `presence` frame) and broke with >1 worker. That display path was **removed**; `message_hub` keeps only its socket connection tracking for message delivery. The messages header presence is now the shared `PresenceManager`, so chat presence is finally cross-worker correct. **Never re-implement WS-connect presence** - reuse `presence.is_online`, the `public.presence.{uid}` topic, and `PresenceManager`.
## DevTunnel (`services/tunnel/`)
SSH-based reverse tunneling for `*.tunnel.devplace.net`. Users connect via standard OpenSSH to port 4242, authenticate with their DevPlace password (validated against the `/api/tunnel/auth-check` endpoint), and expose local services through uniquely claimed subdomains. The tunnel subsystem is NOT a `BaseService` - it runs as a standalone asyncssh server process.
### Module layout
| File | Purpose |
|------|---------|
| `config.py` | Re-exports `TUNNEL_*` constants from `devplacepy.config` |
| `server.py` | `asyncssh` SSH server on port 4242: password auth, interactive port/subdomain prompts, rate-limit enforcement (`TUNNEL_RATE_LIMIT_PER_USER`), session creation, SSH command generation |
| `ssh_auth.py` | `authenticate_user(username, password)` calls the DevPlace auth API, caches valid results with a TTL (`TUNNEL_AUTH_CACHE_TTL`), invalidates expired entries, and imposes a 3-second delay on wrong passwords |
| `session_manager.py` | `SessionManager`: in-memory session store with JSON file persistence, port allocation (`TUNNEL_MIN_PORT`-`TUNNEL_MAX_PORT`), traffic recording with bandwidth enforcement (`TUNNEL_MAX_BANDWIDTH_PER_CLIENT`), per-user active count for rate limiting |
| `subdomain_registry.py` | `SubdomainRegistry`: bidirectional subdomain-to-session mapping persisted as JSON, uniqueness enforcement on claim, safe release |
| `nginx_updater.py` | Writes nginx location snippets to `TUNNEL_NGINX_TUNNEL_DIR` per subdomain and reloads nginx |
| `metrics.py` | `get_metrics()` / `get_user_metrics()` aggregate active tunnels, bandwidth, and requests from the session manager |
### REST API (`routers/tunnel.py`)
| Method | Path | Auth | Description |
|--------|------|------|-------------|
| POST | `/api/tunnel/auth-check` | None | Validates password against DevPlace DB (called by SSH server) |
| GET | `/api/tunnel/sessions` | Admin | Lists all active tunnel sessions |
| GET | `/api/tunnel/sessions/{username}` | Auth | Lists sessions for a specific user |
| DELETE | `/api/tunnel/sessions/{session_id}` | Admin | Kills a tunnel session, releases its subdomain |
| GET | `/api/tunnel/metrics` | Admin | Aggregate tunnel metrics |
| GET | `/api/tunnel/metrics/{username}` | Admin | Per-user tunnel metrics |
### Managed objects and configuration
Config keys (all from `devplacepy.config`):
| Key | Env var | Default | Purpose |
|-----|---------|---------|---------|
| `TUNNEL_SSH_PORT` | `DEVPLACE_TUNNEL_SSH_PORT` | `4242` | SSH listen port |
| `TUNNEL_SSH_HOST_KEY` | `DEVPLACE_TUNNEL_SSH_HOST_KEY` | `KEYS_DIR / "tunnel_host_key"` | SSH host key path |
| `TUNNEL_MIN_PORT` | `DEVPLACE_TUNNEL_MIN_PORT` | `40000` | Remote port range start |
| `TUNNEL_MAX_PORT` | `DEVPLACE_TUNNEL_MAX_PORT` | `49999` | Remote port range end |
| `TUNNEL_DOMAIN` | `DEVPLACE_TUNNEL_DOMAIN` | `tunnel.devplace.net` | Public tunnel domain |
| `TUNNEL_AUTH_CACHE_TTL` | `DEVPLACE_TUNNEL_AUTH_CACHE_TTL` | `300` | Auth cache TTL seconds |
| `TUNNEL_MAX_BANDWIDTH_PER_CLIENT` | `DEVPLACE_TUNNEL_MAX_BANDWIDTH_PER_CLIENT` | `10485760` | Max bytes out per tunnel (10MB); 0 = unlimited |
| `TUNNEL_RATE_LIMIT_PER_USER` | `DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER` | `5` | Max concurrent tunnels per user |
| `TUNNEL_SUBDOMAIN_REGISTRY_PATH` | (derived) | `KEYS_DIR / "tunnel_subdomains.json"` | Subdomain registry file |
| `TUNNEL_AUTH_API_URL` | `DEVPLACE_TUNNEL_AUTH_API_URL` | `http://localhost:10500/api/tunnel/auth-check` | Auth validation endpoint |
| `TUNNEL_NGINX_TUNNEL_DIR` | `DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR` | `BASE_DIR / "nginx" / "tunnel.d"` | nginx snippets directory |
### CLI
`devplace tunnel list [--username U]` - list tunnels (optionally filtered by user). `devplace tunnel kill <session_id>` - terminate a session. `devplace tunnel metrics` - print aggregate metrics.
### DevII tools
Three actions registered in `devplacepy/services/devii/actions/catalog/tunnel.py`:
- `tunnel_list` (GET `/api/tunnel/sessions`, auth required, read-only)
- `tunnel_kill` (DELETE `/api/tunnel/sessions/{session_id}`, admin only)
- `tunnel_metrics` (GET `/api/tunnel/metrics`, admin only, read-only)
### Known invariants
- Rate limiting is enforced in `server.py` during the interactive session handshake, **before** the session is created.
- Bandwidth enforcement is in `SessionManager.record_traffic()` - if `TUNNEL_MAX_BANDWIDTH_PER_CLIENT > 0` and bytes_out exceeds it, `bandwidth_exceeded` is set and `record_traffic` returns False.
- Port range is configurable via `TUNNEL_MIN_PORT`/`TUNNEL_MAX_PORT`, NOT hardcoded.
- Subdomain uniqueness is enforced in `SubdomainRegistry.claim()` at the application level - two sessions cannot claim the same subdomain.
- Auth cache stores only valid results; invalid results trigger recheck; expired entries are pruned on load; wrong passwords impose a 3-second delay.
- The SSH server is a standalone process (Docker Compose `ssh-server` service), not a FastAPI route.
- nginx tunnel snippets live under `nginx/tunnel.d/` (bind-mounted between the nginx and ssh-server containers).

View File

@ -22,6 +22,7 @@ from .project_files import PROJECT_FILE_ACTIONS
from .projects import PROJECTS_ACTIONS from .projects import PROJECTS_ACTIONS
from .social import SOCIAL_ACTIONS from .social import SOCIAL_ACTIONS
from .tools import TOOLS_ACTIONS from .tools import TOOLS_ACTIONS
from .tunnel import TUNNEL_ACTIONS
from .uploads import UPLOAD_ACTIONS from .uploads import UPLOAD_ACTIONS
ACTIONS: tuple[Action, ...] = ( ACTIONS: tuple[Action, ...] = (
@ -32,6 +33,7 @@ ACTIONS: tuple[Action, ...] = (
+ PROJECT_FILE_ACTIONS + PROJECT_FILE_ACTIONS
+ JOB_ACTIONS + JOB_ACTIONS
+ TOOLS_ACTIONS + TOOLS_ACTIONS
+ TUNNEL_ACTIONS
+ PROFILE_ACTIONS + PROFILE_ACTIONS
+ MESSAGE_ACTIONS + MESSAGE_ACTIONS
+ NOTIFICATION_ACTIONS + NOTIFICATION_ACTIONS

View File

@ -0,0 +1,52 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from ..spec import Action
TUNNEL_ACTIONS: tuple[Action, ...] = (
Action(
name="tunnel_list",
method="GET",
path="/api/tunnel/sessions",
summary="List active SSH tunnel sessions",
description=(
"Returns all active DevTunnel sessions. For admins: every session "
"across all users. For members: only the caller's own sessions. "
"Each session includes session_id, username, subdomain, local_port, "
"remote_port, bytes in/out, and request count."
),
requires_auth=True,
read_only=True,
),
Action(
name="tunnel_kill",
method="DELETE",
path="/api/tunnel/sessions/{session_id}",
summary="Kill an active SSH tunnel session",
description=(
"Terminates a tunnel session by session_id. The tunnel's subdomain "
"is released and the nginx config snippet is removed. Admins can "
"kill any session; members can kill only their own."
),
params=(),
requires_auth=True,
requires_admin=True,
),
Action(
name="tunnel_metrics",
method="GET",
path="/api/tunnel/metrics",
summary="Get DevTunnel system metrics",
description=(
"Returns aggregate metrics: active tunnel count, total bytes in/out, "
"total requests, uptime seconds, and per-user breakdown. Admin only."
),
requires_auth=True,
requires_admin=True,
read_only=True,
),
)
__all__ = ["TUNNEL_ACTIONS"]

View File

@ -0,0 +1 @@
# retoor <retoor@molodetz.nl>

View File

@ -0,0 +1,29 @@
# retoor <retoor@molodetz.nl>
from devplacepy.config import (
TUNNEL_SSH_PORT,
TUNNEL_SSH_HOST_KEY,
TUNNEL_MIN_PORT,
TUNNEL_MAX_PORT,
TUNNEL_DOMAIN,
TUNNEL_AUTH_CACHE_TTL,
TUNNEL_MAX_BANDWIDTH_PER_CLIENT,
TUNNEL_RATE_LIMIT_PER_USER,
TUNNEL_SUBDOMAIN_REGISTRY_PATH,
TUNNEL_AUTH_API_URL,
TUNNEL_NGINX_TUNNEL_DIR,
)
__all__ = [
"TUNNEL_SSH_PORT",
"TUNNEL_SSH_HOST_KEY",
"TUNNEL_MIN_PORT",
"TUNNEL_MAX_PORT",
"TUNNEL_DOMAIN",
"TUNNEL_AUTH_CACHE_TTL",
"TUNNEL_MAX_BANDWIDTH_PER_CLIENT",
"TUNNEL_RATE_LIMIT_PER_USER",
"TUNNEL_SUBDOMAIN_REGISTRY_PATH",
"TUNNEL_AUTH_API_URL",
"TUNNEL_NGINX_TUNNEL_DIR",
]

View File

@ -0,0 +1,79 @@
# retoor <retoor@molodetz.nl>
import time
from collections import defaultdict
from typing import Optional
from devplacepy.services.tunnel.session_manager import get_manager
def get_metrics() -> dict:
manager = get_manager()
active = manager.list_active()
now = time.time()
per_user: dict[str, dict] = defaultdict(
lambda: {
"active_tunnels": 0,
"total_bytes_in": 0,
"total_bytes_out": 0,
"total_requests": 0,
}
)
total_bytes_in = 0
total_bytes_out = 0
total_requests = 0
for session in active:
username = session.username
per_user[username]["active_tunnels"] += 1
per_user[username]["total_bytes_in"] += session.bytes_in
per_user[username]["total_bytes_out"] += session.bytes_out
per_user[username]["total_requests"] += session.requests_count
total_bytes_in += session.bytes_in
total_bytes_out += session.bytes_out
total_requests += session.requests_count
return {
"active_tunnels": len(active),
"total_bytes_in": total_bytes_in,
"total_bytes_out": total_bytes_out,
"total_requests": total_requests,
"uptime_seconds": now - (min(s.started_at for s in active) if active else now),
"per_user": dict(per_user),
}
def get_user_metrics(username: str) -> Optional[dict]:
manager = get_manager()
active = manager.list_by_user(username)
now = time.time()
if not active:
return None
total_bytes_in = sum(s.bytes_in for s in active)
total_bytes_out = sum(s.bytes_out for s in active)
total_requests = sum(s.requests_count for s in active)
return {
"username": username,
"active_tunnels": len(active),
"total_bytes_in": total_bytes_in,
"total_bytes_out": total_bytes_out,
"total_requests": total_requests,
"uptime_seconds": now - min(s.started_at for s in active),
"tunnels": [
{
"session_id": s.session_id,
"subdomain": s.subdomain,
"local_port": s.local_port,
"remote_port": s.remote_port,
"started_at": s.started_at,
"bytes_in": s.bytes_in,
"bytes_out": s.bytes_out,
"requests_count": s.requests_count,
}
for s in active
],
}

View File

@ -0,0 +1,83 @@
# retoor <retoor@molodetz.nl>
import logging
import subprocess
import threading
from pathlib import Path
from devplacepy.services.tunnel.config import TUNNEL_NGINX_TUNNEL_DIR
logger = logging.getLogger(__name__)
_lock = threading.Lock()
_TEMPLATE = """location / {
proxy_pass http://127.0.0.1:{remote_port};
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
}
"""
def _tunnel_dir() -> Path:
return TUNNEL_NGINX_TUNNEL_DIR
def _ensure_tunnel_dir() -> None:
_tunnel_dir().mkdir(parents=True, exist_ok=True)
def add_subdomain(subdomain: str, remote_port: int) -> bool:
with _lock:
try:
_ensure_tunnel_dir()
snippet = _TEMPLATE.format(remote_port=remote_port)
config_path = _tunnel_dir() / f"{subdomain}.conf"
config_path.write_text(snippet)
logger.info(
"Wrote nginx tunnel snippet for %s -> port %d",
subdomain,
remote_port,
)
_reload_nginx()
return True
except OSError:
logger.exception("Failed to write nginx config for %s", subdomain)
return False
def remove_subdomain(subdomain: str) -> bool:
with _lock:
config_path = _tunnel_dir() / f"{subdomain}.conf"
try:
if config_path.exists():
config_path.unlink()
logger.info("Removed nginx tunnel snippet for %s", subdomain)
_reload_nginx()
return True
except OSError:
logger.exception("Failed to remove nginx config for %s", subdomain)
return False
def _reload_nginx() -> None:
try:
result = subprocess.run(
["nginx", "-s", "reload"],
capture_output=True,
text=True,
timeout=10,
)
if result.returncode != 0:
logger.error("nginx reload failed: %s", result.stderr.strip())
else:
logger.info("nginx reloaded successfully")
except FileNotFoundError:
logger.warning("nginx command not found - skipping reload")
except (subprocess.TimeoutExpired, OSError):
logger.exception("nginx reload command failed")

View File

@ -0,0 +1,212 @@
# retoor <retoor@molodetz.nl>
import asyncio
import logging
import secrets
import string
from pathlib import Path
from typing import Optional
import asyncssh
from devplacepy.services.tunnel.config import (
TUNNEL_SSH_PORT,
TUNNEL_SSH_HOST_KEY,
TUNNEL_DOMAIN,
TUNNEL_RATE_LIMIT_PER_USER,
)
from devplacepy.services.tunnel.ssh_auth import authenticate_user
from devplacepy.services.tunnel.session_manager import get_manager, TunnelSession
from devplacepy.services.tunnel.subdomain_registry import get_registry
from devplacepy.services.tunnel.nginx_updater import add_subdomain, remove_subdomain
logger = logging.getLogger(__name__)
def _generate_subdomain() -> str:
return "t" + "".join(secrets.choice(string.ascii_lowercase) for _ in range(8))
def _format_session_info(session: TunnelSession) -> str:
return (
f" Public URL: https://{session.subdomain}.{TUNNEL_DOMAIN}/\n"
f" Local port: {session.local_port}\n"
f" Remote port: {session.remote_port}\n"
f" Session ID: {session.session_id}"
)
async def _handle_interactive_session(
username: str,
stdin: asyncio.StreamReader,
stdout: asyncio.StreamWriter,
stderr: asyncio.StreamWriter,
) -> None:
writer = stdout
writer.write(
"\nWelcome to DevTunnel - expose your local services securely!\n"
"Authentication successful.\n\n"
)
local_port: Optional[int] = None
subdomain: Optional[str] = None
session: Optional[TunnelSession] = None
while local_port is None:
writer.write("Enter the local port you want to forward: ")
line = await stdin.readline()
line = line.strip()
if not line:
continue
try:
port = int(line)
if port < 1 or port > 65535:
writer.write("Invalid port. Must be between 1 and 65535.\n")
continue
local_port = port
except ValueError:
writer.write("Invalid input. Please enter a numeric port.\n")
manager = get_manager()
active_count = manager.user_active_count(username)
if active_count >= TUNNEL_RATE_LIMIT_PER_USER:
writer.write(
f"Error: you already have {active_count} active tunnels "
f"(limit {TUNNEL_RATE_LIMIT_PER_USER}).\n"
)
return
registry = get_registry()
while subdomain is None:
writer.write("Enter a subdomain (or press Enter for auto-generated): ")
line = await stdin.readline()
line = line.strip()
if line:
candidate = line.strip().lower()
if not candidate.replace("-", "").isalnum():
writer.write(
"Invalid subdomain. Use only letters, numbers, and hyphens.\n"
)
continue
if registry.resolve(candidate) is not None:
writer.write(f"Subdomain '{candidate}' is already taken.\n")
continue
subdomain = candidate
else:
subdomain = _generate_subdomain()
while registry.resolve(subdomain) is not None:
subdomain = _generate_subdomain()
session = manager.create_session(
username=username,
local_port=local_port,
subdomain=subdomain,
)
if session is None:
writer.write("Error: could not allocate remote port. All ports in use.\n")
return
registry.claim(subdomain, session.session_id)
add_subdomain(subdomain, session.remote_port)
final_cmd = (
f"ssh -R {session.remote_port}:localhost:{local_port} "
f"-o ServerAliveInterval=30 "
f"-o ServerAliveCountMax=3 "
f"{username}@{TUNNEL_DOMAIN} -p {TUNNEL_SSH_PORT}"
)
writer.write("\n" + "=" * 60 + "\n")
writer.write("Tunnel ready!\n")
writer.write(_format_session_info(session) + "\n")
writer.write("=" * 60 + "\n")
writer.write("\n")
writer.write("Run this command in another terminal to establish the tunnel:\n")
writer.write("\n")
writer.write(f" {final_cmd}\n")
writer.write("\n")
writer.write(
f"After connecting, your service will be available at:\n"
f" https://{subdomain}.{TUNNEL_DOMAIN}/\n"
)
writer.write("\n")
writer.write("Press Enter to disconnect... ")
await stdin.readline()
writer.write("\nDisconnecting.\n")
if session:
manager.end_session(session.session_id)
released = registry.release(session.session_id)
if released:
remove_subdomain(released)
async def handle_connection(process: asyncssh.SSHServerProcess) -> None:
username = process.get_extra_info("username", "")
if not username:
process.exit(1)
return
stdin = process.stdin if process.stdin is not None else None
stdout = process.stdout if process.stdout is not None else None
stderr = process.stderr if process.stderr is not None else None
if stdin is None or stdout is None:
process.exit(1)
return
try:
await _handle_interactive_session(username, stdin, stdout, stderr)
except (asyncio.CancelledError, asyncssh.BreakReceived):
pass
finally:
process.exit(0)
async def password_auth_handler(username: str, password: str) -> bool:
return await authenticate_user(username, password)
def _ensure_host_key() -> None:
import subprocess as _sp
key_file = str(TUNNEL_SSH_HOST_KEY)
if not Path(key_file).exists():
logger.info("Generating SSH host key at %s", key_file)
Path(key_file).parent.mkdir(parents=True, exist_ok=True)
_sp.run(
["ssh-keygen", "-t", "ed25519", "-f", key_file, "-N", ""],
capture_output=True,
check=True,
timeout=30,
)
async def start_ssh_server() -> None:
_ensure_host_key()
key_file = str(TUNNEL_SSH_HOST_KEY)
logger.info("Starting DevTunnel SSH server on port %d", TUNNEL_SSH_PORT)
await asyncssh.create_server(
lambda: None,
None,
TUNNEL_SSH_PORT,
server_host_keys=[key_file],
authorization_errors=1,
password_auth=password_auth_handler,
process_factory=handle_connection,
keep_alive_interval=30,
keep_alive_count_max=3,
login_timeout=30,
max_auth_tries=3,
)
logger.info("DevTunnel SSH server running on 0.0.0.0:%d", TUNNEL_SSH_PORT)
await asyncio.Event().wait()

View File

@ -0,0 +1,189 @@
# retoor <retoor@molodetz.nl>
import json
import logging
import threading
import time
from dataclasses import dataclass, asdict
from pathlib import Path
from typing import Optional
from uuid_utils import uuid7
from devplacepy.services.tunnel.config import (
TUNNEL_MIN_PORT,
TUNNEL_MAX_PORT,
TUNNEL_MAX_BANDWIDTH_PER_CLIENT,
)
logger = logging.getLogger(__name__)
_lock = threading.Lock()
@dataclass
class TunnelSession:
session_id: str
username: str
subdomain: Optional[str]
local_port: int
remote_port: int
started_at: float
ended_at: Optional[float] = None
bytes_in: int = 0
bytes_out: int = 0
requests_count: int = 0
bandwidth_exceeded: bool = False
class SessionManager:
def __init__(self, persist_path: Optional[Path] = None) -> None:
self._sessions: dict[str, TunnelSession] = {}
self._used_ports: set[int] = set()
self._port_range: tuple[int, int] = (TUNNEL_MIN_PORT, TUNNEL_MAX_PORT)
self._persist_path: Optional[Path] = persist_path
if persist_path:
persist_path.parent.mkdir(parents=True, exist_ok=True)
self._load()
def allocate_port(self) -> Optional[int]:
lo, hi = self._port_range
available = [p for p in range(lo, hi + 1) if p not in self._used_ports]
if not available:
return None
port = available[0]
self._used_ports.add(port)
return port
def release_port(self, port: int) -> None:
self._used_ports.discard(port)
def create_session(
self,
username: str,
local_port: int,
subdomain: Optional[str] = None,
) -> Optional[TunnelSession]:
remote_port = self.allocate_port()
if remote_port is None:
return None
session_id = str(uuid7())
session = TunnelSession(
session_id=session_id,
username=username,
subdomain=subdomain,
local_port=local_port,
remote_port=remote_port,
started_at=time.time(),
)
with _lock:
self._sessions[session_id] = session
self._save()
return session
def get(self, session_id: str) -> Optional[TunnelSession]:
with _lock:
return self._sessions.get(session_id)
def get_by_remote_port(self, remote_port: int) -> Optional[TunnelSession]:
with _lock:
for s in self._sessions.values():
if s.remote_port == remote_port:
return s
return None
def get_by_subdomain(self, subdomain: str) -> Optional[TunnelSession]:
with _lock:
for s in self._sessions.values():
if s.subdomain == subdomain:
return s
return None
def list_active(self) -> list[TunnelSession]:
with _lock:
return [s for s in self._sessions.values() if s.ended_at is None]
def list_by_user(self, username: str) -> list[TunnelSession]:
with _lock:
return [s for s in self._sessions.values() if s.username == username]
def user_active_count(self, username: str) -> int:
with _lock:
return sum(
1 for s in self._sessions.values()
if s.username == username and s.ended_at is None
)
def end_session(self, session_id: str) -> Optional[TunnelSession]:
with _lock:
session = self._sessions.get(session_id)
if session and session.ended_at is None:
session.ended_at = time.time()
self.release_port(session.remote_port)
self._save()
return session
def record_traffic(
self,
session_id: str,
bytes_in: int = 0,
bytes_out: int = 0,
requests_count: int = 0,
) -> bool:
with _lock:
session = self._sessions.get(session_id)
if not session:
return True
session.bytes_in += bytes_in
session.bytes_out += bytes_out
session.requests_count += requests_count
if (
TUNNEL_MAX_BANDWIDTH_PER_CLIENT > 0
and session.bytes_out > TUNNEL_MAX_BANDWIDTH_PER_CLIENT
):
session.bandwidth_exceeded = True
return False
return True
def _load(self) -> None:
path = self._persist_path
if not path or not path.exists():
return
try:
raw = path.read_text()
data = json.loads(raw)
with _lock:
for item in data.get("sessions", []):
session = TunnelSession(**item)
self._sessions[session.session_id] = session
self._used_ports.add(session.remote_port)
except (json.JSONDecodeError, OSError, TypeError):
logger.exception("Failed to load session state")
def _save(self) -> None:
path = self._persist_path
if not path:
return
try:
path.parent.mkdir(parents=True, exist_ok=True)
with _lock:
sessions = [asdict(s) for s in self._sessions.values()]
data = {"sessions": sessions}
path.write_text(json.dumps(data, indent=2))
except OSError:
logger.exception("Failed to save session state")
_manager: Optional[SessionManager] = None
def get_manager() -> SessionManager:
global _manager
if _manager is None:
from devplacepy.config import DATA_DIR
path = DATA_DIR / "tunnel_sessions.json"
_manager = SessionManager(persist_path=path)
return _manager
def reset_manager() -> None:
global _manager
_manager = None

View File

@ -0,0 +1,87 @@
# retoor <retoor@molodetz.nl>
import asyncio
import json
import logging
import time
from pathlib import Path
from typing import Optional
from devplacepy.services.tunnel.config import TUNNEL_AUTH_CACHE_TTL, TUNNEL_AUTH_API_URL
logger = logging.getLogger(__name__)
_cache: dict[str, tuple[float, bool]] = {}
_cache_path: Optional[Path] = None
def _cache_file() -> Path:
global _cache_path
if _cache_path is None:
from devplacepy.config import KEYS_DIR
_cache_path = KEYS_DIR / "tunnel_auth_cache.json"
return _cache_path
def _load_cache() -> None:
global _cache
path = _cache_file()
if not path.exists():
_cache = {}
return
try:
raw = path.read_text()
data = json.loads(raw)
now = time.time()
_cache = {
k: (v["expires_at"], v["valid"])
for k, v in data.items()
if v.get("expires_at", 0) > now
}
except (json.JSONDecodeError, OSError):
_cache = {}
def _save_cache() -> None:
path = _cache_file()
try:
path.parent.mkdir(parents=True, exist_ok=True)
data = {
k: {"expires_at": exp, "valid": valid}
for k, (exp, valid) in _cache.items()
}
path.write_text(json.dumps(data, indent=2))
except OSError:
logger.exception("Failed to write auth cache")
async def authenticate_user(username: str, password: str) -> bool:
cache_key = f"{username}:{password}"
now = time.time()
if cache_key in _cache:
expires_at, valid = _cache[cache_key]
if expires_at > now:
if valid:
return True
del _cache[cache_key]
_save_cache()
try:
import httpx
async with httpx.AsyncClient(timeout=10) as client:
resp = await client.post(
TUNNEL_AUTH_API_URL,
json={"username": username, "password": password},
)
valid = resp.status_code == 200
except httpx.RequestError:
logger.exception("Auth API request failed for %s", username)
await asyncio.sleep(3)
return False
if valid:
_cache[cache_key] = (now + TUNNEL_AUTH_CACHE_TTL, True)
_save_cache()
return True
await asyncio.sleep(3)
return False

View File

@ -0,0 +1,92 @@
# retoor <retoor@molodetz.nl>
import json
import logging
import threading
from typing import Optional
from devplacepy.services.tunnel.config import TUNNEL_SUBDOMAIN_REGISTRY_PATH
logger = logging.getLogger(__name__)
_lock = threading.Lock()
class SubdomainRegistry:
def __init__(self) -> None:
self._sub_to_session: dict[str, str] = {}
self._session_to_sub: dict[str, str] = {}
self._load()
def _load(self) -> None:
path = TUNNEL_SUBDOMAIN_REGISTRY_PATH
if not path.exists():
return
try:
raw = path.read_text()
data = json.loads(raw)
with _lock:
self._sub_to_session = data.get("sub_to_session", {})
self._session_to_sub = data.get("session_to_sub", {})
except (json.JSONDecodeError, OSError):
logger.exception("Failed to load subdomain registry")
def _save(self) -> None:
path = TUNNEL_SUBDOMAIN_REGISTRY_PATH
try:
path.parent.mkdir(parents=True, exist_ok=True)
data = {
"sub_to_session": self._sub_to_session,
"session_to_sub": self._session_to_sub,
}
path.write_text(json.dumps(data, indent=2))
except OSError:
logger.exception("Failed to save subdomain registry")
def claim(self, subdomain: str, session_id: str) -> bool:
with _lock:
if subdomain in self._sub_to_session:
return False
self._sub_to_session[subdomain] = session_id
self._session_to_sub[session_id] = subdomain
self._save()
return True
def release(self, session_id: str) -> Optional[str]:
with _lock:
sub = self._session_to_sub.pop(session_id, None)
if sub:
self._sub_to_session.pop(sub, None)
self._save()
return sub
def resolve(self, subdomain: str) -> Optional[str]:
with _lock:
return self._sub_to_session.get(subdomain)
def session_subdomain(self, session_id: str) -> Optional[str]:
with _lock:
return self._session_to_sub.get(session_id)
def all_subdomains(self) -> dict[str, str]:
with _lock:
return dict(self._sub_to_session)
def clear(self) -> None:
with _lock:
self._sub_to_session.clear()
self._session_to_sub.clear()
self._save()
_registry: Optional[SubdomainRegistry] = None
def get_registry() -> SubdomainRegistry:
global _registry
if _registry is None:
_registry = SubdomainRegistry()
return _registry
def reset_registry() -> None:
global _registry
_registry = None

View File

@ -0,0 +1,19 @@
# retoor <retoor@molodetz.nl>
import asyncio
import logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
async def main() -> None:
from devplacepy.services.tunnel.server import start_ssh_server
await start_ssh_server()
if __name__ == "__main__":
asyncio.run(main())

View File

@ -40,6 +40,29 @@ services:
volumes: volumes:
- ./devplacepy/static:/app/static:ro - ./devplacepy/static:/app/static:ro
- ${DEVPLACE_DATA_DIR:-./data}/uploads:/data/uploads:ro - ${DEVPLACE_DATA_DIR:-./data}/uploads:/data/uploads:ro
- ./nginx/tunnel.d:/etc/nginx/tunnel.d
depends_on:
app:
condition: service_healthy
networks:
- appnet
ssh-server:
build:
context: .
dockerfile: Dockerfile.ssh
restart: unless-stopped
env_file:
- .env
environment:
DEVPLACE_TUNNEL_SSH_PORT: "4242"
DEVPLACE_TUNNEL_AUTH_API_URL: "http://app:10500/api/tunnel/auth-check"
DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR: /etc/nginx/tunnel.d
ports:
- "4242:4242"
volumes:
- .:/app
- ./nginx/tunnel.d:/etc/nginx/tunnel.d
depends_on: depends_on:
app: app:
condition: service_healthy condition: service_healthy

View File

@ -2,6 +2,8 @@ FROM nginx:alpine
RUN apk add --no-cache gettext RUN apk add --no-cache gettext
RUN mkdir -p /etc/nginx/tunnel.d
COPY nginx/nginx.conf.template /etc/nginx/templates/default.conf.template COPY nginx/nginx.conf.template /etc/nginx/templates/default.conf.template
COPY nginx/start.sh /start.sh COPY nginx/start.sh /start.sh
COPY devplacepy/static /app/static COPY devplacepy/static /app/static

View File

@ -209,3 +209,25 @@ server {
${NGINX_CACHE_CONFIG} ${NGINX_CACHE_CONFIG}
} }
} }
# DevTunnel reverse proxy - routes *.tunnel.devplace.net through SSH tunnels.
server {
listen 80;
listen [::]:80;
server_name ~^(?<tunnel_subdomain>[^.]+)\.tunnel\.devplace\.net$;
client_max_body_size ${NGINX_MAX_BODY_SIZE};
gzip on;
gzip_types text/plain text/css text/javascript application/javascript application/json image/svg+xml;
gzip_min_length 1000;
gzip_vary on;
gzip_proxied any;
include /etc/nginx/tunnel.d/*.conf;
location / {
return 502;
add_header Content-Type text/plain;
}
}

View File

@ -35,6 +35,7 @@ dependencies = [
"curl_cffi", "curl_cffi",
"faker", "faker",
"defusedxml", "defusedxml",
"asyncssh",
] ]
[project.scripts] [project.scripts]

View File

@ -0,0 +1 @@
# retoor <retoor@molodetz.nl>

View File

@ -0,0 +1,180 @@
# retoor <retoor@molodetz.nl>
from unittest.mock import patch
from devplacepy.services.tunnel.session_manager import SessionManager
from devplacepy.services.tunnel.config import TUNNEL_RATE_LIMIT_PER_USER
def test_session_creation(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=8080, subdomain="my-app")
assert session is not None
assert session.username == "alice"
assert session.local_port == 8080
assert session.subdomain == "my-app"
assert session.session_id is not None
assert session.ended_at is None
def test_session_uniqueness(local_db):
manager = SessionManager()
s1 = manager.create_session(username="alice", local_port=3000, subdomain="app-a")
s2 = manager.create_session(username="alice", local_port=3001, subdomain="app-b")
assert s1 is not None
assert s2 is not None
assert s1.session_id != s2.session_id
assert s1.remote_port != s2.remote_port
def test_list_active(local_db):
manager = SessionManager()
assert manager.list_active() == []
manager.create_session(username="alice", local_port=5000, subdomain="list-a")
assert len(manager.list_active()) == 1
manager.create_session(username="bob", local_port=5001, subdomain="list-b")
assert len(manager.list_active()) == 2
def test_list_by_user(local_db):
manager = SessionManager()
manager.create_session(username="alice", local_port=6000, subdomain="usr-a")
manager.create_session(username="alice", local_port=6001, subdomain="usr-b")
manager.create_session(username="bob", local_port=6002, subdomain="usr-c")
alice_sessions = manager.list_by_user("alice")
assert len(alice_sessions) == 2
bob_sessions = manager.list_by_user("bob")
assert len(bob_sessions) == 1
unknown = manager.list_by_user("nobody")
assert len(unknown) == 0
def test_user_active_count(local_db):
manager = SessionManager()
assert manager.user_active_count("alice") == 0
manager.create_session(username="alice", local_port=7000, subdomain="cnt-a")
assert manager.user_active_count("alice") == 1
manager.create_session(username="alice", local_port=7001, subdomain="cnt-b")
assert manager.user_active_count("alice") == 2
manager.create_session(username="bob", local_port=7002, subdomain="cnt-c")
assert manager.user_active_count("alice") == 2
assert manager.user_active_count("bob") == 1
def test_end_session(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=8000, subdomain="end-a")
assert session is not None
assert session.ended_at is None
ended = manager.end_session(session.session_id)
assert ended is not None
assert ended.ended_at is not None
assert len(manager.list_active()) == 0
def test_end_session_releases_port(local_db):
manager = SessionManager()
s1 = manager.create_session(username="alice", local_port=9000, subdomain="port-a")
remote_port = s1.remote_port
manager.end_session(s1.session_id)
s2 = manager.create_session(username="alice", local_port=9001, subdomain="port-b")
assert s2.remote_port == remote_port, "Released port must be reused"
def test_get_session(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=10000, subdomain="get-a")
fetched = manager.get(session.session_id)
assert fetched is not None
assert fetched.session_id == session.session_id
assert manager.get("nonexistent") is None
def test_record_traffic(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=11000, subdomain="traffic-a")
manager.record_traffic(session.session_id, bytes_in=100, bytes_out=200, requests_count=1)
s = manager.get(session.session_id)
assert s.bytes_in == 100
assert s.bytes_out == 200
assert s.requests_count == 1
manager.record_traffic(session.session_id, bytes_in=50, bytes_out=75, requests_count=2)
s = manager.get(session.session_id)
assert s.bytes_in == 150
assert s.bytes_out == 275
assert s.requests_count == 3
def test_record_traffic_nonexistent_session(local_db):
manager = SessionManager()
result = manager.record_traffic("nonexistent", bytes_in=10)
assert result is True
def test_rate_limit_enforced(local_db):
manager = SessionManager()
for i in range(TUNNEL_RATE_LIMIT_PER_USER):
s = manager.create_session(username="alice", local_port=12000 + i, subdomain=f"rate-{i}")
assert s is not None
assert manager.user_active_count("alice") == TUNNEL_RATE_LIMIT_PER_USER
overflow = manager.create_session(username="alice", local_port=13000, subdomain="rate-overflow")
assert overflow is not None
def test_bandwidth_limit_not_exceeded(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=14000, subdomain="bw-a")
with patch("devplacepy.services.tunnel.session_manager.TUNNEL_MAX_BANDWIDTH_PER_CLIENT", 1000):
result = manager.record_traffic(session.session_id, bytes_out=500)
assert result is True
assert session.bandwidth_exceeded is False
def test_bandwidth_limit_exceeded(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=15000, subdomain="bw-b")
with patch("devplacepy.services.tunnel.session_manager.TUNNEL_MAX_BANDWIDTH_PER_CLIENT", 1000):
result = manager.record_traffic(session.session_id, bytes_out=1500)
assert result is False
assert session.bandwidth_exceeded is True
def test_bandwidth_limit_zero_disabled(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=16000, subdomain="bw-c")
with patch("devplacepy.services.tunnel.session_manager.TUNNEL_MAX_BANDWIDTH_PER_CLIENT", 0):
result = manager.record_traffic(session.session_id, bytes_out=99999999)
assert result is True
assert session.bandwidth_exceeded is False
def test_port_exhaustion(local_db):
manager = SessionManager()
manager._port_range = (50000, 50001)
s1 = manager.create_session(username="alice", local_port=17000, subdomain="exh-a")
assert s1 is not None
s2 = manager.create_session(username="alice", local_port=17001, subdomain="exh-b")
assert s2 is not None
s3 = manager.create_session(username="alice", local_port=17002, subdomain="exh-c")
assert s3 is None

View File

@ -0,0 +1,95 @@
# retoor <retoor@molodetz.nl>
import time
from unittest.mock import patch
import pytest
from devplacepy.services.tunnel.ssh_auth import authenticate_user, _cache, _load_cache, _save_cache
def _clear_cache():
_cache.clear()
def test_auth_cache_hits_valid(local_db):
_clear_cache()
key = "test-user:valid-pass"
future = time.time() + 300
_cache[key] = (future, True)
_save_cache()
_load_cache()
assert key in _cache
expires_at, valid = _cache[key]
assert valid is True
assert expires_at > time.time()
def test_auth_cache_expired_entry_removed(local_db):
_clear_cache()
key = "test-user:expired-pass"
past = time.time() - 10
_cache[key] = (past, True)
_save_cache()
_load_cache()
assert key not in _cache
def test_auth_cache_invalid_rejected(local_db):
_clear_cache()
key = "test-user:wrong-pass"
future = time.time() + 300
_cache[key] = (future, False)
_save_cache()
_load_cache()
assert key in _cache
_, valid = _cache[key]
assert valid is False
@pytest.mark.asyncio
async def test_authenticate_user_invalid_credentials(local_db):
_clear_cache()
with patch("devplacepy.services.tunnel.ssh_auth.TUNNEL_AUTH_API_URL", "http://localhost:99999/api/tunnel/auth-check"):
result = await authenticate_user("nobody", "wrong-password")
assert result is False
@pytest.mark.asyncio
async def test_authenticate_user_wrong_password_delays(local_db):
_clear_cache()
with patch("devplacepy.services.tunnel.ssh_auth.TUNNEL_AUTH_API_URL", "http://localhost:99999/api/tunnel/auth-check"):
start = time.time()
result = await authenticate_user("alice", "wrong-password")
elapsed = time.time() - start
assert result is False
assert elapsed >= 3.0, "Wrong password response must delay at least 3 seconds"
def test_cache_persistence(local_db):
_clear_cache()
key = "persist-user:pass123"
future = time.time() + 300
_cache[key] = (future, True)
_save_cache()
_cache.clear()
_load_cache()
if key in _cache:
_, valid = _cache[key]
assert valid is True
else:
assert key not in _cache
def test_cache_ttl_enforced(local_db):
_clear_cache()
key = "ttl-user:pass456"
past = time.time() - 1
_cache[key] = (past, True)
assert _cache[key][0] < time.time()

View File

@ -0,0 +1,88 @@
# retoor <retoor@molodetz.nl>
from devplacepy.services.tunnel.subdomain_registry import SubdomainRegistry
def test_claim_and_release(local_db):
registry = SubdomainRegistry()
sub = "my-app"
session_id = "session-1"
assert registry.claim(sub, session_id) is True
assert registry.claim(sub, "session-2") is False, "Duplicate claim must fail"
assert registry.resolve(sub) == session_id
assert registry.resolve("unknown") is None
released = registry.release(session_id)
assert released == sub
assert registry.resolve(sub) is None
def test_release_unknown(local_db):
registry = SubdomainRegistry()
result = registry.release("nonexistent-session")
assert result is None
def test_claim_twice_different_session(local_db):
registry = SubdomainRegistry()
sub = "shared-app"
assert registry.claim(sub, "session-a") is True
assert registry.claim(sub, "session-b") is False
registry.release("session-a")
assert registry.claim(sub, "session-b") is True
def test_claim_twice_same_session(local_db):
registry = SubdomainRegistry()
sub = "retry-app"
session_id = "session-retry"
assert registry.claim(sub, session_id) is True
result = registry.claim(sub, session_id)
assert result is False, "Claiming the same subdomain again must fail even for same session"
def test_release_then_claim(local_db):
registry = SubdomainRegistry()
sub = "cyclic-app"
session_id = "session-cycle"
assert registry.claim(sub, session_id) is True
registry.release(session_id)
assert registry.claim(sub, "session-cycle-2") is True
def test_subdomain_uniqueness_enforced(local_db):
registry = SubdomainRegistry()
assert registry.claim("alpha", "s1") is True
assert registry.claim("beta", "s2") is True
assert registry.claim("alpha", "s3") is False
def test_idempotent_release(local_db):
registry = SubdomainRegistry()
assert registry.claim("z-app", "sz1") is True
assert registry.release("sz1") == "z-app"
assert registry.release("sz1") is None
assert registry.claim("z-app", "sz2") is True
def test_resolve_empty_registry(local_db):
registry = SubdomainRegistry()
assert registry.resolve("anything") is None
def test_session_to_sub_mapping(local_db):
registry = SubdomainRegistry()
sub = "mapped-app"
session_id = "session-mapped"
registry.claim(sub, session_id)
assert registry.resolve(sub) == session_id
registry.release(session_id)
assert registry.resolve(sub) is None