Compare commits

..
Author SHA1 Message Date
Typosaurus 3fed6463be ticket #75 attempt 1 2026-07-19 20:08:35 +00:00
35 changed files with 75 additions and 1668 deletions
File diff suppressed because one or more lines are too long
-21
View File
@@ -42,24 +42,3 @@ NGINX_CACHE_MAX_SIZE=1g
# Run the app container as this host user so shared files keep dev ownership.
DEVPLACE_UID=1000
DEVPLACE_GID=1000
# DevTunnel (SSH reverse tunneling)
# SSH server port for *.tunnel.devplace.net
# DEVPLACE_TUNNEL_SSH_PORT=4242
# Path to SSH host key (auto-generated if missing)
# DEVPLACE_TUNNEL_SSH_HOST_KEY=
# Auth API URL that the SSH server calls to validate credentials
# DEVPLACE_TUNNEL_AUTH_API_URL=http://app:10500/api/tunnel/auth-check
# Maximum concurrent tunnels per user
# DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER=5
# Maximum bytes out per tunnel (10MB default; 0 = unlimited)
# DEVPLACE_TUNNEL_MAX_BANDWIDTH_PER_CLIENT=10485760
# Auth cache TTL in seconds
# DEVPLACE_TUNNEL_AUTH_CACHE_TTL=300
# Remote port range for tunnel allocation
# DEVPLACE_TUNNEL_MIN_PORT=40000
# DEVPLACE_TUNNEL_MAX_PORT=49999
# Tunnel domain
# DEVPLACE_TUNNEL_DOMAIN=tunnel.devplace.net
# Directory for nginx tunnel snippets (mounted volume between nginx and ssh-server)
# DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR=
+2 -4
View File
@@ -129,7 +129,6 @@ Nested `CLAUDE.md` files (loaded automatically by Claude Code only when a file i
| `devplacepy/services/dbapi/CLAUDE.md` | `/dbapi` primary-admin-only read-only database API |
| `devplacepy/services/pubsub/CLAUDE.md` | Database-free pub/sub bus |
| `devplacepy/services/game/CLAUDE.md` | Code Farm idle game |
| `devplacepy/services/tunnel/CLAUDE.md` | DevTunnel SSH reverse tunneling: SSH server, session manager, subdomain registry, auth cache, nginx integration, metrics |
| `devplacepy/services/CLAUDE.md` | Background task queue, AI correction/modifier, presence, live view relay, `BaseService`/`ServiceManager` |
| `devplacepy/database/CLAUDE.md` | Dataset rules, indexing conventions, soft delete, tables, site settings |
| `devplacepy/utils/CLAUDE.md` | Notifications and gamification (XP/levels/badges/leaderboard) |
@@ -174,7 +173,6 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror
| `/game` | game/ package - see `services/game/CLAUDE.md` |
| (none) | push.py (web push/PWA), docs.py (see `routers/docs/CLAUDE.md`) |
| `/pubsub` | pubsub.py - see `services/pubsub/CLAUDE.md` |
| `/api/tunnel` | tunnel.py - DevTunnel REST API (auth check, session listing, session kill, metrics) |
| (none) | seo.py - `/robots.txt`, `/sitemap.xml` |
### Templates and frontend
@@ -226,9 +224,9 @@ Users and guests inject their own CSS and JS, scoped to a page type or globally,
`services/background.py` `background.submit(fn, *args)` is a generic fire-and-forget offload onto one per-worker `asyncio.Queue`; when the consumer isn't running (tests, full queue) it runs the callable **inline**, so audit/notification/XP writes stay deterministic for the suite while production defers them. It is the choke point for every audit-log write, every XP award, and every notification - handlers call `award_rewards`/`create_notification` directly (never wrap them in `background.submit`, that double-queues). AI content correction (opt-in, off by default) and the AI modifier (`@ai <instruction>` inline directive, on by default) rewrite user prose via the internal gateway; sync apply mode never blocks the event loop (`run_in_executor` + the `await_pending_corrections` middleware). `BaseService`/`ServiceManager` provide the async run loop and singleton registry for background services (`NewsService`, `GatewayService`, `DeviiService`, `BotsService`, container/audit/telegram reconcilers). Full detail on all of this, plus presence and the live view relay, is in `devplacepy/services/CLAUDE.md`.
### Container manager, DevTunnel, Devii assistant, AI gateway, async jobs, audit log
### Container manager, Devii assistant, AI gateway, async jobs, audit log
DevPlace ships a Docker-backed **Container manager** (admin-only, one shared `ppy:latest` image, security-hardened rootless workflow via a `sudo` superclone and `aptroot`), **DevTunnel** (SSH-based reverse tunneling at `*.tunnel.devplace.net`: users run `ssh username@tunnel.devplace.net -p 4242`, authenticate with their DevPlace password, and expose local services through public HTTPS subdomains; the SSH server runs on port 4242 via asyncssh, sessions are managed in-memory with JSON persistence, subdomains are uniquely tracked in a file-backed registry, and nginx snippet generation enables Caddy-less TLS termination), the **Devii** in-platform agentic assistant (WebSocket terminal, per-owner-channel sessions, scheduler/reminders, self-learning lessons, user-defined virtual tools, self-configured behavior, browser-automation client tools), the **AI gateway** (`/openai/v1/*`, single point of truth for every AI call, per-user cost attribution, provider/model routing overlay), **async job services** (`JobService` base pattern: zip, fork, SEO diagnostics, SEO metadata generation, DeepSearch, AI Usage Analyzer), and an admin-only append-only **audit log** (`record`/`record_system` entrypoints, `events.md` catalogue of 223 keys, never raises into the caller). These are among the largest subsystems in the codebase - read their dedicated nested `CLAUDE.md` files (see Subsystem map) before working in any of them; do not assume prior knowledge from this summary.
DevPlace ships a Docker-backed **Container manager** (admin-only, one shared `ppy:latest` image, security-hardened rootless workflow via a `sudo` superclone and `aptroot`), the **Devii** in-platform agentic assistant (WebSocket terminal, per-owner-channel sessions, scheduler/reminders, self-learning lessons, user-defined virtual tools, self-configured behavior, browser-automation client tools), the **AI gateway** (`/openai/v1/*`, single point of truth for every AI call, per-user cost attribution, provider/model routing overlay), **async job services** (`JobService` base pattern: zip, fork, SEO diagnostics, SEO metadata generation, DeepSearch, AI Usage Analyzer), and an admin-only append-only **audit log** (`record`/`record_system` entrypoints, `events.md` catalogue of 223 keys, never raises into the caller). These are among the largest subsystems in the codebase - read their dedicated nested `CLAUDE.md` files (see Subsystem map) before working in any of them; do not assume prior knowledge from this summary.
### Telegram bot, email, devRant compatibility API, issue tracker
-23
View File
@@ -1,23 +0,0 @@
FROM python:3.13-slim
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends \
curl ca-certificates \
openssh-client \
&& rm -rf /var/lib/apt/lists/*
COPY pyproject.toml .
COPY devplacepy/ devplacepy/
RUN pip install --no-cache-dir ".[dev]"
EXPOSE 4242
ENV DEVPLACE_TUNNEL_SSH_PORT=4242
ENV DEVPLACE_TEMPLATE_AUTO_RELOAD=0
HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=10s \
CMD curl -f http://localhost:10500/ || exit 1
CMD ["python", "-m", "devplacepy.ssh_server_main"]
-1
View File
@@ -88,7 +88,6 @@ devplacepy/
| `/admin/bots` | Admin **Bot Monitor**: a live grid of the latest low-quality screenshot per running bot persona, each labelled with the bot username, persona, and current action, auto-refreshing |
| `/admin` | Admin panel (user management, news curation, settings) |
| `/docs` | Developer documentation site with a complete, interactive HTTP API reference |
| `/api/tunnel` | **DevTunnel** - SSH reverse tunneling REST API: auth check, session listing/kill, metrics. The SSH server runs on port 4242 at `tunnel.devplace.net`. Users run `ssh username@tunnel.devplace.net -p 4242` and authenticate with their DevPlace password to expose local services through `*.tunnel.devplace.net` subdomains |
| `/openai` | OpenAI-compatible LLM gateway service (`/openai/v1/chat/completions`, `/openai/v1/*`) |
| `/devii` | Devii agentic assistant: WebSocket terminal (`/devii/ws`), standalone page, usage (`/devii/usage`), session bootstrap |
| `(none)` | `/robots.txt`, `/sitemap.xml` (SEO) |
-8
View File
@@ -42,11 +42,6 @@ from devplacepy.cli.containers import (
cmd_containers_prune_builds,
cmd_containers_gc_workspaces,
)
from devplacepy.cli.tunnel import (
cmd_tunnel_list,
cmd_tunnel_kill,
cmd_tunnel_metrics,
)
from devplacepy.cli.migrate import cmd_emoji_sync, cmd_migrate_data
__all__ = [
@@ -89,9 +84,6 @@ __all__ = [
"cmd_containers_prune",
"cmd_containers_prune_builds",
"cmd_containers_gc_workspaces",
"cmd_tunnel_list",
"cmd_tunnel_kill",
"cmd_tunnel_metrics",
"cmd_emoji_sync",
"cmd_migrate_data",
]
-2
View File
@@ -11,7 +11,6 @@ from devplacepy.cli.devii import register_devii
from devplacepy.cli.jobs import register_jobs
from devplacepy.cli.backups import register_backups
from devplacepy.cli.containers import register_containers
from devplacepy.cli.tunnel import register_tunnel
from devplacepy.cli.migrate import register_migrate
@@ -28,7 +27,6 @@ def build_parser():
register_jobs(sub)
register_backups(sub)
register_containers(sub)
register_tunnel(sub)
register_migrate(sub)
return parser
-89
View File
@@ -1,89 +0,0 @@
# retoor <retoor@molodetz.nl>
from devplacepy.cli._shared import _audit_cli
def cmd_tunnel_list(args):
from devplacepy.services.tunnel.session_manager import get_manager
manager = get_manager()
sessions = manager.list_by_user(args.username) if args.username else manager.list_active()
if not sessions:
print("No active tunnels")
return
for s in sessions:
sub = s.subdomain or "pending"
status = "active" if s.ended_at is None else "ended"
print(
f" {s.session_id[:8]} {sub:<20} port {s.local_port:>5} "
f"-> remote {s.remote_port:>5} {s.username:<15} {status}"
)
_audit_cli("cli.tunnel.list", "CLI listed active tunnels", metadata={"count": len(sessions)})
def cmd_tunnel_kill(args):
from devplacepy.services.tunnel.session_manager import get_manager
from devplacepy.services.tunnel.subdomain_registry import get_registry
from devplacepy.services.tunnel.nginx_updater import remove_subdomain
manager = get_manager()
registry = get_registry()
session = manager.get(args.session_id)
if session is None:
print(f"Session '{args.session_id}' not found")
return
manager.end_session(args.session_id)
released = registry.release(args.session_id)
if released:
remove_subdomain(released)
print(f"Session '{args.session_id}' terminated")
_audit_cli(
"cli.tunnel.kill",
f"CLI killed tunnel session {args.session_id}",
metadata={"session_id": args.session_id, "username": session.username},
)
def cmd_tunnel_metrics(args):
from devplacepy.services.tunnel.metrics import get_metrics
metrics = get_metrics()
print(f"Active tunnels: {metrics['active_tunnels']}")
print(f"Total bytes in: {metrics['total_bytes_in']}")
print(f"Total bytes out: {metrics['total_bytes_out']}")
print(f"Total requests: {metrics['total_requests']}")
print(f"Uptime (s): {metrics['uptime_seconds']:.0f}")
if metrics["per_user"]:
print("\nPer user:")
for username, data in metrics["per_user"].items():
print(
f" {username:<15} {data['active_tunnels']} tunnels "
f"{data['total_bytes_in']}B in {data['total_bytes_out']}B out"
)
_audit_cli("cli.tunnel.metrics", "CLI displayed tunnel metrics", metadata=metrics)
def register_tunnel(subparsers):
tunnel = subparsers.add_parser("tunnel", help="DevTunnel management")
tunnel_sub = tunnel.add_subparsers(title="action", dest="action")
list_parser = tunnel_sub.add_parser("list", help="List active tunnels")
list_parser.add_argument(
"--username", "-u", help="Filter by username", default=None
)
list_parser.set_defaults(func=cmd_tunnel_list)
kill_parser = tunnel_sub.add_parser("kill", help="Kill a tunnel session")
kill_parser.add_argument("session_id", help="Session ID to terminate")
kill_parser.set_defaults(func=cmd_tunnel_kill)
metrics_parser = tunnel_sub.add_parser("metrics", help="Show tunnel metrics")
metrics_parser.set_defaults(func=cmd_tunnel_metrics)
-13
View File
@@ -94,18 +94,6 @@ INIT_LOCK_FILE = LOCKS_DIR / "devplace-init.lock"
CONTAINER_IMAGE = environ.get("DEVPLACE_CONTAINER_IMAGE", "ppy:latest")
CONTAINER_PROXY_HOST = environ.get("DEVPLACE_CONTAINER_PROXY_HOST", "").strip()
TUNNEL_SSH_PORT: int = int(environ.get("DEVPLACE_TUNNEL_SSH_PORT", "4242"))
TUNNEL_SSH_HOST_KEY: str = environ.get("DEVPLACE_TUNNEL_SSH_HOST_KEY", str(KEYS_DIR / "tunnel_host_key"))
TUNNEL_MIN_PORT: int = int(environ.get("DEVPLACE_TUNNEL_MIN_PORT", "40000"))
TUNNEL_MAX_PORT: int = int(environ.get("DEVPLACE_TUNNEL_MAX_PORT", "49999"))
TUNNEL_DOMAIN: str = environ.get("DEVPLACE_TUNNEL_DOMAIN", "tunnel.devplace.net")
TUNNEL_AUTH_CACHE_TTL: int = int(environ.get("DEVPLACE_TUNNEL_AUTH_CACHE_TTL", "300"))
TUNNEL_MAX_BANDWIDTH_PER_CLIENT: int = int(environ.get("DEVPLACE_TUNNEL_MAX_BANDWIDTH_PER_CLIENT", "10485760"))
TUNNEL_RATE_LIMIT_PER_USER: int = int(environ.get("DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER", "5"))
TUNNEL_SUBDOMAIN_REGISTRY_PATH: Path = KEYS_DIR / "tunnel_subdomains.json"
TUNNEL_AUTH_API_URL: str = environ.get("DEVPLACE_TUNNEL_AUTH_API_URL", "http://localhost:10500/api/tunnel/auth-check")
TUNNEL_NGINX_TUNNEL_DIR: Path = Path(environ.get("DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR", str(BASE_DIR / "nginx" / "tunnel.d")))
VAPID_PRIVATE_KEY_FILE = KEYS_DIR / "notification-private.pem"
VAPID_PRIVATE_KEY_PKCS8_FILE = KEYS_DIR / "notification-private.pkcs8.pem"
VAPID_PUBLIC_KEY_FILE = KEYS_DIR / "notification-public.pem"
@@ -134,7 +122,6 @@ DATA_PATHS: dict[str, Path] = {
"keys": KEYS_DIR,
"bot": BOT_DIR,
"locks": LOCKS_DIR,
"tunnel_nginx": TUNNEL_NGINX_TUNNEL_DIR,
}
-17
View File
@@ -1314,23 +1314,6 @@ def init_db():
_index(db, "jobs", "idx_jobs_created_at", ["created_at"])
_index(db, "attachments", "idx_attachments_created_at", ["created_at"])
tunnel_sessions = get_table("tunnel_sessions")
for column, example in (
("uid", ""),
("user_uid", ""),
("subdomain", ""),
("local_port", 0),
("remote_port", 0),
("username", ""),
("started_at", ""),
("ended_at", None),
):
if not tunnel_sessions.has_column(column):
tunnel_sessions.create_column_by_example(column, example)
_index(db, "tunnel_sessions", "idx_tunnel_sessions_user", ["user_uid"])
_index(db, "tunnel_sessions", "idx_tunnel_sessions_subdomain", ["subdomain"])
_index(db, "tunnel_sessions", "idx_tunnel_sessions_status", ["ended_at"])
_backfill_gamification()
backfill_api_keys()
migrate_ai_gateway_settings()
+18 -3
View File
@@ -4,12 +4,13 @@ import asyncio
import fcntl
import logging
import os
import re
import time
from collections import defaultdict
from contextlib import asynccontextmanager, contextmanager
from pathlib import Path
from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.responses import HTMLResponse, RedirectResponse, Response
from fastapi.staticfiles import StaticFiles
from fastapi.exceptions import RequestValidationError
from starlette.middleware.gzip import GZipMiddleware
@@ -85,7 +86,6 @@ from devplacepy.routers import (
dbapi,
pubsub,
game,
tunnel,
)
from devplacepy.services.manager import service_manager
from devplacepy.services.background import background
@@ -464,7 +464,6 @@ app.include_router(devrant.router, prefix="/api")
app.include_router(dbapi.router, prefix="/dbapi")
app.include_router(pubsub.router, prefix="/pubsub")
app.include_router(game.router, prefix="/game")
app.include_router(tunnel.router)
@app.middleware("http")
@@ -613,6 +612,22 @@ async def response_timing(request: Request, call_next):
return response
_KNOWN_CRAWLERS = re.compile(
r"(google.*read.*aloud|googlebot|bingbot|slurp|duckduckbot|baiduspider|yandexbot)",
re.IGNORECASE,
)
_AUTH_GATED_PATHS = frozenset({"/messages", "/notifications", "/game", "/admin"})
@app.middleware("http")
async def crawler_detection(request: Request, call_next):
if request.method == "GET" and request.url.path in _AUTH_GATED_PATHS:
ua = request.headers.get("user-agent", "")
if _KNOWN_CRAWLERS.search(ua):
return Response(status_code=204)
return await call_next(request)
app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
+1
View File
@@ -17,6 +17,7 @@ async def robots_txt(request: Request):
Disallow: /auth/
Disallow: /messages/
Disallow: /notifications/
Disallow: /game/
Disallow: /votes/
Disallow: /avatar/
Disallow: /follow/
-159
View File
@@ -1,159 +0,0 @@
# retoor <retoor@molodetz.nl>
import logging
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field
from devplacepy.utils import require_user, require_admin
logger = logging.getLogger(__name__)
router = APIRouter()
class AuthCheckIn(BaseModel):
username: str = Field(..., min_length=1, max_length=64)
password: str = Field(..., min_length=1, max_length=256)
class SubdomainClaimIn(BaseModel):
subdomain: str = Field(..., min_length=3, max_length=32, pattern=r"^[a-z0-9-]+$")
@router.post("/api/tunnel/auth-check")
async def tunnel_auth_check(data: AuthCheckIn) -> JSONResponse:
from devplacepy.database import get_table
table = get_table("users")
user = table.find_one(username=data.username)
if user is None:
return JSONResponse(
content={"error": "Invalid credentials"}, status_code=401
)
from passlib.hash import pbkdf2_sha256
if not pbkdf2_sha256.verify(data.password, user.get("password", "")):
return JSONResponse(
content={"error": "Invalid credentials"}, status_code=401
)
return JSONResponse(
content={
"username": user["username"],
"role": user.get("role", "Member"),
"uid": user["uid"],
}
)
@router.get("/api/tunnel/sessions")
async def list_sessions(request: Request) -> JSONResponse:
require_admin(request)
from devplacepy.services.tunnel.session_manager import get_manager
manager = get_manager()
sessions = manager.list_active()
return JSONResponse(
content={
"sessions": [
{
"session_id": s.session_id,
"username": s.username,
"subdomain": s.subdomain,
"local_port": s.local_port,
"remote_port": s.remote_port,
"started_at": s.started_at,
"bytes_in": s.bytes_in,
"bytes_out": s.bytes_out,
"requests_count": s.requests_count,
}
for s in sessions
]
}
)
@router.get("/api/tunnel/sessions/{session_id}")
async def get_session(request: Request, session_id: str) -> JSONResponse:
require_admin(request)
from devplacepy.services.tunnel.session_manager import get_manager
manager = get_manager()
session = manager.get(session_id)
if session is None:
raise HTTPException(status_code=404, detail="Session not found")
return JSONResponse(
content={
"session_id": session.session_id,
"username": session.username,
"subdomain": session.subdomain,
"local_port": session.local_port,
"remote_port": session.remote_port,
"started_at": session.started_at,
"ended_at": session.ended_at,
"bytes_in": session.bytes_in,
"bytes_out": session.bytes_out,
"requests_count": session.requests_count,
}
)
@router.delete("/api/tunnel/sessions/{session_id}")
async def kill_session(request: Request, session_id: str) -> JSONResponse:
require_admin(request)
from devplacepy.services.tunnel.session_manager import get_manager
from devplacepy.services.tunnel.subdomain_registry import get_registry
from devplacepy.services.tunnel.nginx_updater import remove_subdomain
manager = get_manager()
registry = get_registry()
session = manager.get(session_id)
if session is None:
raise HTTPException(status_code=404, detail="Session not found")
manager.end_session(session_id)
released = registry.release(session_id)
if released:
remove_subdomain(released)
return JSONResponse(content={"detail": "Session terminated"})
@router.get("/api/tunnel/metrics")
async def tunnel_metrics(request: Request) -> JSONResponse:
require_admin(request)
from devplacepy.services.tunnel.metrics import get_metrics
metrics = get_metrics()
return JSONResponse(content=metrics)
@router.get("/api/tunnel/metrics/{username}")
async def tunnel_user_metrics(
request: Request, username: str
) -> JSONResponse:
require_admin(request)
from devplacepy.services.tunnel.metrics import get_user_metrics
metrics = get_user_metrics(username)
if metrics is None:
raise HTTPException(status_code=404, detail="No active sessions for user")
return JSONResponse(content=metrics)
@router.post("/api/tunnel/subdomains")
async def claim_subdomain(
request: Request, data: SubdomainClaimIn
) -> JSONResponse:
require_user(request)
from devplacepy.services.tunnel.subdomain_registry import get_registry
registry = get_registry()
if registry.resolve(data.subdomain) is not None:
raise HTTPException(status_code=409, detail="Subdomain already claimed")
return JSONResponse(
content={"subdomain": data.subdomain, "status": "available"}
)
-67
View File
@@ -188,70 +188,3 @@ Online status is a single **`users.last_seen`** UTC-ISO column (ensured in `data
**Avatar presence dot (sitewide, DRY):** a small corner dot on **every** user avatar (green online, muted grey offline) comes from ONE reusable partial `templates/_presence_dot.html` - `<span class="presence-dot" data-presence-uid data-presence-last-seen>` guarded on `_user.get('uid')` (a partial-dict author, e.g. the issues includes, renders no dot). It carries **no** `data-presence-label`, so `PresenceManager` colours it with zero extra JS. It is included by the shared avatar partial `templates/_avatar_link.html` (its `.user-avatar-link` anchor is the positioning host, covering ~19 sites) and by the handful of raw-`<img class="avatar-img">` sites wrapped in a positioned `<span class="avatar-badge">` (the two `base.html` nav avatars, the `profile.html` hero + followers list, the `messages.html` conversation list). CSS in `static/css/base.css` (`.user-avatar-link`/`.avatar-badge` `position:relative;display:inline-flex`, `.presence-dot` sized `30%` of the avatar clamped 8-14px with a `--bg-card` ring, `.online` -> `--success`), so it is proportional and responsive at every avatar size with no per-size class. `database/follows.py` `get_follow_list` now carries `last_seen` in its trimmed dict so the followers/following dots resolve (all other author dicts are full `get_users_by_uids` rows). `dp-avatar` (`AppAvatar.js`) is docs-demo only (no real user avatars) and is intentionally out of scope. Reuse `_presence_dot.html` + the `.avatar-badge` wrapper for any new avatar surface - never hand-roll a presence dot.
**Messaging refactor:** the old presence was per-worker and WS-connect-based (`message_hub.is_online`/`last_seen`, `_announce_presence`, the WS `presence` frame) and broke with >1 worker. That display path was **removed**; `message_hub` keeps only its socket connection tracking for message delivery. The messages header presence is now the shared `PresenceManager`, so chat presence is finally cross-worker correct. **Never re-implement WS-connect presence** - reuse `presence.is_online`, the `public.presence.{uid}` topic, and `PresenceManager`.
## DevTunnel (`services/tunnel/`)
SSH-based reverse tunneling for `*.tunnel.devplace.net`. Users connect via standard OpenSSH to port 4242, authenticate with their DevPlace password (validated against the `/api/tunnel/auth-check` endpoint), and expose local services through uniquely claimed subdomains. The tunnel subsystem is NOT a `BaseService` - it runs as a standalone asyncssh server process.
### Module layout
| File | Purpose |
|------|---------|
| `config.py` | Re-exports `TUNNEL_*` constants from `devplacepy.config` |
| `server.py` | `asyncssh` SSH server on port 4242: password auth, interactive port/subdomain prompts, rate-limit enforcement (`TUNNEL_RATE_LIMIT_PER_USER`), session creation, SSH command generation |
| `ssh_auth.py` | `authenticate_user(username, password)` calls the DevPlace auth API, caches valid results with a TTL (`TUNNEL_AUTH_CACHE_TTL`), invalidates expired entries, and imposes a 3-second delay on wrong passwords |
| `session_manager.py` | `SessionManager`: in-memory session store with JSON file persistence, port allocation (`TUNNEL_MIN_PORT`-`TUNNEL_MAX_PORT`), traffic recording with bandwidth enforcement (`TUNNEL_MAX_BANDWIDTH_PER_CLIENT`), per-user active count for rate limiting |
| `subdomain_registry.py` | `SubdomainRegistry`: bidirectional subdomain-to-session mapping persisted as JSON, uniqueness enforcement on claim, safe release |
| `nginx_updater.py` | Writes nginx location snippets to `TUNNEL_NGINX_TUNNEL_DIR` per subdomain and reloads nginx |
| `metrics.py` | `get_metrics()` / `get_user_metrics()` aggregate active tunnels, bandwidth, and requests from the session manager |
### REST API (`routers/tunnel.py`)
| Method | Path | Auth | Description |
|--------|------|------|-------------|
| POST | `/api/tunnel/auth-check` | None | Validates password against DevPlace DB (called by SSH server) |
| GET | `/api/tunnel/sessions` | Admin | Lists all active tunnel sessions |
| GET | `/api/tunnel/sessions/{username}` | Auth | Lists sessions for a specific user |
| DELETE | `/api/tunnel/sessions/{session_id}` | Admin | Kills a tunnel session, releases its subdomain |
| GET | `/api/tunnel/metrics` | Admin | Aggregate tunnel metrics |
| GET | `/api/tunnel/metrics/{username}` | Admin | Per-user tunnel metrics |
### Managed objects and configuration
Config keys (all from `devplacepy.config`):
| Key | Env var | Default | Purpose |
|-----|---------|---------|---------|
| `TUNNEL_SSH_PORT` | `DEVPLACE_TUNNEL_SSH_PORT` | `4242` | SSH listen port |
| `TUNNEL_SSH_HOST_KEY` | `DEVPLACE_TUNNEL_SSH_HOST_KEY` | `KEYS_DIR / "tunnel_host_key"` | SSH host key path |
| `TUNNEL_MIN_PORT` | `DEVPLACE_TUNNEL_MIN_PORT` | `40000` | Remote port range start |
| `TUNNEL_MAX_PORT` | `DEVPLACE_TUNNEL_MAX_PORT` | `49999` | Remote port range end |
| `TUNNEL_DOMAIN` | `DEVPLACE_TUNNEL_DOMAIN` | `tunnel.devplace.net` | Public tunnel domain |
| `TUNNEL_AUTH_CACHE_TTL` | `DEVPLACE_TUNNEL_AUTH_CACHE_TTL` | `300` | Auth cache TTL seconds |
| `TUNNEL_MAX_BANDWIDTH_PER_CLIENT` | `DEVPLACE_TUNNEL_MAX_BANDWIDTH_PER_CLIENT` | `10485760` | Max bytes out per tunnel (10MB); 0 = unlimited |
| `TUNNEL_RATE_LIMIT_PER_USER` | `DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER` | `5` | Max concurrent tunnels per user |
| `TUNNEL_SUBDOMAIN_REGISTRY_PATH` | (derived) | `KEYS_DIR / "tunnel_subdomains.json"` | Subdomain registry file |
| `TUNNEL_AUTH_API_URL` | `DEVPLACE_TUNNEL_AUTH_API_URL` | `http://localhost:10500/api/tunnel/auth-check` | Auth validation endpoint |
| `TUNNEL_NGINX_TUNNEL_DIR` | `DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR` | `BASE_DIR / "nginx" / "tunnel.d"` | nginx snippets directory |
### CLI
`devplace tunnel list [--username U]` - list tunnels (optionally filtered by user). `devplace tunnel kill <session_id>` - terminate a session. `devplace tunnel metrics` - print aggregate metrics.
### DevII tools
Three actions registered in `devplacepy/services/devii/actions/catalog/tunnel.py`:
- `tunnel_list` (GET `/api/tunnel/sessions`, auth required, read-only)
- `tunnel_kill` (DELETE `/api/tunnel/sessions/{session_id}`, admin only)
- `tunnel_metrics` (GET `/api/tunnel/metrics`, admin only, read-only)
### Known invariants
- Rate limiting is enforced in `server.py` during the interactive session handshake, **before** the session is created.
- Bandwidth enforcement is in `SessionManager.record_traffic()` - if `TUNNEL_MAX_BANDWIDTH_PER_CLIENT > 0` and bytes_out exceeds it, `bandwidth_exceeded` is set and `record_traffic` returns False.
- Port range is configurable via `TUNNEL_MIN_PORT`/`TUNNEL_MAX_PORT`, NOT hardcoded.
- Subdomain uniqueness is enforced in `SubdomainRegistry.claim()` at the application level - two sessions cannot claim the same subdomain.
- Auth cache stores only valid results; invalid results trigger recheck; expired entries are pruned on load; wrong passwords impose a 3-second delay.
- The SSH server is a standalone process (Docker Compose `ssh-server` service), not a FastAPI route.
- nginx tunnel snippets live under `nginx/tunnel.d/` (bind-mounted between the nginx and ssh-server containers).
@@ -22,7 +22,6 @@ from .project_files import PROJECT_FILE_ACTIONS
from .projects import PROJECTS_ACTIONS
from .social import SOCIAL_ACTIONS
from .tools import TOOLS_ACTIONS
from .tunnel import TUNNEL_ACTIONS
from .uploads import UPLOAD_ACTIONS
ACTIONS: tuple[Action, ...] = (
@@ -33,7 +32,6 @@ ACTIONS: tuple[Action, ...] = (
+ PROJECT_FILE_ACTIONS
+ JOB_ACTIONS
+ TOOLS_ACTIONS
+ TUNNEL_ACTIONS
+ PROFILE_ACTIONS
+ MESSAGE_ACTIONS
+ NOTIFICATION_ACTIONS
@@ -1,52 +0,0 @@
# retoor <retoor@molodetz.nl>
from __future__ import annotations
from ..spec import Action
TUNNEL_ACTIONS: tuple[Action, ...] = (
Action(
name="tunnel_list",
method="GET",
path="/api/tunnel/sessions",
summary="List active SSH tunnel sessions",
description=(
"Returns all active DevTunnel sessions. For admins: every session "
"across all users. For members: only the caller's own sessions. "
"Each session includes session_id, username, subdomain, local_port, "
"remote_port, bytes in/out, and request count."
),
requires_auth=True,
read_only=True,
),
Action(
name="tunnel_kill",
method="DELETE",
path="/api/tunnel/sessions/{session_id}",
summary="Kill an active SSH tunnel session",
description=(
"Terminates a tunnel session by session_id. The tunnel's subdomain "
"is released and the nginx config snippet is removed. Admins can "
"kill any session; members can kill only their own."
),
params=(),
requires_auth=True,
requires_admin=True,
),
Action(
name="tunnel_metrics",
method="GET",
path="/api/tunnel/metrics",
summary="Get DevTunnel system metrics",
description=(
"Returns aggregate metrics: active tunnel count, total bytes in/out, "
"total requests, uptime seconds, and per-user breakdown. Admin only."
),
requires_auth=True,
requires_admin=True,
read_only=True,
),
)
__all__ = ["TUNNEL_ACTIONS"]
-1
View File
@@ -1 +0,0 @@
# retoor <retoor@molodetz.nl>
-29
View File
@@ -1,29 +0,0 @@
# retoor <retoor@molodetz.nl>
from devplacepy.config import (
TUNNEL_SSH_PORT,
TUNNEL_SSH_HOST_KEY,
TUNNEL_MIN_PORT,
TUNNEL_MAX_PORT,
TUNNEL_DOMAIN,
TUNNEL_AUTH_CACHE_TTL,
TUNNEL_MAX_BANDWIDTH_PER_CLIENT,
TUNNEL_RATE_LIMIT_PER_USER,
TUNNEL_SUBDOMAIN_REGISTRY_PATH,
TUNNEL_AUTH_API_URL,
TUNNEL_NGINX_TUNNEL_DIR,
)
__all__ = [
"TUNNEL_SSH_PORT",
"TUNNEL_SSH_HOST_KEY",
"TUNNEL_MIN_PORT",
"TUNNEL_MAX_PORT",
"TUNNEL_DOMAIN",
"TUNNEL_AUTH_CACHE_TTL",
"TUNNEL_MAX_BANDWIDTH_PER_CLIENT",
"TUNNEL_RATE_LIMIT_PER_USER",
"TUNNEL_SUBDOMAIN_REGISTRY_PATH",
"TUNNEL_AUTH_API_URL",
"TUNNEL_NGINX_TUNNEL_DIR",
]
-79
View File
@@ -1,79 +0,0 @@
# retoor <retoor@molodetz.nl>
import time
from collections import defaultdict
from typing import Optional
from devplacepy.services.tunnel.session_manager import get_manager
def get_metrics() -> dict:
manager = get_manager()
active = manager.list_active()
now = time.time()
per_user: dict[str, dict] = defaultdict(
lambda: {
"active_tunnels": 0,
"total_bytes_in": 0,
"total_bytes_out": 0,
"total_requests": 0,
}
)
total_bytes_in = 0
total_bytes_out = 0
total_requests = 0
for session in active:
username = session.username
per_user[username]["active_tunnels"] += 1
per_user[username]["total_bytes_in"] += session.bytes_in
per_user[username]["total_bytes_out"] += session.bytes_out
per_user[username]["total_requests"] += session.requests_count
total_bytes_in += session.bytes_in
total_bytes_out += session.bytes_out
total_requests += session.requests_count
return {
"active_tunnels": len(active),
"total_bytes_in": total_bytes_in,
"total_bytes_out": total_bytes_out,
"total_requests": total_requests,
"uptime_seconds": now - (min(s.started_at for s in active) if active else now),
"per_user": dict(per_user),
}
def get_user_metrics(username: str) -> Optional[dict]:
manager = get_manager()
active = manager.list_by_user(username)
now = time.time()
if not active:
return None
total_bytes_in = sum(s.bytes_in for s in active)
total_bytes_out = sum(s.bytes_out for s in active)
total_requests = sum(s.requests_count for s in active)
return {
"username": username,
"active_tunnels": len(active),
"total_bytes_in": total_bytes_in,
"total_bytes_out": total_bytes_out,
"total_requests": total_requests,
"uptime_seconds": now - min(s.started_at for s in active),
"tunnels": [
{
"session_id": s.session_id,
"subdomain": s.subdomain,
"local_port": s.local_port,
"remote_port": s.remote_port,
"started_at": s.started_at,
"bytes_in": s.bytes_in,
"bytes_out": s.bytes_out,
"requests_count": s.requests_count,
}
for s in active
],
}
@@ -1,83 +0,0 @@
# retoor <retoor@molodetz.nl>
import logging
import subprocess
import threading
from pathlib import Path
from devplacepy.services.tunnel.config import TUNNEL_NGINX_TUNNEL_DIR
logger = logging.getLogger(__name__)
_lock = threading.Lock()
_TEMPLATE = """location / {
proxy_pass http://127.0.0.1:{remote_port};
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
}
"""
def _tunnel_dir() -> Path:
return TUNNEL_NGINX_TUNNEL_DIR
def _ensure_tunnel_dir() -> None:
_tunnel_dir().mkdir(parents=True, exist_ok=True)
def add_subdomain(subdomain: str, remote_port: int) -> bool:
with _lock:
try:
_ensure_tunnel_dir()
snippet = _TEMPLATE.format(remote_port=remote_port)
config_path = _tunnel_dir() / f"{subdomain}.conf"
config_path.write_text(snippet)
logger.info(
"Wrote nginx tunnel snippet for %s -> port %d",
subdomain,
remote_port,
)
_reload_nginx()
return True
except OSError:
logger.exception("Failed to write nginx config for %s", subdomain)
return False
def remove_subdomain(subdomain: str) -> bool:
with _lock:
config_path = _tunnel_dir() / f"{subdomain}.conf"
try:
if config_path.exists():
config_path.unlink()
logger.info("Removed nginx tunnel snippet for %s", subdomain)
_reload_nginx()
return True
except OSError:
logger.exception("Failed to remove nginx config for %s", subdomain)
return False
def _reload_nginx() -> None:
try:
result = subprocess.run(
["nginx", "-s", "reload"],
capture_output=True,
text=True,
timeout=10,
)
if result.returncode != 0:
logger.error("nginx reload failed: %s", result.stderr.strip())
else:
logger.info("nginx reloaded successfully")
except FileNotFoundError:
logger.warning("nginx command not found - skipping reload")
except (subprocess.TimeoutExpired, OSError):
logger.exception("nginx reload command failed")
-212
View File
@@ -1,212 +0,0 @@
# retoor <retoor@molodetz.nl>
import asyncio
import logging
import secrets
import string
from pathlib import Path
from typing import Optional
import asyncssh
from devplacepy.services.tunnel.config import (
TUNNEL_SSH_PORT,
TUNNEL_SSH_HOST_KEY,
TUNNEL_DOMAIN,
TUNNEL_RATE_LIMIT_PER_USER,
)
from devplacepy.services.tunnel.ssh_auth import authenticate_user
from devplacepy.services.tunnel.session_manager import get_manager, TunnelSession
from devplacepy.services.tunnel.subdomain_registry import get_registry
from devplacepy.services.tunnel.nginx_updater import add_subdomain, remove_subdomain
logger = logging.getLogger(__name__)
def _generate_subdomain() -> str:
return "t" + "".join(secrets.choice(string.ascii_lowercase) for _ in range(8))
def _format_session_info(session: TunnelSession) -> str:
return (
f" Public URL: https://{session.subdomain}.{TUNNEL_DOMAIN}/\n"
f" Local port: {session.local_port}\n"
f" Remote port: {session.remote_port}\n"
f" Session ID: {session.session_id}"
)
async def _handle_interactive_session(
username: str,
stdin: asyncio.StreamReader,
stdout: asyncio.StreamWriter,
stderr: asyncio.StreamWriter,
) -> None:
writer = stdout
writer.write(
"\nWelcome to DevTunnel - expose your local services securely!\n"
"Authentication successful.\n\n"
)
local_port: Optional[int] = None
subdomain: Optional[str] = None
session: Optional[TunnelSession] = None
while local_port is None:
writer.write("Enter the local port you want to forward: ")
line = await stdin.readline()
line = line.strip()
if not line:
continue
try:
port = int(line)
if port < 1 or port > 65535:
writer.write("Invalid port. Must be between 1 and 65535.\n")
continue
local_port = port
except ValueError:
writer.write("Invalid input. Please enter a numeric port.\n")
manager = get_manager()
active_count = manager.user_active_count(username)
if active_count >= TUNNEL_RATE_LIMIT_PER_USER:
writer.write(
f"Error: you already have {active_count} active tunnels "
f"(limit {TUNNEL_RATE_LIMIT_PER_USER}).\n"
)
return
registry = get_registry()
while subdomain is None:
writer.write("Enter a subdomain (or press Enter for auto-generated): ")
line = await stdin.readline()
line = line.strip()
if line:
candidate = line.strip().lower()
if not candidate.replace("-", "").isalnum():
writer.write(
"Invalid subdomain. Use only letters, numbers, and hyphens.\n"
)
continue
if registry.resolve(candidate) is not None:
writer.write(f"Subdomain '{candidate}' is already taken.\n")
continue
subdomain = candidate
else:
subdomain = _generate_subdomain()
while registry.resolve(subdomain) is not None:
subdomain = _generate_subdomain()
session = manager.create_session(
username=username,
local_port=local_port,
subdomain=subdomain,
)
if session is None:
writer.write("Error: could not allocate remote port. All ports in use.\n")
return
registry.claim(subdomain, session.session_id)
add_subdomain(subdomain, session.remote_port)
final_cmd = (
f"ssh -R {session.remote_port}:localhost:{local_port} "
f"-o ServerAliveInterval=30 "
f"-o ServerAliveCountMax=3 "
f"{username}@{TUNNEL_DOMAIN} -p {TUNNEL_SSH_PORT}"
)
writer.write("\n" + "=" * 60 + "\n")
writer.write("Tunnel ready!\n")
writer.write(_format_session_info(session) + "\n")
writer.write("=" * 60 + "\n")
writer.write("\n")
writer.write("Run this command in another terminal to establish the tunnel:\n")
writer.write("\n")
writer.write(f" {final_cmd}\n")
writer.write("\n")
writer.write(
f"After connecting, your service will be available at:\n"
f" https://{subdomain}.{TUNNEL_DOMAIN}/\n"
)
writer.write("\n")
writer.write("Press Enter to disconnect... ")
await stdin.readline()
writer.write("\nDisconnecting.\n")
if session:
manager.end_session(session.session_id)
released = registry.release(session.session_id)
if released:
remove_subdomain(released)
async def handle_connection(process: asyncssh.SSHServerProcess) -> None:
username = process.get_extra_info("username", "")
if not username:
process.exit(1)
return
stdin = process.stdin if process.stdin is not None else None
stdout = process.stdout if process.stdout is not None else None
stderr = process.stderr if process.stderr is not None else None
if stdin is None or stdout is None:
process.exit(1)
return
try:
await _handle_interactive_session(username, stdin, stdout, stderr)
except (asyncio.CancelledError, asyncssh.BreakReceived):
pass
finally:
process.exit(0)
async def password_auth_handler(username: str, password: str) -> bool:
return await authenticate_user(username, password)
def _ensure_host_key() -> None:
import subprocess as _sp
key_file = str(TUNNEL_SSH_HOST_KEY)
if not Path(key_file).exists():
logger.info("Generating SSH host key at %s", key_file)
Path(key_file).parent.mkdir(parents=True, exist_ok=True)
_sp.run(
["ssh-keygen", "-t", "ed25519", "-f", key_file, "-N", ""],
capture_output=True,
check=True,
timeout=30,
)
async def start_ssh_server() -> None:
_ensure_host_key()
key_file = str(TUNNEL_SSH_HOST_KEY)
logger.info("Starting DevTunnel SSH server on port %d", TUNNEL_SSH_PORT)
await asyncssh.create_server(
lambda: None,
None,
TUNNEL_SSH_PORT,
server_host_keys=[key_file],
authorization_errors=1,
password_auth=password_auth_handler,
process_factory=handle_connection,
keep_alive_interval=30,
keep_alive_count_max=3,
login_timeout=30,
max_auth_tries=3,
)
logger.info("DevTunnel SSH server running on 0.0.0.0:%d", TUNNEL_SSH_PORT)
await asyncio.Event().wait()
@@ -1,189 +0,0 @@
# retoor <retoor@molodetz.nl>
import json
import logging
import threading
import time
from dataclasses import dataclass, asdict
from pathlib import Path
from typing import Optional
from uuid_utils import uuid7
from devplacepy.services.tunnel.config import (
TUNNEL_MIN_PORT,
TUNNEL_MAX_PORT,
TUNNEL_MAX_BANDWIDTH_PER_CLIENT,
)
logger = logging.getLogger(__name__)
_lock = threading.Lock()
@dataclass
class TunnelSession:
session_id: str
username: str
subdomain: Optional[str]
local_port: int
remote_port: int
started_at: float
ended_at: Optional[float] = None
bytes_in: int = 0
bytes_out: int = 0
requests_count: int = 0
bandwidth_exceeded: bool = False
class SessionManager:
def __init__(self, persist_path: Optional[Path] = None) -> None:
self._sessions: dict[str, TunnelSession] = {}
self._used_ports: set[int] = set()
self._port_range: tuple[int, int] = (TUNNEL_MIN_PORT, TUNNEL_MAX_PORT)
self._persist_path: Optional[Path] = persist_path
if persist_path:
persist_path.parent.mkdir(parents=True, exist_ok=True)
self._load()
def allocate_port(self) -> Optional[int]:
lo, hi = self._port_range
available = [p for p in range(lo, hi + 1) if p not in self._used_ports]
if not available:
return None
port = available[0]
self._used_ports.add(port)
return port
def release_port(self, port: int) -> None:
self._used_ports.discard(port)
def create_session(
self,
username: str,
local_port: int,
subdomain: Optional[str] = None,
) -> Optional[TunnelSession]:
remote_port = self.allocate_port()
if remote_port is None:
return None
session_id = str(uuid7())
session = TunnelSession(
session_id=session_id,
username=username,
subdomain=subdomain,
local_port=local_port,
remote_port=remote_port,
started_at=time.time(),
)
with _lock:
self._sessions[session_id] = session
self._save()
return session
def get(self, session_id: str) -> Optional[TunnelSession]:
with _lock:
return self._sessions.get(session_id)
def get_by_remote_port(self, remote_port: int) -> Optional[TunnelSession]:
with _lock:
for s in self._sessions.values():
if s.remote_port == remote_port:
return s
return None
def get_by_subdomain(self, subdomain: str) -> Optional[TunnelSession]:
with _lock:
for s in self._sessions.values():
if s.subdomain == subdomain:
return s
return None
def list_active(self) -> list[TunnelSession]:
with _lock:
return [s for s in self._sessions.values() if s.ended_at is None]
def list_by_user(self, username: str) -> list[TunnelSession]:
with _lock:
return [s for s in self._sessions.values() if s.username == username]
def user_active_count(self, username: str) -> int:
with _lock:
return sum(
1 for s in self._sessions.values()
if s.username == username and s.ended_at is None
)
def end_session(self, session_id: str) -> Optional[TunnelSession]:
with _lock:
session = self._sessions.get(session_id)
if session and session.ended_at is None:
session.ended_at = time.time()
self.release_port(session.remote_port)
self._save()
return session
def record_traffic(
self,
session_id: str,
bytes_in: int = 0,
bytes_out: int = 0,
requests_count: int = 0,
) -> bool:
with _lock:
session = self._sessions.get(session_id)
if not session:
return True
session.bytes_in += bytes_in
session.bytes_out += bytes_out
session.requests_count += requests_count
if (
TUNNEL_MAX_BANDWIDTH_PER_CLIENT > 0
and session.bytes_out > TUNNEL_MAX_BANDWIDTH_PER_CLIENT
):
session.bandwidth_exceeded = True
return False
return True
def _load(self) -> None:
path = self._persist_path
if not path or not path.exists():
return
try:
raw = path.read_text()
data = json.loads(raw)
with _lock:
for item in data.get("sessions", []):
session = TunnelSession(**item)
self._sessions[session.session_id] = session
self._used_ports.add(session.remote_port)
except (json.JSONDecodeError, OSError, TypeError):
logger.exception("Failed to load session state")
def _save(self) -> None:
path = self._persist_path
if not path:
return
try:
path.parent.mkdir(parents=True, exist_ok=True)
with _lock:
sessions = [asdict(s) for s in self._sessions.values()]
data = {"sessions": sessions}
path.write_text(json.dumps(data, indent=2))
except OSError:
logger.exception("Failed to save session state")
_manager: Optional[SessionManager] = None
def get_manager() -> SessionManager:
global _manager
if _manager is None:
from devplacepy.config import DATA_DIR
path = DATA_DIR / "tunnel_sessions.json"
_manager = SessionManager(persist_path=path)
return _manager
def reset_manager() -> None:
global _manager
_manager = None
-87
View File
@@ -1,87 +0,0 @@
# retoor <retoor@molodetz.nl>
import asyncio
import json
import logging
import time
from pathlib import Path
from typing import Optional
from devplacepy.services.tunnel.config import TUNNEL_AUTH_CACHE_TTL, TUNNEL_AUTH_API_URL
logger = logging.getLogger(__name__)
_cache: dict[str, tuple[float, bool]] = {}
_cache_path: Optional[Path] = None
def _cache_file() -> Path:
global _cache_path
if _cache_path is None:
from devplacepy.config import KEYS_DIR
_cache_path = KEYS_DIR / "tunnel_auth_cache.json"
return _cache_path
def _load_cache() -> None:
global _cache
path = _cache_file()
if not path.exists():
_cache = {}
return
try:
raw = path.read_text()
data = json.loads(raw)
now = time.time()
_cache = {
k: (v["expires_at"], v["valid"])
for k, v in data.items()
if v.get("expires_at", 0) > now
}
except (json.JSONDecodeError, OSError):
_cache = {}
def _save_cache() -> None:
path = _cache_file()
try:
path.parent.mkdir(parents=True, exist_ok=True)
data = {
k: {"expires_at": exp, "valid": valid}
for k, (exp, valid) in _cache.items()
}
path.write_text(json.dumps(data, indent=2))
except OSError:
logger.exception("Failed to write auth cache")
async def authenticate_user(username: str, password: str) -> bool:
cache_key = f"{username}:{password}"
now = time.time()
if cache_key in _cache:
expires_at, valid = _cache[cache_key]
if expires_at > now:
if valid:
return True
del _cache[cache_key]
_save_cache()
try:
import httpx
async with httpx.AsyncClient(timeout=10) as client:
resp = await client.post(
TUNNEL_AUTH_API_URL,
json={"username": username, "password": password},
)
valid = resp.status_code == 200
except httpx.RequestError:
logger.exception("Auth API request failed for %s", username)
await asyncio.sleep(3)
return False
if valid:
_cache[cache_key] = (now + TUNNEL_AUTH_CACHE_TTL, True)
_save_cache()
return True
await asyncio.sleep(3)
return False
@@ -1,92 +0,0 @@
# retoor <retoor@molodetz.nl>
import json
import logging
import threading
from typing import Optional
from devplacepy.services.tunnel.config import TUNNEL_SUBDOMAIN_REGISTRY_PATH
logger = logging.getLogger(__name__)
_lock = threading.Lock()
class SubdomainRegistry:
def __init__(self) -> None:
self._sub_to_session: dict[str, str] = {}
self._session_to_sub: dict[str, str] = {}
self._load()
def _load(self) -> None:
path = TUNNEL_SUBDOMAIN_REGISTRY_PATH
if not path.exists():
return
try:
raw = path.read_text()
data = json.loads(raw)
with _lock:
self._sub_to_session = data.get("sub_to_session", {})
self._session_to_sub = data.get("session_to_sub", {})
except (json.JSONDecodeError, OSError):
logger.exception("Failed to load subdomain registry")
def _save(self) -> None:
path = TUNNEL_SUBDOMAIN_REGISTRY_PATH
try:
path.parent.mkdir(parents=True, exist_ok=True)
data = {
"sub_to_session": self._sub_to_session,
"session_to_sub": self._session_to_sub,
}
path.write_text(json.dumps(data, indent=2))
except OSError:
logger.exception("Failed to save subdomain registry")
def claim(self, subdomain: str, session_id: str) -> bool:
with _lock:
if subdomain in self._sub_to_session:
return False
self._sub_to_session[subdomain] = session_id
self._session_to_sub[session_id] = subdomain
self._save()
return True
def release(self, session_id: str) -> Optional[str]:
with _lock:
sub = self._session_to_sub.pop(session_id, None)
if sub:
self._sub_to_session.pop(sub, None)
self._save()
return sub
def resolve(self, subdomain: str) -> Optional[str]:
with _lock:
return self._sub_to_session.get(subdomain)
def session_subdomain(self, session_id: str) -> Optional[str]:
with _lock:
return self._session_to_sub.get(session_id)
def all_subdomains(self) -> dict[str, str]:
with _lock:
return dict(self._sub_to_session)
def clear(self) -> None:
with _lock:
self._sub_to_session.clear()
self._session_to_sub.clear()
self._save()
_registry: Optional[SubdomainRegistry] = None
def get_registry() -> SubdomainRegistry:
global _registry
if _registry is None:
_registry = SubdomainRegistry()
return _registry
def reset_registry() -> None:
global _registry
_registry = None
-19
View File
@@ -1,19 +0,0 @@
# retoor <retoor@molodetz.nl>
import asyncio
import logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
async def main() -> None:
from devplacepy.services.tunnel.server import start_ssh_server
await start_ssh_server()
if __name__ == "__main__":
asyncio.run(main())
-23
View File
@@ -40,29 +40,6 @@ services:
volumes:
- ./devplacepy/static:/app/static:ro
- ${DEVPLACE_DATA_DIR:-./data}/uploads:/data/uploads:ro
- ./nginx/tunnel.d:/etc/nginx/tunnel.d
depends_on:
app:
condition: service_healthy
networks:
- appnet
ssh-server:
build:
context: .
dockerfile: Dockerfile.ssh
restart: unless-stopped
env_file:
- .env
environment:
DEVPLACE_TUNNEL_SSH_PORT: "4242"
DEVPLACE_TUNNEL_AUTH_API_URL: "http://app:10500/api/tunnel/auth-check"
DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR: /etc/nginx/tunnel.d
ports:
- "4242:4242"
volumes:
- .:/app
- ./nginx/tunnel.d:/etc/nginx/tunnel.d
depends_on:
app:
condition: service_healthy
-2
View File
@@ -2,8 +2,6 @@ FROM nginx:alpine
RUN apk add --no-cache gettext
RUN mkdir -p /etc/nginx/tunnel.d
COPY nginx/nginx.conf.template /etc/nginx/templates/default.conf.template
COPY nginx/start.sh /start.sh
COPY devplacepy/static /app/static
-22
View File
@@ -209,25 +209,3 @@ server {
${NGINX_CACHE_CONFIG}
}
}
# DevTunnel reverse proxy - routes *.tunnel.devplace.net through SSH tunnels.
server {
listen 80;
listen [::]:80;
server_name ~^(?<tunnel_subdomain>[^.]+)\.tunnel\.devplace\.net$;
client_max_body_size ${NGINX_MAX_BODY_SIZE};
gzip on;
gzip_types text/plain text/css text/javascript application/javascript application/json image/svg+xml;
gzip_min_length 1000;
gzip_vary on;
gzip_proxied any;
include /etc/nginx/tunnel.d/*.conf;
location / {
return 502;
add_header Content-Type text/plain;
}
}
-1
View File
@@ -35,7 +35,6 @@ dependencies = [
"curl_cffi",
"faker",
"defusedxml",
"asyncssh",
]
[project.scripts]
+4 -3
View File
@@ -196,7 +196,8 @@ def test_robots_txt_exists(app_server):
assert "Sitemap:" in r.text
def test_robots_disallows_admin_and_uploads(app_server):
def test_robots_disallows_auth_gated_paths(app_server):
r = requests.get(f"{BASE_URL}/robots.txt")
assert "Disallow: /admin/" in r.text
assert "Disallow: /uploads/" in r.text
required = ["/auth/", "/game/", "/messages/", "/notifications/", "/admin/", "/uploads/"]
for path in required:
assert f"Disallow: {path}" in r.text, f"Missing Disallow: {path}"
+50
View File
@@ -0,0 +1,50 @@
# retoor <retoor@molodetz.nl>
import requests
from tests.conftest import BASE_URL
def test_bot_blocked_on_auth_path(app_server):
headers = {"User-Agent": "Google-Read-Aloud"}
r = requests.get(f"{BASE_URL}/messages", headers=headers)
assert r.status_code == 204
assert r.content == b""
def test_bot_blocked_on_game_path(app_server):
headers = {"User-Agent": "Google-Read-Aloud"}
r = requests.get(f"{BASE_URL}/game", headers=headers)
assert r.status_code == 204
assert r.content == b""
def test_bot_blocked_on_admin_path(app_server):
headers = {"User-Agent": "Google-Read-Aloud"}
r = requests.get(f"{BASE_URL}/admin", headers=headers)
assert r.status_code == 204
assert r.content == b""
def test_bot_not_blocked_on_public_path(app_server):
headers = {"User-Agent": "Google-Read-Aloud"}
r = requests.get(f"{BASE_URL}/", headers=headers)
assert r.status_code == 200
def test_normal_user_not_blocked(app_server):
r = requests.get(f"{BASE_URL}/messages", allow_redirects=False)
assert r.status_code == 303
def test_known_crawler_user_agent_variants(app_server):
ua_list = [
"Googlebot",
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
"bingbot/2.0; +http://www.bing.com/bingbot.htm",
"Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; ...)",
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
]
for ua in ua_list:
headers = {"User-Agent": ua}
r = requests.get(f"{BASE_URL}/admin", headers=headers)
assert r.status_code == 204, f"Expected 204 for UA: {ua}"
-1
View File
@@ -1 +0,0 @@
# retoor <retoor@molodetz.nl>
-180
View File
@@ -1,180 +0,0 @@
# retoor <retoor@molodetz.nl>
from unittest.mock import patch
from devplacepy.services.tunnel.session_manager import SessionManager
from devplacepy.services.tunnel.config import TUNNEL_RATE_LIMIT_PER_USER
def test_session_creation(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=8080, subdomain="my-app")
assert session is not None
assert session.username == "alice"
assert session.local_port == 8080
assert session.subdomain == "my-app"
assert session.session_id is not None
assert session.ended_at is None
def test_session_uniqueness(local_db):
manager = SessionManager()
s1 = manager.create_session(username="alice", local_port=3000, subdomain="app-a")
s2 = manager.create_session(username="alice", local_port=3001, subdomain="app-b")
assert s1 is not None
assert s2 is not None
assert s1.session_id != s2.session_id
assert s1.remote_port != s2.remote_port
def test_list_active(local_db):
manager = SessionManager()
assert manager.list_active() == []
manager.create_session(username="alice", local_port=5000, subdomain="list-a")
assert len(manager.list_active()) == 1
manager.create_session(username="bob", local_port=5001, subdomain="list-b")
assert len(manager.list_active()) == 2
def test_list_by_user(local_db):
manager = SessionManager()
manager.create_session(username="alice", local_port=6000, subdomain="usr-a")
manager.create_session(username="alice", local_port=6001, subdomain="usr-b")
manager.create_session(username="bob", local_port=6002, subdomain="usr-c")
alice_sessions = manager.list_by_user("alice")
assert len(alice_sessions) == 2
bob_sessions = manager.list_by_user("bob")
assert len(bob_sessions) == 1
unknown = manager.list_by_user("nobody")
assert len(unknown) == 0
def test_user_active_count(local_db):
manager = SessionManager()
assert manager.user_active_count("alice") == 0
manager.create_session(username="alice", local_port=7000, subdomain="cnt-a")
assert manager.user_active_count("alice") == 1
manager.create_session(username="alice", local_port=7001, subdomain="cnt-b")
assert manager.user_active_count("alice") == 2
manager.create_session(username="bob", local_port=7002, subdomain="cnt-c")
assert manager.user_active_count("alice") == 2
assert manager.user_active_count("bob") == 1
def test_end_session(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=8000, subdomain="end-a")
assert session is not None
assert session.ended_at is None
ended = manager.end_session(session.session_id)
assert ended is not None
assert ended.ended_at is not None
assert len(manager.list_active()) == 0
def test_end_session_releases_port(local_db):
manager = SessionManager()
s1 = manager.create_session(username="alice", local_port=9000, subdomain="port-a")
remote_port = s1.remote_port
manager.end_session(s1.session_id)
s2 = manager.create_session(username="alice", local_port=9001, subdomain="port-b")
assert s2.remote_port == remote_port, "Released port must be reused"
def test_get_session(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=10000, subdomain="get-a")
fetched = manager.get(session.session_id)
assert fetched is not None
assert fetched.session_id == session.session_id
assert manager.get("nonexistent") is None
def test_record_traffic(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=11000, subdomain="traffic-a")
manager.record_traffic(session.session_id, bytes_in=100, bytes_out=200, requests_count=1)
s = manager.get(session.session_id)
assert s.bytes_in == 100
assert s.bytes_out == 200
assert s.requests_count == 1
manager.record_traffic(session.session_id, bytes_in=50, bytes_out=75, requests_count=2)
s = manager.get(session.session_id)
assert s.bytes_in == 150
assert s.bytes_out == 275
assert s.requests_count == 3
def test_record_traffic_nonexistent_session(local_db):
manager = SessionManager()
result = manager.record_traffic("nonexistent", bytes_in=10)
assert result is True
def test_rate_limit_enforced(local_db):
manager = SessionManager()
for i in range(TUNNEL_RATE_LIMIT_PER_USER):
s = manager.create_session(username="alice", local_port=12000 + i, subdomain=f"rate-{i}")
assert s is not None
assert manager.user_active_count("alice") == TUNNEL_RATE_LIMIT_PER_USER
overflow = manager.create_session(username="alice", local_port=13000, subdomain="rate-overflow")
assert overflow is not None
def test_bandwidth_limit_not_exceeded(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=14000, subdomain="bw-a")
with patch("devplacepy.services.tunnel.session_manager.TUNNEL_MAX_BANDWIDTH_PER_CLIENT", 1000):
result = manager.record_traffic(session.session_id, bytes_out=500)
assert result is True
assert session.bandwidth_exceeded is False
def test_bandwidth_limit_exceeded(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=15000, subdomain="bw-b")
with patch("devplacepy.services.tunnel.session_manager.TUNNEL_MAX_BANDWIDTH_PER_CLIENT", 1000):
result = manager.record_traffic(session.session_id, bytes_out=1500)
assert result is False
assert session.bandwidth_exceeded is True
def test_bandwidth_limit_zero_disabled(local_db):
manager = SessionManager()
session = manager.create_session(username="alice", local_port=16000, subdomain="bw-c")
with patch("devplacepy.services.tunnel.session_manager.TUNNEL_MAX_BANDWIDTH_PER_CLIENT", 0):
result = manager.record_traffic(session.session_id, bytes_out=99999999)
assert result is True
assert session.bandwidth_exceeded is False
def test_port_exhaustion(local_db):
manager = SessionManager()
manager._port_range = (50000, 50001)
s1 = manager.create_session(username="alice", local_port=17000, subdomain="exh-a")
assert s1 is not None
s2 = manager.create_session(username="alice", local_port=17001, subdomain="exh-b")
assert s2 is not None
s3 = manager.create_session(username="alice", local_port=17002, subdomain="exh-c")
assert s3 is None
@@ -1,95 +0,0 @@
# retoor <retoor@molodetz.nl>
import time
from unittest.mock import patch
import pytest
from devplacepy.services.tunnel.ssh_auth import authenticate_user, _cache, _load_cache, _save_cache
def _clear_cache():
_cache.clear()
def test_auth_cache_hits_valid(local_db):
_clear_cache()
key = "test-user:valid-pass"
future = time.time() + 300
_cache[key] = (future, True)
_save_cache()
_load_cache()
assert key in _cache
expires_at, valid = _cache[key]
assert valid is True
assert expires_at > time.time()
def test_auth_cache_expired_entry_removed(local_db):
_clear_cache()
key = "test-user:expired-pass"
past = time.time() - 10
_cache[key] = (past, True)
_save_cache()
_load_cache()
assert key not in _cache
def test_auth_cache_invalid_rejected(local_db):
_clear_cache()
key = "test-user:wrong-pass"
future = time.time() + 300
_cache[key] = (future, False)
_save_cache()
_load_cache()
assert key in _cache
_, valid = _cache[key]
assert valid is False
@pytest.mark.asyncio
async def test_authenticate_user_invalid_credentials(local_db):
_clear_cache()
with patch("devplacepy.services.tunnel.ssh_auth.TUNNEL_AUTH_API_URL", "http://localhost:99999/api/tunnel/auth-check"):
result = await authenticate_user("nobody", "wrong-password")
assert result is False
@pytest.mark.asyncio
async def test_authenticate_user_wrong_password_delays(local_db):
_clear_cache()
with patch("devplacepy.services.tunnel.ssh_auth.TUNNEL_AUTH_API_URL", "http://localhost:99999/api/tunnel/auth-check"):
start = time.time()
result = await authenticate_user("alice", "wrong-password")
elapsed = time.time() - start
assert result is False
assert elapsed >= 3.0, "Wrong password response must delay at least 3 seconds"
def test_cache_persistence(local_db):
_clear_cache()
key = "persist-user:pass123"
future = time.time() + 300
_cache[key] = (future, True)
_save_cache()
_cache.clear()
_load_cache()
if key in _cache:
_, valid = _cache[key]
assert valid is True
else:
assert key not in _cache
def test_cache_ttl_enforced(local_db):
_clear_cache()
key = "ttl-user:pass456"
past = time.time() - 1
_cache[key] = (past, True)
assert _cache[key][0] < time.time()
@@ -1,88 +0,0 @@
# retoor <retoor@molodetz.nl>
from devplacepy.services.tunnel.subdomain_registry import SubdomainRegistry
def test_claim_and_release(local_db):
registry = SubdomainRegistry()
sub = "my-app"
session_id = "session-1"
assert registry.claim(sub, session_id) is True
assert registry.claim(sub, "session-2") is False, "Duplicate claim must fail"
assert registry.resolve(sub) == session_id
assert registry.resolve("unknown") is None
released = registry.release(session_id)
assert released == sub
assert registry.resolve(sub) is None
def test_release_unknown(local_db):
registry = SubdomainRegistry()
result = registry.release("nonexistent-session")
assert result is None
def test_claim_twice_different_session(local_db):
registry = SubdomainRegistry()
sub = "shared-app"
assert registry.claim(sub, "session-a") is True
assert registry.claim(sub, "session-b") is False
registry.release("session-a")
assert registry.claim(sub, "session-b") is True
def test_claim_twice_same_session(local_db):
registry = SubdomainRegistry()
sub = "retry-app"
session_id = "session-retry"
assert registry.claim(sub, session_id) is True
result = registry.claim(sub, session_id)
assert result is False, "Claiming the same subdomain again must fail even for same session"
def test_release_then_claim(local_db):
registry = SubdomainRegistry()
sub = "cyclic-app"
session_id = "session-cycle"
assert registry.claim(sub, session_id) is True
registry.release(session_id)
assert registry.claim(sub, "session-cycle-2") is True
def test_subdomain_uniqueness_enforced(local_db):
registry = SubdomainRegistry()
assert registry.claim("alpha", "s1") is True
assert registry.claim("beta", "s2") is True
assert registry.claim("alpha", "s3") is False
def test_idempotent_release(local_db):
registry = SubdomainRegistry()
assert registry.claim("z-app", "sz1") is True
assert registry.release("sz1") == "z-app"
assert registry.release("sz1") is None
assert registry.claim("z-app", "sz2") is True
def test_resolve_empty_registry(local_db):
registry = SubdomainRegistry()
assert registry.resolve("anything") is None
def test_session_to_sub_mapping(local_db):
registry = SubdomainRegistry()
sub = "mapped-app"
session_id = "session-mapped"
registry.claim(sub, session_id)
assert registry.resolve(sub) == session_id
registry.release(session_id)
assert registry.resolve(sub) is None