ticket #81 attempt 2
This commit is contained in:
parent
7152db1fd2
commit
73052e3e1b
1
.dpc/verify_cache.json
Normal file
1
.dpc/verify_cache.json
Normal file
File diff suppressed because one or more lines are too long
@ -52,6 +52,8 @@ DEVPLACE_GID=1000
|
|||||||
# DEVPLACE_TUNNEL_AUTH_API_URL=http://app:10500/api/tunnel/auth-check
|
# DEVPLACE_TUNNEL_AUTH_API_URL=http://app:10500/api/tunnel/auth-check
|
||||||
# Maximum concurrent tunnels per user
|
# Maximum concurrent tunnels per user
|
||||||
# DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER=5
|
# DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER=5
|
||||||
|
# Maximum bytes out per tunnel (10MB default; 0 = unlimited)
|
||||||
|
# DEVPLACE_TUNNEL_MAX_BANDWIDTH_PER_CLIENT=10485760
|
||||||
# Auth cache TTL in seconds
|
# Auth cache TTL in seconds
|
||||||
# DEVPLACE_TUNNEL_AUTH_CACHE_TTL=300
|
# DEVPLACE_TUNNEL_AUTH_CACHE_TTL=300
|
||||||
# Remote port range for tunnel allocation
|
# Remote port range for tunnel allocation
|
||||||
|
|||||||
@ -129,6 +129,7 @@ Nested `CLAUDE.md` files (loaded automatically by Claude Code only when a file i
|
|||||||
| `devplacepy/services/dbapi/CLAUDE.md` | `/dbapi` primary-admin-only read-only database API |
|
| `devplacepy/services/dbapi/CLAUDE.md` | `/dbapi` primary-admin-only read-only database API |
|
||||||
| `devplacepy/services/pubsub/CLAUDE.md` | Database-free pub/sub bus |
|
| `devplacepy/services/pubsub/CLAUDE.md` | Database-free pub/sub bus |
|
||||||
| `devplacepy/services/game/CLAUDE.md` | Code Farm idle game |
|
| `devplacepy/services/game/CLAUDE.md` | Code Farm idle game |
|
||||||
|
| `devplacepy/services/tunnel/CLAUDE.md` | DevTunnel SSH reverse tunneling: SSH server, session manager, subdomain registry, auth cache, nginx integration, metrics |
|
||||||
| `devplacepy/services/CLAUDE.md` | Background task queue, AI correction/modifier, presence, live view relay, `BaseService`/`ServiceManager` |
|
| `devplacepy/services/CLAUDE.md` | Background task queue, AI correction/modifier, presence, live view relay, `BaseService`/`ServiceManager` |
|
||||||
| `devplacepy/database/CLAUDE.md` | Dataset rules, indexing conventions, soft delete, tables, site settings |
|
| `devplacepy/database/CLAUDE.md` | Dataset rules, indexing conventions, soft delete, tables, site settings |
|
||||||
| `devplacepy/utils/CLAUDE.md` | Notifications and gamification (XP/levels/badges/leaderboard) |
|
| `devplacepy/utils/CLAUDE.md` | Notifications and gamification (XP/levels/badges/leaderboard) |
|
||||||
@ -173,6 +174,7 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror
|
|||||||
| `/game` | game/ package - see `services/game/CLAUDE.md` |
|
| `/game` | game/ package - see `services/game/CLAUDE.md` |
|
||||||
| (none) | push.py (web push/PWA), docs.py (see `routers/docs/CLAUDE.md`) |
|
| (none) | push.py (web push/PWA), docs.py (see `routers/docs/CLAUDE.md`) |
|
||||||
| `/pubsub` | pubsub.py - see `services/pubsub/CLAUDE.md` |
|
| `/pubsub` | pubsub.py - see `services/pubsub/CLAUDE.md` |
|
||||||
|
| `/api/tunnel` | tunnel.py - DevTunnel REST API (auth check, session listing, session kill, metrics) |
|
||||||
| (none) | seo.py - `/robots.txt`, `/sitemap.xml` |
|
| (none) | seo.py - `/robots.txt`, `/sitemap.xml` |
|
||||||
|
|
||||||
### Templates and frontend
|
### Templates and frontend
|
||||||
@ -224,9 +226,9 @@ Users and guests inject their own CSS and JS, scoped to a page type or globally,
|
|||||||
|
|
||||||
`services/background.py` `background.submit(fn, *args)` is a generic fire-and-forget offload onto one per-worker `asyncio.Queue`; when the consumer isn't running (tests, full queue) it runs the callable **inline**, so audit/notification/XP writes stay deterministic for the suite while production defers them. It is the choke point for every audit-log write, every XP award, and every notification - handlers call `award_rewards`/`create_notification` directly (never wrap them in `background.submit`, that double-queues). AI content correction (opt-in, off by default) and the AI modifier (`@ai <instruction>` inline directive, on by default) rewrite user prose via the internal gateway; sync apply mode never blocks the event loop (`run_in_executor` + the `await_pending_corrections` middleware). `BaseService`/`ServiceManager` provide the async run loop and singleton registry for background services (`NewsService`, `GatewayService`, `DeviiService`, `BotsService`, container/audit/telegram reconcilers). Full detail on all of this, plus presence and the live view relay, is in `devplacepy/services/CLAUDE.md`.
|
`services/background.py` `background.submit(fn, *args)` is a generic fire-and-forget offload onto one per-worker `asyncio.Queue`; when the consumer isn't running (tests, full queue) it runs the callable **inline**, so audit/notification/XP writes stay deterministic for the suite while production defers them. It is the choke point for every audit-log write, every XP award, and every notification - handlers call `award_rewards`/`create_notification` directly (never wrap them in `background.submit`, that double-queues). AI content correction (opt-in, off by default) and the AI modifier (`@ai <instruction>` inline directive, on by default) rewrite user prose via the internal gateway; sync apply mode never blocks the event loop (`run_in_executor` + the `await_pending_corrections` middleware). `BaseService`/`ServiceManager` provide the async run loop and singleton registry for background services (`NewsService`, `GatewayService`, `DeviiService`, `BotsService`, container/audit/telegram reconcilers). Full detail on all of this, plus presence and the live view relay, is in `devplacepy/services/CLAUDE.md`.
|
||||||
|
|
||||||
### Container manager, Devii assistant, AI gateway, async jobs, audit log
|
### Container manager, DevTunnel, Devii assistant, AI gateway, async jobs, audit log
|
||||||
|
|
||||||
DevPlace ships a Docker-backed **Container manager** (admin-only, one shared `ppy:latest` image, security-hardened rootless workflow via a `sudo` superclone and `aptroot`), the **Devii** in-platform agentic assistant (WebSocket terminal, per-owner-channel sessions, scheduler/reminders, self-learning lessons, user-defined virtual tools, self-configured behavior, browser-automation client tools), the **AI gateway** (`/openai/v1/*`, single point of truth for every AI call, per-user cost attribution, provider/model routing overlay), **async job services** (`JobService` base pattern: zip, fork, SEO diagnostics, SEO metadata generation, DeepSearch, AI Usage Analyzer), and an admin-only append-only **audit log** (`record`/`record_system` entrypoints, `events.md` catalogue of 223 keys, never raises into the caller). These are among the largest subsystems in the codebase - read their dedicated nested `CLAUDE.md` files (see Subsystem map) before working in any of them; do not assume prior knowledge from this summary.
|
DevPlace ships a Docker-backed **Container manager** (admin-only, one shared `ppy:latest` image, security-hardened rootless workflow via a `sudo` superclone and `aptroot`), **DevTunnel** (SSH-based reverse tunneling at `*.tunnel.devplace.net`: users run `ssh username@tunnel.devplace.net -p 4242`, authenticate with their DevPlace password, and expose local services through public HTTPS subdomains; the SSH server runs on port 4242 via asyncssh, sessions are managed in-memory with JSON persistence, subdomains are uniquely tracked in a file-backed registry, and nginx snippet generation enables Caddy-less TLS termination), the **Devii** in-platform agentic assistant (WebSocket terminal, per-owner-channel sessions, scheduler/reminders, self-learning lessons, user-defined virtual tools, self-configured behavior, browser-automation client tools), the **AI gateway** (`/openai/v1/*`, single point of truth for every AI call, per-user cost attribution, provider/model routing overlay), **async job services** (`JobService` base pattern: zip, fork, SEO diagnostics, SEO metadata generation, DeepSearch, AI Usage Analyzer), and an admin-only append-only **audit log** (`record`/`record_system` entrypoints, `events.md` catalogue of 223 keys, never raises into the caller). These are among the largest subsystems in the codebase - read their dedicated nested `CLAUDE.md` files (see Subsystem map) before working in any of them; do not assume prior knowledge from this summary.
|
||||||
|
|
||||||
### Telegram bot, email, devRant compatibility API, issue tracker
|
### Telegram bot, email, devRant compatibility API, issue tracker
|
||||||
|
|
||||||
|
|||||||
@ -88,6 +88,7 @@ devplacepy/
|
|||||||
| `/admin/bots` | Admin **Bot Monitor**: a live grid of the latest low-quality screenshot per running bot persona, each labelled with the bot username, persona, and current action, auto-refreshing |
|
| `/admin/bots` | Admin **Bot Monitor**: a live grid of the latest low-quality screenshot per running bot persona, each labelled with the bot username, persona, and current action, auto-refreshing |
|
||||||
| `/admin` | Admin panel (user management, news curation, settings) |
|
| `/admin` | Admin panel (user management, news curation, settings) |
|
||||||
| `/docs` | Developer documentation site with a complete, interactive HTTP API reference |
|
| `/docs` | Developer documentation site with a complete, interactive HTTP API reference |
|
||||||
|
| `/api/tunnel` | **DevTunnel** - SSH reverse tunneling REST API: auth check, session listing/kill, metrics. The SSH server runs on port 4242 at `tunnel.devplace.net`. Users run `ssh username@tunnel.devplace.net -p 4242` and authenticate with their DevPlace password to expose local services through `*.tunnel.devplace.net` subdomains |
|
||||||
| `/openai` | OpenAI-compatible LLM gateway service (`/openai/v1/chat/completions`, `/openai/v1/*`) |
|
| `/openai` | OpenAI-compatible LLM gateway service (`/openai/v1/chat/completions`, `/openai/v1/*`) |
|
||||||
| `/devii` | Devii agentic assistant: WebSocket terminal (`/devii/ws`), standalone page, usage (`/devii/usage`), session bootstrap |
|
| `/devii` | Devii agentic assistant: WebSocket terminal (`/devii/ws`), standalone page, usage (`/devii/usage`), session bootstrap |
|
||||||
| `(none)` | `/robots.txt`, `/sitemap.xml` (SEO) |
|
| `(none)` | `/robots.txt`, `/sitemap.xml` (SEO) |
|
||||||
|
|||||||
@ -188,3 +188,70 @@ Online status is a single **`users.last_seen`** UTC-ISO column (ensured in `data
|
|||||||
**Avatar presence dot (sitewide, DRY):** a small corner dot on **every** user avatar (green online, muted grey offline) comes from ONE reusable partial `templates/_presence_dot.html` - `<span class="presence-dot" data-presence-uid data-presence-last-seen>` guarded on `_user.get('uid')` (a partial-dict author, e.g. the issues includes, renders no dot). It carries **no** `data-presence-label`, so `PresenceManager` colours it with zero extra JS. It is included by the shared avatar partial `templates/_avatar_link.html` (its `.user-avatar-link` anchor is the positioning host, covering ~19 sites) and by the handful of raw-`<img class="avatar-img">` sites wrapped in a positioned `<span class="avatar-badge">` (the two `base.html` nav avatars, the `profile.html` hero + followers list, the `messages.html` conversation list). CSS in `static/css/base.css` (`.user-avatar-link`/`.avatar-badge` `position:relative;display:inline-flex`, `.presence-dot` sized `30%` of the avatar clamped 8-14px with a `--bg-card` ring, `.online` -> `--success`), so it is proportional and responsive at every avatar size with no per-size class. `database/follows.py` `get_follow_list` now carries `last_seen` in its trimmed dict so the followers/following dots resolve (all other author dicts are full `get_users_by_uids` rows). `dp-avatar` (`AppAvatar.js`) is docs-demo only (no real user avatars) and is intentionally out of scope. Reuse `_presence_dot.html` + the `.avatar-badge` wrapper for any new avatar surface - never hand-roll a presence dot.
|
**Avatar presence dot (sitewide, DRY):** a small corner dot on **every** user avatar (green online, muted grey offline) comes from ONE reusable partial `templates/_presence_dot.html` - `<span class="presence-dot" data-presence-uid data-presence-last-seen>` guarded on `_user.get('uid')` (a partial-dict author, e.g. the issues includes, renders no dot). It carries **no** `data-presence-label`, so `PresenceManager` colours it with zero extra JS. It is included by the shared avatar partial `templates/_avatar_link.html` (its `.user-avatar-link` anchor is the positioning host, covering ~19 sites) and by the handful of raw-`<img class="avatar-img">` sites wrapped in a positioned `<span class="avatar-badge">` (the two `base.html` nav avatars, the `profile.html` hero + followers list, the `messages.html` conversation list). CSS in `static/css/base.css` (`.user-avatar-link`/`.avatar-badge` `position:relative;display:inline-flex`, `.presence-dot` sized `30%` of the avatar clamped 8-14px with a `--bg-card` ring, `.online` -> `--success`), so it is proportional and responsive at every avatar size with no per-size class. `database/follows.py` `get_follow_list` now carries `last_seen` in its trimmed dict so the followers/following dots resolve (all other author dicts are full `get_users_by_uids` rows). `dp-avatar` (`AppAvatar.js`) is docs-demo only (no real user avatars) and is intentionally out of scope. Reuse `_presence_dot.html` + the `.avatar-badge` wrapper for any new avatar surface - never hand-roll a presence dot.
|
||||||
|
|
||||||
**Messaging refactor:** the old presence was per-worker and WS-connect-based (`message_hub.is_online`/`last_seen`, `_announce_presence`, the WS `presence` frame) and broke with >1 worker. That display path was **removed**; `message_hub` keeps only its socket connection tracking for message delivery. The messages header presence is now the shared `PresenceManager`, so chat presence is finally cross-worker correct. **Never re-implement WS-connect presence** - reuse `presence.is_online`, the `public.presence.{uid}` topic, and `PresenceManager`.
|
**Messaging refactor:** the old presence was per-worker and WS-connect-based (`message_hub.is_online`/`last_seen`, `_announce_presence`, the WS `presence` frame) and broke with >1 worker. That display path was **removed**; `message_hub` keeps only its socket connection tracking for message delivery. The messages header presence is now the shared `PresenceManager`, so chat presence is finally cross-worker correct. **Never re-implement WS-connect presence** - reuse `presence.is_online`, the `public.presence.{uid}` topic, and `PresenceManager`.
|
||||||
|
|
||||||
|
## DevTunnel (`services/tunnel/`)
|
||||||
|
|
||||||
|
SSH-based reverse tunneling for `*.tunnel.devplace.net`. Users connect via standard OpenSSH to port 4242, authenticate with their DevPlace password (validated against the `/api/tunnel/auth-check` endpoint), and expose local services through uniquely claimed subdomains. The tunnel subsystem is NOT a `BaseService` - it runs as a standalone asyncssh server process.
|
||||||
|
|
||||||
|
### Module layout
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| `config.py` | Re-exports `TUNNEL_*` constants from `devplacepy.config` |
|
||||||
|
| `server.py` | `asyncssh` SSH server on port 4242: password auth, interactive port/subdomain prompts, rate-limit enforcement (`TUNNEL_RATE_LIMIT_PER_USER`), session creation, SSH command generation |
|
||||||
|
| `ssh_auth.py` | `authenticate_user(username, password)` calls the DevPlace auth API, caches valid results with a TTL (`TUNNEL_AUTH_CACHE_TTL`), invalidates expired entries, and imposes a 3-second delay on wrong passwords |
|
||||||
|
| `session_manager.py` | `SessionManager`: in-memory session store with JSON file persistence, port allocation (`TUNNEL_MIN_PORT`-`TUNNEL_MAX_PORT`), traffic recording with bandwidth enforcement (`TUNNEL_MAX_BANDWIDTH_PER_CLIENT`), per-user active count for rate limiting |
|
||||||
|
| `subdomain_registry.py` | `SubdomainRegistry`: bidirectional subdomain-to-session mapping persisted as JSON, uniqueness enforcement on claim, safe release |
|
||||||
|
| `nginx_updater.py` | Writes nginx location snippets to `TUNNEL_NGINX_TUNNEL_DIR` per subdomain and reloads nginx |
|
||||||
|
| `metrics.py` | `get_metrics()` / `get_user_metrics()` aggregate active tunnels, bandwidth, and requests from the session manager |
|
||||||
|
|
||||||
|
### REST API (`routers/tunnel.py`)
|
||||||
|
|
||||||
|
| Method | Path | Auth | Description |
|
||||||
|
|--------|------|------|-------------|
|
||||||
|
| POST | `/api/tunnel/auth-check` | None | Validates password against DevPlace DB (called by SSH server) |
|
||||||
|
| GET | `/api/tunnel/sessions` | Admin | Lists all active tunnel sessions |
|
||||||
|
| GET | `/api/tunnel/sessions/{username}` | Auth | Lists sessions for a specific user |
|
||||||
|
| DELETE | `/api/tunnel/sessions/{session_id}` | Admin | Kills a tunnel session, releases its subdomain |
|
||||||
|
| GET | `/api/tunnel/metrics` | Admin | Aggregate tunnel metrics |
|
||||||
|
| GET | `/api/tunnel/metrics/{username}` | Admin | Per-user tunnel metrics |
|
||||||
|
|
||||||
|
### Managed objects and configuration
|
||||||
|
|
||||||
|
Config keys (all from `devplacepy.config`):
|
||||||
|
|
||||||
|
| Key | Env var | Default | Purpose |
|
||||||
|
|-----|---------|---------|---------|
|
||||||
|
| `TUNNEL_SSH_PORT` | `DEVPLACE_TUNNEL_SSH_PORT` | `4242` | SSH listen port |
|
||||||
|
| `TUNNEL_SSH_HOST_KEY` | `DEVPLACE_TUNNEL_SSH_HOST_KEY` | `KEYS_DIR / "tunnel_host_key"` | SSH host key path |
|
||||||
|
| `TUNNEL_MIN_PORT` | `DEVPLACE_TUNNEL_MIN_PORT` | `40000` | Remote port range start |
|
||||||
|
| `TUNNEL_MAX_PORT` | `DEVPLACE_TUNNEL_MAX_PORT` | `49999` | Remote port range end |
|
||||||
|
| `TUNNEL_DOMAIN` | `DEVPLACE_TUNNEL_DOMAIN` | `tunnel.devplace.net` | Public tunnel domain |
|
||||||
|
| `TUNNEL_AUTH_CACHE_TTL` | `DEVPLACE_TUNNEL_AUTH_CACHE_TTL` | `300` | Auth cache TTL seconds |
|
||||||
|
| `TUNNEL_MAX_BANDWIDTH_PER_CLIENT` | `DEVPLACE_TUNNEL_MAX_BANDWIDTH_PER_CLIENT` | `10485760` | Max bytes out per tunnel (10MB); 0 = unlimited |
|
||||||
|
| `TUNNEL_RATE_LIMIT_PER_USER` | `DEVPLACE_TUNNEL_RATE_LIMIT_PER_USER` | `5` | Max concurrent tunnels per user |
|
||||||
|
| `TUNNEL_SUBDOMAIN_REGISTRY_PATH` | (derived) | `KEYS_DIR / "tunnel_subdomains.json"` | Subdomain registry file |
|
||||||
|
| `TUNNEL_AUTH_API_URL` | `DEVPLACE_TUNNEL_AUTH_API_URL` | `http://localhost:10500/api/tunnel/auth-check` | Auth validation endpoint |
|
||||||
|
| `TUNNEL_NGINX_TUNNEL_DIR` | `DEVPLACE_TUNNEL_NGINX_TUNNEL_DIR` | `BASE_DIR / "nginx" / "tunnel.d"` | nginx snippets directory |
|
||||||
|
|
||||||
|
### CLI
|
||||||
|
|
||||||
|
`devplace tunnel list [--username U]` - list tunnels (optionally filtered by user). `devplace tunnel kill <session_id>` - terminate a session. `devplace tunnel metrics` - print aggregate metrics.
|
||||||
|
|
||||||
|
### DevII tools
|
||||||
|
|
||||||
|
Three actions registered in `devplacepy/services/devii/actions/catalog/tunnel.py`:
|
||||||
|
|
||||||
|
- `tunnel_list` (GET `/api/tunnel/sessions`, auth required, read-only)
|
||||||
|
- `tunnel_kill` (DELETE `/api/tunnel/sessions/{session_id}`, admin only)
|
||||||
|
- `tunnel_metrics` (GET `/api/tunnel/metrics`, admin only, read-only)
|
||||||
|
|
||||||
|
### Known invariants
|
||||||
|
|
||||||
|
- Rate limiting is enforced in `server.py` during the interactive session handshake, **before** the session is created.
|
||||||
|
- Bandwidth enforcement is in `SessionManager.record_traffic()` - if `TUNNEL_MAX_BANDWIDTH_PER_CLIENT > 0` and bytes_out exceeds it, `bandwidth_exceeded` is set and `record_traffic` returns False.
|
||||||
|
- Port range is configurable via `TUNNEL_MIN_PORT`/`TUNNEL_MAX_PORT`, NOT hardcoded.
|
||||||
|
- Subdomain uniqueness is enforced in `SubdomainRegistry.claim()` at the application level - two sessions cannot claim the same subdomain.
|
||||||
|
- Auth cache stores only valid results; invalid results trigger recheck; expired entries are pruned on load; wrong passwords impose a 3-second delay.
|
||||||
|
- The SSH server is a standalone process (Docker Compose `ssh-server` service), not a FastAPI route.
|
||||||
|
- nginx tunnel snippets live under `nginx/tunnel.d/` (bind-mounted between the nginx and ssh-server containers).
|
||||||
|
|||||||
@ -22,6 +22,7 @@ from .project_files import PROJECT_FILE_ACTIONS
|
|||||||
from .projects import PROJECTS_ACTIONS
|
from .projects import PROJECTS_ACTIONS
|
||||||
from .social import SOCIAL_ACTIONS
|
from .social import SOCIAL_ACTIONS
|
||||||
from .tools import TOOLS_ACTIONS
|
from .tools import TOOLS_ACTIONS
|
||||||
|
from .tunnel import TUNNEL_ACTIONS
|
||||||
from .uploads import UPLOAD_ACTIONS
|
from .uploads import UPLOAD_ACTIONS
|
||||||
|
|
||||||
ACTIONS: tuple[Action, ...] = (
|
ACTIONS: tuple[Action, ...] = (
|
||||||
@ -32,6 +33,7 @@ ACTIONS: tuple[Action, ...] = (
|
|||||||
+ PROJECT_FILE_ACTIONS
|
+ PROJECT_FILE_ACTIONS
|
||||||
+ JOB_ACTIONS
|
+ JOB_ACTIONS
|
||||||
+ TOOLS_ACTIONS
|
+ TOOLS_ACTIONS
|
||||||
|
+ TUNNEL_ACTIONS
|
||||||
+ PROFILE_ACTIONS
|
+ PROFILE_ACTIONS
|
||||||
+ MESSAGE_ACTIONS
|
+ MESSAGE_ACTIONS
|
||||||
+ NOTIFICATION_ACTIONS
|
+ NOTIFICATION_ACTIONS
|
||||||
|
|||||||
52
devplacepy/services/devii/actions/catalog/tunnel.py
Normal file
52
devplacepy/services/devii/actions/catalog/tunnel.py
Normal file
@ -0,0 +1,52 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from ..spec import Action
|
||||||
|
|
||||||
|
|
||||||
|
TUNNEL_ACTIONS: tuple[Action, ...] = (
|
||||||
|
Action(
|
||||||
|
name="tunnel_list",
|
||||||
|
method="GET",
|
||||||
|
path="/api/tunnel/sessions",
|
||||||
|
summary="List active SSH tunnel sessions",
|
||||||
|
description=(
|
||||||
|
"Returns all active DevTunnel sessions. For admins: every session "
|
||||||
|
"across all users. For members: only the caller's own sessions. "
|
||||||
|
"Each session includes session_id, username, subdomain, local_port, "
|
||||||
|
"remote_port, bytes in/out, and request count."
|
||||||
|
),
|
||||||
|
requires_auth=True,
|
||||||
|
read_only=True,
|
||||||
|
),
|
||||||
|
Action(
|
||||||
|
name="tunnel_kill",
|
||||||
|
method="DELETE",
|
||||||
|
path="/api/tunnel/sessions/{session_id}",
|
||||||
|
summary="Kill an active SSH tunnel session",
|
||||||
|
description=(
|
||||||
|
"Terminates a tunnel session by session_id. The tunnel's subdomain "
|
||||||
|
"is released and the nginx config snippet is removed. Admins can "
|
||||||
|
"kill any session; members can kill only their own."
|
||||||
|
),
|
||||||
|
params=(),
|
||||||
|
requires_auth=True,
|
||||||
|
requires_admin=True,
|
||||||
|
),
|
||||||
|
Action(
|
||||||
|
name="tunnel_metrics",
|
||||||
|
method="GET",
|
||||||
|
path="/api/tunnel/metrics",
|
||||||
|
summary="Get DevTunnel system metrics",
|
||||||
|
description=(
|
||||||
|
"Returns aggregate metrics: active tunnel count, total bytes in/out, "
|
||||||
|
"total requests, uptime seconds, and per-user breakdown. Admin only."
|
||||||
|
),
|
||||||
|
requires_auth=True,
|
||||||
|
requires_admin=True,
|
||||||
|
read_only=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = ["TUNNEL_ACTIONS"]
|
||||||
@ -9,6 +9,12 @@ from pathlib import Path
|
|||||||
from typing import Optional
|
from typing import Optional
|
||||||
from uuid_utils import uuid7
|
from uuid_utils import uuid7
|
||||||
|
|
||||||
|
from devplacepy.services.tunnel.config import (
|
||||||
|
TUNNEL_MIN_PORT,
|
||||||
|
TUNNEL_MAX_PORT,
|
||||||
|
TUNNEL_MAX_BANDWIDTH_PER_CLIENT,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
_lock = threading.Lock()
|
_lock = threading.Lock()
|
||||||
|
|
||||||
@ -25,13 +31,14 @@ class TunnelSession:
|
|||||||
bytes_in: int = 0
|
bytes_in: int = 0
|
||||||
bytes_out: int = 0
|
bytes_out: int = 0
|
||||||
requests_count: int = 0
|
requests_count: int = 0
|
||||||
|
bandwidth_exceeded: bool = False
|
||||||
|
|
||||||
|
|
||||||
class SessionManager:
|
class SessionManager:
|
||||||
def __init__(self, persist_path: Optional[Path] = None) -> None:
|
def __init__(self, persist_path: Optional[Path] = None) -> None:
|
||||||
self._sessions: dict[str, TunnelSession] = {}
|
self._sessions: dict[str, TunnelSession] = {}
|
||||||
self._used_ports: set[int] = set()
|
self._used_ports: set[int] = set()
|
||||||
self._port_range: tuple[int, int] = (40000, 49999)
|
self._port_range: tuple[int, int] = (TUNNEL_MIN_PORT, TUNNEL_MAX_PORT)
|
||||||
self._persist_path: Optional[Path] = persist_path
|
self._persist_path: Optional[Path] = persist_path
|
||||||
if persist_path:
|
if persist_path:
|
||||||
persist_path.parent.mkdir(parents=True, exist_ok=True)
|
persist_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
@ -120,13 +127,21 @@ class SessionManager:
|
|||||||
bytes_in: int = 0,
|
bytes_in: int = 0,
|
||||||
bytes_out: int = 0,
|
bytes_out: int = 0,
|
||||||
requests_count: int = 0,
|
requests_count: int = 0,
|
||||||
) -> None:
|
) -> bool:
|
||||||
with _lock:
|
with _lock:
|
||||||
session = self._sessions.get(session_id)
|
session = self._sessions.get(session_id)
|
||||||
if session:
|
if not session:
|
||||||
|
return True
|
||||||
session.bytes_in += bytes_in
|
session.bytes_in += bytes_in
|
||||||
session.bytes_out += bytes_out
|
session.bytes_out += bytes_out
|
||||||
session.requests_count += requests_count
|
session.requests_count += requests_count
|
||||||
|
if (
|
||||||
|
TUNNEL_MAX_BANDWIDTH_PER_CLIENT > 0
|
||||||
|
and session.bytes_out > TUNNEL_MAX_BANDWIDTH_PER_CLIENT
|
||||||
|
):
|
||||||
|
session.bandwidth_exceeded = True
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
def _load(self) -> None:
|
def _load(self) -> None:
|
||||||
path = self._persist_path
|
path = self._persist_path
|
||||||
|
|||||||
1
tests/unit/services/tunnel/__init__.py
Normal file
1
tests/unit/services/tunnel/__init__.py
Normal file
@ -0,0 +1 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
180
tests/unit/services/tunnel/test_server.py
Normal file
180
tests/unit/services/tunnel/test_server.py
Normal file
@ -0,0 +1,180 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from devplacepy.services.tunnel.session_manager import SessionManager
|
||||||
|
from devplacepy.services.tunnel.config import TUNNEL_RATE_LIMIT_PER_USER
|
||||||
|
|
||||||
|
|
||||||
|
def test_session_creation(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
session = manager.create_session(username="alice", local_port=8080, subdomain="my-app")
|
||||||
|
assert session is not None
|
||||||
|
assert session.username == "alice"
|
||||||
|
assert session.local_port == 8080
|
||||||
|
assert session.subdomain == "my-app"
|
||||||
|
assert session.session_id is not None
|
||||||
|
assert session.ended_at is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_session_uniqueness(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
s1 = manager.create_session(username="alice", local_port=3000, subdomain="app-a")
|
||||||
|
s2 = manager.create_session(username="alice", local_port=3001, subdomain="app-b")
|
||||||
|
assert s1 is not None
|
||||||
|
assert s2 is not None
|
||||||
|
assert s1.session_id != s2.session_id
|
||||||
|
assert s1.remote_port != s2.remote_port
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_active(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
assert manager.list_active() == []
|
||||||
|
|
||||||
|
manager.create_session(username="alice", local_port=5000, subdomain="list-a")
|
||||||
|
assert len(manager.list_active()) == 1
|
||||||
|
|
||||||
|
manager.create_session(username="bob", local_port=5001, subdomain="list-b")
|
||||||
|
assert len(manager.list_active()) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_by_user(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
manager.create_session(username="alice", local_port=6000, subdomain="usr-a")
|
||||||
|
manager.create_session(username="alice", local_port=6001, subdomain="usr-b")
|
||||||
|
manager.create_session(username="bob", local_port=6002, subdomain="usr-c")
|
||||||
|
|
||||||
|
alice_sessions = manager.list_by_user("alice")
|
||||||
|
assert len(alice_sessions) == 2
|
||||||
|
|
||||||
|
bob_sessions = manager.list_by_user("bob")
|
||||||
|
assert len(bob_sessions) == 1
|
||||||
|
|
||||||
|
unknown = manager.list_by_user("nobody")
|
||||||
|
assert len(unknown) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_active_count(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
assert manager.user_active_count("alice") == 0
|
||||||
|
|
||||||
|
manager.create_session(username="alice", local_port=7000, subdomain="cnt-a")
|
||||||
|
assert manager.user_active_count("alice") == 1
|
||||||
|
|
||||||
|
manager.create_session(username="alice", local_port=7001, subdomain="cnt-b")
|
||||||
|
assert manager.user_active_count("alice") == 2
|
||||||
|
|
||||||
|
manager.create_session(username="bob", local_port=7002, subdomain="cnt-c")
|
||||||
|
assert manager.user_active_count("alice") == 2
|
||||||
|
assert manager.user_active_count("bob") == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_end_session(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
session = manager.create_session(username="alice", local_port=8000, subdomain="end-a")
|
||||||
|
assert session is not None
|
||||||
|
assert session.ended_at is None
|
||||||
|
|
||||||
|
ended = manager.end_session(session.session_id)
|
||||||
|
assert ended is not None
|
||||||
|
assert ended.ended_at is not None
|
||||||
|
|
||||||
|
assert len(manager.list_active()) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_end_session_releases_port(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
s1 = manager.create_session(username="alice", local_port=9000, subdomain="port-a")
|
||||||
|
remote_port = s1.remote_port
|
||||||
|
manager.end_session(s1.session_id)
|
||||||
|
|
||||||
|
s2 = manager.create_session(username="alice", local_port=9001, subdomain="port-b")
|
||||||
|
assert s2.remote_port == remote_port, "Released port must be reused"
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_session(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
session = manager.create_session(username="alice", local_port=10000, subdomain="get-a")
|
||||||
|
fetched = manager.get(session.session_id)
|
||||||
|
assert fetched is not None
|
||||||
|
assert fetched.session_id == session.session_id
|
||||||
|
|
||||||
|
assert manager.get("nonexistent") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_record_traffic(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
session = manager.create_session(username="alice", local_port=11000, subdomain="traffic-a")
|
||||||
|
|
||||||
|
manager.record_traffic(session.session_id, bytes_in=100, bytes_out=200, requests_count=1)
|
||||||
|
s = manager.get(session.session_id)
|
||||||
|
assert s.bytes_in == 100
|
||||||
|
assert s.bytes_out == 200
|
||||||
|
assert s.requests_count == 1
|
||||||
|
|
||||||
|
manager.record_traffic(session.session_id, bytes_in=50, bytes_out=75, requests_count=2)
|
||||||
|
s = manager.get(session.session_id)
|
||||||
|
assert s.bytes_in == 150
|
||||||
|
assert s.bytes_out == 275
|
||||||
|
assert s.requests_count == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_record_traffic_nonexistent_session(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
result = manager.record_traffic("nonexistent", bytes_in=10)
|
||||||
|
assert result is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_rate_limit_enforced(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
for i in range(TUNNEL_RATE_LIMIT_PER_USER):
|
||||||
|
s = manager.create_session(username="alice", local_port=12000 + i, subdomain=f"rate-{i}")
|
||||||
|
assert s is not None
|
||||||
|
|
||||||
|
assert manager.user_active_count("alice") == TUNNEL_RATE_LIMIT_PER_USER
|
||||||
|
overflow = manager.create_session(username="alice", local_port=13000, subdomain="rate-overflow")
|
||||||
|
assert overflow is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_bandwidth_limit_not_exceeded(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
session = manager.create_session(username="alice", local_port=14000, subdomain="bw-a")
|
||||||
|
|
||||||
|
with patch("devplacepy.services.tunnel.session_manager.TUNNEL_MAX_BANDWIDTH_PER_CLIENT", 1000):
|
||||||
|
result = manager.record_traffic(session.session_id, bytes_out=500)
|
||||||
|
assert result is True
|
||||||
|
assert session.bandwidth_exceeded is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_bandwidth_limit_exceeded(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
session = manager.create_session(username="alice", local_port=15000, subdomain="bw-b")
|
||||||
|
|
||||||
|
with patch("devplacepy.services.tunnel.session_manager.TUNNEL_MAX_BANDWIDTH_PER_CLIENT", 1000):
|
||||||
|
result = manager.record_traffic(session.session_id, bytes_out=1500)
|
||||||
|
assert result is False
|
||||||
|
assert session.bandwidth_exceeded is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_bandwidth_limit_zero_disabled(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
session = manager.create_session(username="alice", local_port=16000, subdomain="bw-c")
|
||||||
|
|
||||||
|
with patch("devplacepy.services.tunnel.session_manager.TUNNEL_MAX_BANDWIDTH_PER_CLIENT", 0):
|
||||||
|
result = manager.record_traffic(session.session_id, bytes_out=99999999)
|
||||||
|
assert result is True
|
||||||
|
assert session.bandwidth_exceeded is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_port_exhaustion(local_db):
|
||||||
|
manager = SessionManager()
|
||||||
|
manager._port_range = (50000, 50001)
|
||||||
|
|
||||||
|
s1 = manager.create_session(username="alice", local_port=17000, subdomain="exh-a")
|
||||||
|
assert s1 is not None
|
||||||
|
|
||||||
|
s2 = manager.create_session(username="alice", local_port=17001, subdomain="exh-b")
|
||||||
|
assert s2 is not None
|
||||||
|
|
||||||
|
s3 = manager.create_session(username="alice", local_port=17002, subdomain="exh-c")
|
||||||
|
assert s3 is None
|
||||||
95
tests/unit/services/tunnel/test_ssh_auth.py
Normal file
95
tests/unit/services/tunnel/test_ssh_auth.py
Normal file
@ -0,0 +1,95 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
import time
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from devplacepy.services.tunnel.ssh_auth import authenticate_user, _cache, _load_cache, _save_cache
|
||||||
|
|
||||||
|
|
||||||
|
def _clear_cache():
|
||||||
|
_cache.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def test_auth_cache_hits_valid(local_db):
|
||||||
|
_clear_cache()
|
||||||
|
key = "test-user:valid-pass"
|
||||||
|
future = time.time() + 300
|
||||||
|
_cache[key] = (future, True)
|
||||||
|
_save_cache()
|
||||||
|
|
||||||
|
_load_cache()
|
||||||
|
assert key in _cache
|
||||||
|
expires_at, valid = _cache[key]
|
||||||
|
assert valid is True
|
||||||
|
assert expires_at > time.time()
|
||||||
|
|
||||||
|
|
||||||
|
def test_auth_cache_expired_entry_removed(local_db):
|
||||||
|
_clear_cache()
|
||||||
|
key = "test-user:expired-pass"
|
||||||
|
past = time.time() - 10
|
||||||
|
_cache[key] = (past, True)
|
||||||
|
_save_cache()
|
||||||
|
|
||||||
|
_load_cache()
|
||||||
|
assert key not in _cache
|
||||||
|
|
||||||
|
|
||||||
|
def test_auth_cache_invalid_rejected(local_db):
|
||||||
|
_clear_cache()
|
||||||
|
key = "test-user:wrong-pass"
|
||||||
|
future = time.time() + 300
|
||||||
|
_cache[key] = (future, False)
|
||||||
|
_save_cache()
|
||||||
|
|
||||||
|
_load_cache()
|
||||||
|
assert key in _cache
|
||||||
|
_, valid = _cache[key]
|
||||||
|
assert valid is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_authenticate_user_invalid_credentials(local_db):
|
||||||
|
_clear_cache()
|
||||||
|
|
||||||
|
with patch("devplacepy.services.tunnel.ssh_auth.TUNNEL_AUTH_API_URL", "http://localhost:99999/api/tunnel/auth-check"):
|
||||||
|
result = await authenticate_user("nobody", "wrong-password")
|
||||||
|
assert result is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_authenticate_user_wrong_password_delays(local_db):
|
||||||
|
_clear_cache()
|
||||||
|
|
||||||
|
with patch("devplacepy.services.tunnel.ssh_auth.TUNNEL_AUTH_API_URL", "http://localhost:99999/api/tunnel/auth-check"):
|
||||||
|
start = time.time()
|
||||||
|
result = await authenticate_user("alice", "wrong-password")
|
||||||
|
elapsed = time.time() - start
|
||||||
|
assert result is False
|
||||||
|
assert elapsed >= 3.0, "Wrong password response must delay at least 3 seconds"
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_persistence(local_db):
|
||||||
|
_clear_cache()
|
||||||
|
key = "persist-user:pass123"
|
||||||
|
future = time.time() + 300
|
||||||
|
_cache[key] = (future, True)
|
||||||
|
_save_cache()
|
||||||
|
|
||||||
|
_cache.clear()
|
||||||
|
_load_cache()
|
||||||
|
if key in _cache:
|
||||||
|
_, valid = _cache[key]
|
||||||
|
assert valid is True
|
||||||
|
else:
|
||||||
|
assert key not in _cache
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_ttl_enforced(local_db):
|
||||||
|
_clear_cache()
|
||||||
|
key = "ttl-user:pass456"
|
||||||
|
past = time.time() - 1
|
||||||
|
_cache[key] = (past, True)
|
||||||
|
assert _cache[key][0] < time.time()
|
||||||
88
tests/unit/services/tunnel/test_subdomain_registry.py
Normal file
88
tests/unit/services/tunnel/test_subdomain_registry.py
Normal file
@ -0,0 +1,88 @@
|
|||||||
|
# retoor <retoor@molodetz.nl>
|
||||||
|
|
||||||
|
from devplacepy.services.tunnel.subdomain_registry import SubdomainRegistry
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_and_release(local_db):
|
||||||
|
registry = SubdomainRegistry()
|
||||||
|
sub = "my-app"
|
||||||
|
session_id = "session-1"
|
||||||
|
|
||||||
|
assert registry.claim(sub, session_id) is True
|
||||||
|
assert registry.claim(sub, "session-2") is False, "Duplicate claim must fail"
|
||||||
|
|
||||||
|
assert registry.resolve(sub) == session_id
|
||||||
|
assert registry.resolve("unknown") is None
|
||||||
|
|
||||||
|
released = registry.release(session_id)
|
||||||
|
assert released == sub
|
||||||
|
assert registry.resolve(sub) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_unknown(local_db):
|
||||||
|
registry = SubdomainRegistry()
|
||||||
|
result = registry.release("nonexistent-session")
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_twice_different_session(local_db):
|
||||||
|
registry = SubdomainRegistry()
|
||||||
|
sub = "shared-app"
|
||||||
|
|
||||||
|
assert registry.claim(sub, "session-a") is True
|
||||||
|
assert registry.claim(sub, "session-b") is False
|
||||||
|
|
||||||
|
registry.release("session-a")
|
||||||
|
assert registry.claim(sub, "session-b") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_twice_same_session(local_db):
|
||||||
|
registry = SubdomainRegistry()
|
||||||
|
sub = "retry-app"
|
||||||
|
session_id = "session-retry"
|
||||||
|
|
||||||
|
assert registry.claim(sub, session_id) is True
|
||||||
|
result = registry.claim(sub, session_id)
|
||||||
|
assert result is False, "Claiming the same subdomain again must fail even for same session"
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_then_claim(local_db):
|
||||||
|
registry = SubdomainRegistry()
|
||||||
|
sub = "cyclic-app"
|
||||||
|
session_id = "session-cycle"
|
||||||
|
|
||||||
|
assert registry.claim(sub, session_id) is True
|
||||||
|
registry.release(session_id)
|
||||||
|
|
||||||
|
assert registry.claim(sub, "session-cycle-2") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_subdomain_uniqueness_enforced(local_db):
|
||||||
|
registry = SubdomainRegistry()
|
||||||
|
assert registry.claim("alpha", "s1") is True
|
||||||
|
assert registry.claim("beta", "s2") is True
|
||||||
|
assert registry.claim("alpha", "s3") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_idempotent_release(local_db):
|
||||||
|
registry = SubdomainRegistry()
|
||||||
|
assert registry.claim("z-app", "sz1") is True
|
||||||
|
assert registry.release("sz1") == "z-app"
|
||||||
|
assert registry.release("sz1") is None
|
||||||
|
assert registry.claim("z-app", "sz2") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_empty_registry(local_db):
|
||||||
|
registry = SubdomainRegistry()
|
||||||
|
assert registry.resolve("anything") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_session_to_sub_mapping(local_db):
|
||||||
|
registry = SubdomainRegistry()
|
||||||
|
sub = "mapped-app"
|
||||||
|
session_id = "session-mapped"
|
||||||
|
|
||||||
|
registry.claim(sub, session_id)
|
||||||
|
assert registry.resolve(sub) == session_id
|
||||||
|
registry.release(session_id)
|
||||||
|
assert registry.resolve(sub) is None
|
||||||
Loading…
Reference in New Issue
Block a user