Files
moloapp/audit/lib/molo_audit.py
T

368 lines
11 KiB
Python

"""Shared harness for the audit/ checks. Spec: audit/README.md.
Every check is a reproduce.py that imports this module, starts a scratch
server (or none), probes it over a real socket and ends in exactly one of
verdict_pass / verdict_fail / verdict_skip. Everything here has a hard cap:
check wall time (WATCHDOG_SECS), bytes sent, frames sent, read waits.
Python 3 stdlib only. No third-party dependency.
"""
import atexit
import os
import select
import shutil
import signal
import socket
import struct
import subprocess
import sys
import tempfile
import time
WATCHDOG_SECS = 90 # hard ceiling per check, see audit/README.md
READ_SLICE_SECS = 0.2 # select granularity
MAX_SEND_BYTES = 64 * 1024 * 1024 # no check may push more than this
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
BUILD = os.environ.get("MOLO_BUILD", os.path.join(ROOT, "build"))
INTEROP = os.environ.get("MOLO_INTEROP", os.path.join(BUILD, "bin", "runtime_interop_server"))
LIVING = os.environ.get("MOLO_LIVING", os.path.join(BUILD, "example", "living-server"))
MOLO = os.environ.get("MOLO_BIN", os.path.join(BUILD, "molo"))
_cleanups = []
def _run_cleanups():
while _cleanups:
try:
_cleanups.pop()()
except Exception:
pass
atexit.register(_run_cleanups)
def _finish(tag, msg, code):
print("%s: %s" % (tag, msg))
sys.stdout.flush()
_run_cleanups()
os._exit(code)
def verdict_pass(msg):
_finish("PASS", msg, 0)
def verdict_skip(msg):
_finish("SKIP", msg, 0)
def verdict_fail(msg):
_finish("FAIL", msg, 1)
def _watchdog(signum, frame):
verdict_fail("check exceeded its %ds budget (server hang or unbounded work)" % WATCHDOG_SECS)
def arm_watchdog(secs=WATCHDOG_SECS):
signal.signal(signal.SIGALRM, _watchdog)
signal.alarm(secs)
def scratch_dir():
d = tempfile.mkdtemp(prefix="molo-audit-")
_cleanups.append(lambda: shutil.rmtree(d, ignore_errors=True))
return d
class Server:
"""A scratch server process on 127.0.0.1, killed on exit."""
def __init__(self, argv, ready_prefix="READY", ready_secs=10):
self.proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
text=True, cwd=argv_cwd(argv))
_cleanups.append(self.stop)
self.port = None
deadline = time.time() + ready_secs
while time.time() < deadline:
line = self.proc.stdout.readline()
if not line:
break
if line.startswith(ready_prefix) or "http://127.0.0.1:" in line:
self.port = int(line.strip().split(":")[-1].split()[-1])
break
if self.port is None:
verdict_skip("server did not report READY: %s" % " ".join(argv))
def rss_kb(self):
try:
with open("/proc/%d/status" % self.proc.pid) as f:
for ln in f:
if ln.startswith("VmRSS:"):
return int(ln.split()[1])
except OSError:
pass
return -1
def alive(self):
return self.proc.poll() is None
def stop(self):
if self.proc.poll() is None:
self.proc.kill()
try:
self.proc.wait(timeout=3)
except Exception:
pass
def argv_cwd(argv):
return None
def start_interop(www_files=None):
"""Start runtime_interop_server on an ephemeral port with a scratch www dir.
Returns (server, tmpdir). tmpdir/www is served, tmpdir/secret.txt is outside it."""
if not os.path.exists(INTEROP):
verdict_skip("%s not built (run make)" % INTEROP)
tmp = scratch_dir()
www = os.path.join(tmp, "www")
os.makedirs(www)
with open(os.path.join(www, "hello.txt"), "w") as f:
f.write("hello\n")
with open(os.path.join(www, "big.bin"), "wb") as f:
f.write(os.urandom(200 * 1024))
with open(os.path.join(tmp, "secret.txt"), "w") as f:
f.write("MOLO-AUDIT-SECRET-OUTSIDE-WWW\n")
for name, data in (www_files or {}).items():
with open(os.path.join(www, name), "wb") as f:
f.write(data)
return Server([INTEROP, "0", www]), tmp
def start_living():
if not os.path.exists(LIVING):
verdict_skip("%s not built (run make)" % LIVING)
s = socket.socket()
s.bind(("127.0.0.1", 0))
port = s.getsockname()[1]
s.close()
srv = Server([LIVING, str(port)], ready_prefix="living example")
return srv
# ---------------------------------------------------------------- HTTP/1
def connect(port, timeout=5):
s = socket.create_connection(("127.0.0.1", port), timeout=timeout)
return s
def recv_all(s, secs=1.0, cap=4 * 1024 * 1024):
"""Read until the peer closes, the cap is hit, or `secs` pass without data."""
buf = b""
s.setblocking(False)
end = time.time() + secs
closed = False
while time.time() < end and len(buf) < cap:
r, _, _ = select.select([s], [], [], READ_SLICE_SECS)
if r:
try:
d = s.recv(65536)
except (ConnectionResetError, BrokenPipeError):
closed = True
break
if not d:
closed = True
break
buf += d
end = time.time() + secs
return buf, closed
def h1(port, data, secs=1.0):
"""Send raw bytes, return (response bytes, closed_by_peer)."""
s = connect(port)
try:
s.sendall(data)
except OSError:
pass
out = recv_all(s, secs)
s.close()
return out
def status_lines(resp):
"""All 'HTTP/1.1 NNN' status codes found at line starts, in order."""
codes = []
for ln in resp.split(b"\r\n"):
if ln.startswith(b"HTTP/1.") and len(ln) >= 12:
try:
codes.append(int(ln[9:12]))
except ValueError:
pass
return codes
def alive_h1(port):
try:
resp, _ = h1(port, b"GET /echo HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n", 1.5)
return 200 in status_lines(resp)
except OSError:
return False
# ---------------------------------------------------------------- HTTP/2
PREFACE = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
T_DATA, T_HEADERS, T_PRIORITY, T_RST, T_SETTINGS, T_PUSH, T_PING, T_GOAWAY, T_WINDOW, T_CONT = range(10)
F_ACK = 0x1
F_END_STREAM = 0x1
F_END_HEADERS = 0x4
def frame(ftype, flags, sid, payload=b""):
n = len(payload)
return struct.pack(">BHBBI", (n >> 16) & 0xFF, n & 0xFFFF, ftype, flags, sid & 0x7FFFFFFF) + payload
def settings(pairs=()):
return frame(T_SETTINGS, 0, 0, b"".join(struct.pack(">HI", k, v) for k, v in pairs))
def _lit(idx, value):
v = value.encode() if isinstance(value, str) else value
return bytes([idx]) + bytes([len(v)]) + v if len(v) < 127 else None
def hpack_request(method="GET", path="/", authority="x", extra=()):
"""Header block using only static-table refs and plain literals."""
block = b"\x82" if method == "GET" else b"\x83" # :method GET / POST
block += b"\x86" # :scheme http
block += _lit(0x04, path) # :path (literal, name idx 4)
block += _lit(0x01, authority) # :authority (name idx 1)
for name, value in extra:
n, v = name.encode(), value.encode()
block += b"\x00" + bytes([len(n)]) + n + bytes([len(v)]) + v
return block
def hpack_int(prefix_bits, first_byte_flags, value):
mx = (1 << prefix_bits) - 1
if value < mx:
return bytes([first_byte_flags | value])
out = bytearray([first_byte_flags | mx])
value -= mx
while value >= 128:
out.append((value & 0x7F) | 0x80)
value >>= 7
out.append(value)
return bytes(out)
def parse_frames(buf):
frames = []
i = 0
while len(buf) - i >= 9:
n = (buf[i] << 16) | (buf[i + 1] << 8) | buf[i + 2]
if len(buf) - i - 9 < n:
break
ftype, flags = buf[i + 3], buf[i + 4]
sid = struct.unpack(">I", buf[i + 5:i + 9])[0] & 0x7FFFFFFF
frames.append((ftype, flags, sid, buf[i + 9:i + 9 + n]))
i += 9 + n
return frames, buf[i:]
class H2:
"""Prior-knowledge h2c client. Tracks every frame the server sends back."""
def __init__(self, port, initial_settings=()):
self.s = connect(port)
self.s.setblocking(False)
self.rbuf = b""
self.frames = []
self.closed = False
self.goaway = None # error code
self.sent = 0
self.sent_bytes = 0
self.raw(PREFACE + settings(initial_settings))
def _drain_read(self, secs=0.0):
try:
r, _, _ = select.select([self.s], [], [], secs)
while r:
d = self.s.recv(262144)
if not d:
self.closed = True
break
self.rbuf += d
r, _, _ = select.select([self.s], [], [], 0)
except (ConnectionResetError, BrokenPipeError, OSError):
self.closed = True
fr, self.rbuf = parse_frames(self.rbuf)
for f in fr:
self.frames.append(f)
if f[0] == T_GOAWAY and self.goaway is None:
self.goaway = struct.unpack(">I", f[3][4:8])[0] if len(f[3]) >= 8 else 0
return fr
def raw(self, data):
"""Blocking-ish send that keeps reading so a full buffer cannot deadlock."""
if self.sent_bytes + len(data) > MAX_SEND_BYTES:
verdict_fail("harness cap: refusing to send more than %d bytes" % MAX_SEND_BYTES)
view = memoryview(data)
deadline = time.time() + 20
while view and not self.closed and time.time() < deadline:
self._drain_read(0)
try:
_, w, _ = select.select([], [self.s], [], 0.05)
if w:
n = self.s.send(view[:65536])
view = view[n:]
self.sent_bytes += n
except (BlockingIOError, InterruptedError):
continue
except (ConnectionResetError, BrokenPipeError, OSError):
self.closed = True
return not view
def cut(self):
"""True once the server tore the connection down or sent GOAWAY."""
return self.closed or self.goaway is not None
def settle(self, secs=1.0):
end = time.time() + secs
while time.time() < end and not self.closed:
self._drain_read(0.1)
def count(self, ftype, flags_mask=None):
return sum(1 for f in self.frames
if f[0] == ftype and (flags_mask is None or f[1] & flags_mask))
def status_of(self, sid):
""":status first byte guess for a stream: 0x88 => 200 indexed."""
for f in self.frames:
if f[0] == T_HEADERS and f[2] == sid:
return f[3]
return None
def close(self):
try:
self.s.close()
except OSError:
pass
def flood(h2, make_frame, count, batch=100):
"""Send `count` frames in batches, stop early when the server cuts. Returns frames sent."""
sent = 0
while sent < count and not h2.cut():
n = min(batch, count - sent)
h2.raw(b"".join(make_frame(sent + i) for i in range(n)))
sent += n
h2.settle(1.0)
return sent