368 lines
11 KiB
Python
368 lines
11 KiB
Python
"""Shared harness for the audit/ checks. Spec: audit/README.md.
|
|||
|
|
|
||
|
|
Every check is a reproduce.py that imports this module, starts a scratch
|
||
|
|
server (or none), probes it over a real socket and ends in exactly one of
|
||
|
|
verdict_pass / verdict_fail / verdict_skip. Everything here has a hard cap:
|
||
|
|
check wall time (WATCHDOG_SECS), bytes sent, frames sent, read waits.
|
||
|
|
Python 3 stdlib only. No third-party dependency.
|
||
|
|
"""
|
||
|
|
import atexit
|
||
|
|
import os
|
||
|
|
import select
|
||
|
|
import shutil
|
||
|
|
import signal
|
||
|
|
import socket
|
||
|
|
import struct
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
import time
|
||
|
|
|
||
|
|
WATCHDOG_SECS = 90 # hard ceiling per check, see audit/README.md
|
||
|
|
READ_SLICE_SECS = 0.2 # select granularity
|
||
|
|
MAX_SEND_BYTES = 64 * 1024 * 1024 # no check may push more than this
|
||
|
|
|
||
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||
|
|
BUILD = os.environ.get("MOLO_BUILD", os.path.join(ROOT, "build"))
|
||
|
|
INTEROP = os.environ.get("MOLO_INTEROP", os.path.join(BUILD, "bin", "runtime_interop_server"))
|
||
|
|
LIVING = os.environ.get("MOLO_LIVING", os.path.join(BUILD, "example", "living-server"))
|
||
|
|
MOLO = os.environ.get("MOLO_BIN", os.path.join(BUILD, "molo"))
|
||
|
|
|
||
|
|
_cleanups = []
|
||
|
|
|
||
|
|
|
||
|
|
def _run_cleanups():
|
||
|
|
while _cleanups:
|
||
|
|
try:
|
||
|
|
_cleanups.pop()()
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
atexit.register(_run_cleanups)
|
||
|
|
|
||
|
|
|
||
|
|
def _finish(tag, msg, code):
|
||
|
|
print("%s: %s" % (tag, msg))
|
||
|
|
sys.stdout.flush()
|
||
|
|
_run_cleanups()
|
||
|
|
os._exit(code)
|
||
|
|
|
||
|
|
|
||
|
|
def verdict_pass(msg):
|
||
|
|
_finish("PASS", msg, 0)
|
||
|
|
|
||
|
|
|
||
|
|
def verdict_skip(msg):
|
||
|
|
_finish("SKIP", msg, 0)
|
||
|
|
|
||
|
|
|
||
|
|
def verdict_fail(msg):
|
||
|
|
_finish("FAIL", msg, 1)
|
||
|
|
|
||
|
|
|
||
|
|
def _watchdog(signum, frame):
|
||
|
|
verdict_fail("check exceeded its %ds budget (server hang or unbounded work)" % WATCHDOG_SECS)
|
||
|
|
|
||
|
|
|
||
|
|
def arm_watchdog(secs=WATCHDOG_SECS):
|
||
|
|
signal.signal(signal.SIGALRM, _watchdog)
|
||
|
|
signal.alarm(secs)
|
||
|
|
|
||
|
|
|
||
|
|
def scratch_dir():
|
||
|
|
d = tempfile.mkdtemp(prefix="molo-audit-")
|
||
|
|
_cleanups.append(lambda: shutil.rmtree(d, ignore_errors=True))
|
||
|
|
return d
|
||
|
|
|
||
|
|
|
||
|
|
class Server:
|
||
|
|
"""A scratch server process on 127.0.0.1, killed on exit."""
|
||
|
|
|
||
|
|
def __init__(self, argv, ready_prefix="READY", ready_secs=10):
|
||
|
|
self.proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
||
|
|
text=True, cwd=argv_cwd(argv))
|
||
|
|
_cleanups.append(self.stop)
|
||
|
|
self.port = None
|
||
|
|
deadline = time.time() + ready_secs
|
||
|
|
while time.time() < deadline:
|
||
|
|
line = self.proc.stdout.readline()
|
||
|
|
if not line:
|
||
|
|
break
|
||
|
|
if line.startswith(ready_prefix) or "http://127.0.0.1:" in line:
|
||
|
|
self.port = int(line.strip().split(":")[-1].split()[-1])
|
||
|
|
break
|
||
|
|
if self.port is None:
|
||
|
|
verdict_skip("server did not report READY: %s" % " ".join(argv))
|
||
|
|
|
||
|
|
def rss_kb(self):
|
||
|
|
try:
|
||
|
|
with open("/proc/%d/status" % self.proc.pid) as f:
|
||
|
|
for ln in f:
|
||
|
|
if ln.startswith("VmRSS:"):
|
||
|
|
return int(ln.split()[1])
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
return -1
|
||
|
|
|
||
|
|
def alive(self):
|
||
|
|
return self.proc.poll() is None
|
||
|
|
|
||
|
|
def stop(self):
|
||
|
|
if self.proc.poll() is None:
|
||
|
|
self.proc.kill()
|
||
|
|
try:
|
||
|
|
self.proc.wait(timeout=3)
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
def argv_cwd(argv):
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def start_interop(www_files=None):
|
||
|
|
"""Start runtime_interop_server on an ephemeral port with a scratch www dir.
|
||
|
|
Returns (server, tmpdir). tmpdir/www is served, tmpdir/secret.txt is outside it."""
|
||
|
|
if not os.path.exists(INTEROP):
|
||
|
|
verdict_skip("%s not built (run make)" % INTEROP)
|
||
|
|
tmp = scratch_dir()
|
||
|
|
www = os.path.join(tmp, "www")
|
||
|
|
os.makedirs(www)
|
||
|
|
with open(os.path.join(www, "hello.txt"), "w") as f:
|
||
|
|
f.write("hello\n")
|
||
|
|
with open(os.path.join(www, "big.bin"), "wb") as f:
|
||
|
|
f.write(os.urandom(200 * 1024))
|
||
|
|
with open(os.path.join(tmp, "secret.txt"), "w") as f:
|
||
|
|
f.write("MOLO-AUDIT-SECRET-OUTSIDE-WWW\n")
|
||
|
|
for name, data in (www_files or {}).items():
|
||
|
|
with open(os.path.join(www, name), "wb") as f:
|
||
|
|
f.write(data)
|
||
|
|
return Server([INTEROP, "0", www]), tmp
|
||
|
|
|
||
|
|
|
||
|
|
def start_living():
|
||
|
|
if not os.path.exists(LIVING):
|
||
|
|
verdict_skip("%s not built (run make)" % LIVING)
|
||
|
|
s = socket.socket()
|
||
|
|
s.bind(("127.0.0.1", 0))
|
||
|
|
port = s.getsockname()[1]
|
||
|
|
s.close()
|
||
|
|
srv = Server([LIVING, str(port)], ready_prefix="living example")
|
||
|
|
return srv
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------- HTTP/1
|
||
|
|
|
||
|
|
|
||
|
|
def connect(port, timeout=5):
|
||
|
|
s = socket.create_connection(("127.0.0.1", port), timeout=timeout)
|
||
|
|
return s
|
||
|
|
|
||
|
|
|
||
|
|
def recv_all(s, secs=1.0, cap=4 * 1024 * 1024):
|
||
|
|
"""Read until the peer closes, the cap is hit, or `secs` pass without data."""
|
||
|
|
buf = b""
|
||
|
|
s.setblocking(False)
|
||
|
|
end = time.time() + secs
|
||
|
|
closed = False
|
||
|
|
while time.time() < end and len(buf) < cap:
|
||
|
|
r, _, _ = select.select([s], [], [], READ_SLICE_SECS)
|
||
|
|
if r:
|
||
|
|
try:
|
||
|
|
d = s.recv(65536)
|
||
|
|
except (ConnectionResetError, BrokenPipeError):
|
||
|
|
closed = True
|
||
|
|
break
|
||
|
|
if not d:
|
||
|
|
closed = True
|
||
|
|
break
|
||
|
|
buf += d
|
||
|
|
end = time.time() + secs
|
||
|
|
return buf, closed
|
||
|
|
|
||
|
|
|
||
|
|
def h1(port, data, secs=1.0):
|
||
|
|
"""Send raw bytes, return (response bytes, closed_by_peer)."""
|
||
|
|
s = connect(port)
|
||
|
|
try:
|
||
|
|
s.sendall(data)
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
out = recv_all(s, secs)
|
||
|
|
s.close()
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def status_lines(resp):
|
||
|
|
"""All 'HTTP/1.1 NNN' status codes found at line starts, in order."""
|
||
|
|
codes = []
|
||
|
|
for ln in resp.split(b"\r\n"):
|
||
|
|
if ln.startswith(b"HTTP/1.") and len(ln) >= 12:
|
||
|
|
try:
|
||
|
|
codes.append(int(ln[9:12]))
|
||
|
|
except ValueError:
|
||
|
|
pass
|
||
|
|
return codes
|
||
|
|
|
||
|
|
|
||
|
|
def alive_h1(port):
|
||
|
|
try:
|
||
|
|
resp, _ = h1(port, b"GET /echo HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n", 1.5)
|
||
|
|
return 200 in status_lines(resp)
|
||
|
|
except OSError:
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------- HTTP/2
|
||
|
|
|
||
|
|
PREFACE = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
|
||
|
|
T_DATA, T_HEADERS, T_PRIORITY, T_RST, T_SETTINGS, T_PUSH, T_PING, T_GOAWAY, T_WINDOW, T_CONT = range(10)
|
||
|
|
F_ACK = 0x1
|
||
|
|
F_END_STREAM = 0x1
|
||
|
|
F_END_HEADERS = 0x4
|
||
|
|
|
||
|
|
|
||
|
|
def frame(ftype, flags, sid, payload=b""):
|
||
|
|
n = len(payload)
|
||
|
|
return struct.pack(">BHBBI", (n >> 16) & 0xFF, n & 0xFFFF, ftype, flags, sid & 0x7FFFFFFF) + payload
|
||
|
|
|
||
|
|
|
||
|
|
def settings(pairs=()):
|
||
|
|
return frame(T_SETTINGS, 0, 0, b"".join(struct.pack(">HI", k, v) for k, v in pairs))
|
||
|
|
|
||
|
|
|
||
|
|
def _lit(idx, value):
|
||
|
|
v = value.encode() if isinstance(value, str) else value
|
||
|
|
return bytes([idx]) + bytes([len(v)]) + v if len(v) < 127 else None
|
||
|
|
|
||
|
|
|
||
|
|
def hpack_request(method="GET", path="/", authority="x", extra=()):
|
||
|
|
"""Header block using only static-table refs and plain literals."""
|
||
|
|
block = b"\x82" if method == "GET" else b"\x83" # :method GET / POST
|
||
|
|
block += b"\x86" # :scheme http
|
||
|
|
block += _lit(0x04, path) # :path (literal, name idx 4)
|
||
|
|
block += _lit(0x01, authority) # :authority (name idx 1)
|
||
|
|
for name, value in extra:
|
||
|
|
n, v = name.encode(), value.encode()
|
||
|
|
block += b"\x00" + bytes([len(n)]) + n + bytes([len(v)]) + v
|
||
|
|
return block
|
||
|
|
|
||
|
|
|
||
|
|
def hpack_int(prefix_bits, first_byte_flags, value):
|
||
|
|
mx = (1 << prefix_bits) - 1
|
||
|
|
if value < mx:
|
||
|
|
return bytes([first_byte_flags | value])
|
||
|
|
out = bytearray([first_byte_flags | mx])
|
||
|
|
value -= mx
|
||
|
|
while value >= 128:
|
||
|
|
out.append((value & 0x7F) | 0x80)
|
||
|
|
value >>= 7
|
||
|
|
out.append(value)
|
||
|
|
return bytes(out)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_frames(buf):
|
||
|
|
frames = []
|
||
|
|
i = 0
|
||
|
|
while len(buf) - i >= 9:
|
||
|
|
n = (buf[i] << 16) | (buf[i + 1] << 8) | buf[i + 2]
|
||
|
|
if len(buf) - i - 9 < n:
|
||
|
|
break
|
||
|
|
ftype, flags = buf[i + 3], buf[i + 4]
|
||
|
|
sid = struct.unpack(">I", buf[i + 5:i + 9])[0] & 0x7FFFFFFF
|
||
|
|
frames.append((ftype, flags, sid, buf[i + 9:i + 9 + n]))
|
||
|
|
i += 9 + n
|
||
|
|
return frames, buf[i:]
|
||
|
|
|
||
|
|
|
||
|
|
class H2:
|
||
|
|
"""Prior-knowledge h2c client. Tracks every frame the server sends back."""
|
||
|
|
|
||
|
|
def __init__(self, port, initial_settings=()):
|
||
|
|
self.s = connect(port)
|
||
|
|
self.s.setblocking(False)
|
||
|
|
self.rbuf = b""
|
||
|
|
self.frames = []
|
||
|
|
self.closed = False
|
||
|
|
self.goaway = None # error code
|
||
|
|
self.sent = 0
|
||
|
|
self.sent_bytes = 0
|
||
|
|
self.raw(PREFACE + settings(initial_settings))
|
||
|
|
|
||
|
|
def _drain_read(self, secs=0.0):
|
||
|
|
try:
|
||
|
|
r, _, _ = select.select([self.s], [], [], secs)
|
||
|
|
while r:
|
||
|
|
d = self.s.recv(262144)
|
||
|
|
if not d:
|
||
|
|
self.closed = True
|
||
|
|
break
|
||
|
|
self.rbuf += d
|
||
|
|
r, _, _ = select.select([self.s], [], [], 0)
|
||
|
|
except (ConnectionResetError, BrokenPipeError, OSError):
|
||
|
|
self.closed = True
|
||
|
|
fr, self.rbuf = parse_frames(self.rbuf)
|
||
|
|
for f in fr:
|
||
|
|
self.frames.append(f)
|
||
|
|
if f[0] == T_GOAWAY and self.goaway is None:
|
||
|
|
self.goaway = struct.unpack(">I", f[3][4:8])[0] if len(f[3]) >= 8 else 0
|
||
|
|
return fr
|
||
|
|
|
||
|
|
def raw(self, data):
|
||
|
|
"""Blocking-ish send that keeps reading so a full buffer cannot deadlock."""
|
||
|
|
if self.sent_bytes + len(data) > MAX_SEND_BYTES:
|
||
|
|
verdict_fail("harness cap: refusing to send more than %d bytes" % MAX_SEND_BYTES)
|
||
|
|
view = memoryview(data)
|
||
|
|
deadline = time.time() + 20
|
||
|
|
while view and not self.closed and time.time() < deadline:
|
||
|
|
self._drain_read(0)
|
||
|
|
try:
|
||
|
|
_, w, _ = select.select([], [self.s], [], 0.05)
|
||
|
|
if w:
|
||
|
|
n = self.s.send(view[:65536])
|
||
|
|
view = view[n:]
|
||
|
|
self.sent_bytes += n
|
||
|
|
except (BlockingIOError, InterruptedError):
|
||
|
|
continue
|
||
|
|
except (ConnectionResetError, BrokenPipeError, OSError):
|
||
|
|
self.closed = True
|
||
|
|
return not view
|
||
|
|
|
||
|
|
def cut(self):
|
||
|
|
"""True once the server tore the connection down or sent GOAWAY."""
|
||
|
|
return self.closed or self.goaway is not None
|
||
|
|
|
||
|
|
def settle(self, secs=1.0):
|
||
|
|
end = time.time() + secs
|
||
|
|
while time.time() < end and not self.closed:
|
||
|
|
self._drain_read(0.1)
|
||
|
|
|
||
|
|
def count(self, ftype, flags_mask=None):
|
||
|
|
return sum(1 for f in self.frames
|
||
|
|
if f[0] == ftype and (flags_mask is None or f[1] & flags_mask))
|
||
|
|
|
||
|
|
def status_of(self, sid):
|
||
|
|
""":status first byte guess for a stream: 0x88 => 200 indexed."""
|
||
|
|
for f in self.frames:
|
||
|
|
if f[0] == T_HEADERS and f[2] == sid:
|
||
|
|
return f[3]
|
||
|
|
return None
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
try:
|
||
|
|
self.s.close()
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
def flood(h2, make_frame, count, batch=100):
|
||
|
|
"""Send `count` frames in batches, stop early when the server cuts. Returns frames sent."""
|
||
|
|
sent = 0
|
||
|
|
while sent < count and not h2.cut():
|
||
|
|
n = min(batch, count - sent)
|
||
|
|
h2.raw(b"".join(make_frame(sent + i) for i in range(n)))
|
||
|
|
sent += n
|
||
|
|
h2.settle(1.0)
|
||
|
|
return sent
|