From f49b480587c23818dda46c545a9b90eef7b8a71a Mon Sep 17 00:00:00 2001 From: retoor Date: Fri, 6 Nov 2026 04:58:05 +0100 Subject: [PATCH] feat(workflow): add scan_pushed tool and bring README up to date - tools/scan_pushed.py: pre-push secret scan of a commit range, docs in README - README: document molo doos and version commands, fix scaffold listing, add specs and docs map lines, 18 plans, drop installed AppArmor gap - Verified: example suite 265 passed 0 failed, CLI rows against --help, all referenced files exist --- README.md | 23 ++++++------ tools/scan_pushed.py | 84 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 11 deletions(-) create mode 100755 tools/scan_pushed.py diff --git a/README.md b/README.md index dabbbe7..b6e5944 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,14 @@ Native C++23 web framework: HTTP/1.1 + HTTP/2 runtime, routing, views, validation, templates, ORM (SQLite + Postgres), sessions, realtime, supervisor, clients, observability, multi-tenancy, audit, governance, plus -clients for Google Cloud and the main LLM vendors. One third-party dependency +clients for Google Cloud and the main LLM vendors. One third-party library for TLS (wolfSSL). One CLI binary: `molo`. Current version: see `VERSION` (0.1.0), history in `CHANGELOG.md`. This README states what works today. Gaps are listed at the bottom and in the spec next to the code that will close them. Where README and a spec disagree, the spec wins: `plan.md`, `plans/`, `src/*/SPEC.md`, `reverse.md`, -`example/SPEC.md`. +`example/SPEC.md`, `specs/`. ## Requirements @@ -34,11 +34,12 @@ make cve-audit # vulnerability-class regression checks, see audit/README.md make deploy # restart the living server from build/ on :8080, pid file, health wait make verify # live check of the deployed server (health, metrics, 360px markup, top 10 routes) make clean # rm -rf build +python3 tools/scan_pushed.py # pre-push secret scan of origin/main..HEAD ``` Everything lands under `build/`. Warnings are errors in spirit: fix the cause, never silence them. Per-part targets (`google`, `google-test`, `ai`, -`ai-test`, `data-test`, `cli-test`, ...) also run standalone. +`ai-test`, `data-test`, `cli-test`, `doos`, `doos-test`, ...) also run standalone. ## The living example @@ -182,7 +183,8 @@ primary output only; logs go to stderr. `--format json` is always parseable. | Command | What it does | |---|---| -| `new [--backend postgres\|sqlite] [--force]` | Scaffold a project (Makefile, AGENTS.md, `.env.json`, assets, templates, locales, `git init`). Refuses a non-empty dir without `--force`. | +| `version` | Print the version from `VERSION`. | +| `new [--backend postgres\|sqlite] [--force]` | Scaffold a project (Makefile, AGENTS.md, `.env.json`, `.gitignore`, assets, templates, locales, src, `git init`). Refuses a non-empty dir without `--force`. | | `serve [--port N] [--reload] [--config F]` | Dev server on `0.0.0.0` (default 8000): HTTP/1.1 + h2c on the in-house runtime, one SO_REUSEPORT listener per core, serves `./assets` plus `/healthz /readyz /startupz /metrics`. Exits 1 if the port is taken. `--reload` watches and rebuilds; refused on prod config without `--allow-prod`. | | `build [--embed dir] [--out dir]` | Build, optionally embedding assets first. | | `embed --out f.cpp [--hpp f.hpp] [--namespace ns] [--gzip]` | Turn a static dir into constexpr C++ arrays. Output is byte-identical across runs. | @@ -192,6 +194,7 @@ primary output only; logs go to stderr. `--format json` is always parseable. | `task enqueue\|retry\|cancel\|inspect` | Background task queue (`--id` for inspect). Same wiring caveat. | | `ai new\|continue` | The AI agent, see below. | | `expose serve\|agent\|list\|stats\|expose-key rotate` | Reverse tunnels, see below. | +| `doos list\|get_or_create\|attach\|delete\|apparmor\|version\|image\|davmount` | Rootless containers (spec `src/doos/SPEC.md`); text matches standalone `build/doos/doos`, `--format json` renders JSON. | | `verify [dir]`, `supply-check [dir]`, `release --commit SHA --suite-stamp F` | Offline artifact verification, pin and advisory gate, ordered release gate. | | `rollout start\|abort`, `rollback`, `experiment start\|abort`, `debug snapshot` | Operability: rolling or blue-green rollout (`--strategy --instances --batch --max-unavailable`), chaos experiment (`--blast --duration --max-burn`), debug snapshot. These are one-shot: `abort` and `rollback` report that nothing runs in the process. | @@ -210,7 +213,7 @@ Start a new app: ```sh ./build/molo new shop --backend sqlite -cd shop && ls -A # .env.json AGENTS.md Makefile assets locales src templates +cd shop && ls -A # .env.json .git .gitignore AGENTS.md Makefile assets locales src templates ``` ## The agent: `molo ai` @@ -312,10 +315,12 @@ src/operability deploys, drains, locks, breakers, SLO, chaos src/supplychain SBOM, provenance, pin policy, release attestation src/doos rootless user-level containers (standalone doos plus molo doos) audit/ vulnerability-class regression checks (make cve-audit) -tools/ deploy verification, release and token helpers +tools/ deploy verification, release, token and secret-scan helpers example/ living example (skeleton fw + demo routes, Locust file) tests/ integration tests, mirrors URLs -plans/ 17 detailed plans, index in plans/README.md +specs/ example-site spec (spec-first workflow) +docs/ research notes +plans/ 18 detailed plans, index in plans/README.md ``` Each part ships a `.mk` fragment that the root Makefile includes, and @@ -348,10 +353,6 @@ Measured on this checkout, not aspirational: the binary is built with (`src/cli/cli/cli_app.cpp`). - `molo serve --reload` supervises `./build/molo-app` directly; running the child inside a doos container is a later phase, not wired yet. -- On hosts with `kernel.apparmor_restrict_unprivileged_userns=1` the new - `build/doos/doos`, `build/molo` and `build/doos/test_*` binaries need - AppArmor profiles before they can start containers (one-time setup, see - `src/doos/README.md` troubleshooting; remove this line once installed). - `example/fw/` is a skeleton, not the real framework (its TLS context is still `tls-stub-no-wolfssl`, `example/fw/Tls.cpp`); the swap to `src/` namespaces is incremental. diff --git a/tools/scan_pushed.py b/tools/scan_pushed.py new file mode 100755 index 0000000..502b0e2 --- /dev/null +++ b/tools/scan_pushed.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Scan a commit range for accidentally pushed secrets. + +Usage: + python3 tools/scan_pushed.py [range] # default range: origin/main..HEAD + +Reports added lines matching secret patterns, suspicious filenames, +and an extension histogram. Read-only: never modifies the repo. +Exit 0 always; review HIT lines by hand since test fixtures match too. +""" +import re +import subprocess +import sys + +RANGE = sys.argv[1] if len(sys.argv) > 1 else "origin/main..HEAD" + +PATTERNS = [ + ("private-key", re.compile(r"BEGIN.{0,20}PRIVATE KEY")), + ("github-token", re.compile(r"ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}")), + ("openai-key", re.compile(r"sk-(proj-|live-|test-)?[A-Za-z0-9]{10,}")), + ("aws-key", re.compile(r"AKIA[0-9A-Z]{16}")), + ("gcp-api-key", re.compile(r"AIza[0-9A-Za-z_-]{35}")), + ("slack-token", re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}")), + ("google-oauth", re.compile(r"ya29\.[A-Za-z0-9_-]{10,}")), + ("sendgrid", re.compile(r"SG\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")), + ("bearer", re.compile(r"Bearer [A-Za-z0-9._~+/-]{20,}")), + ("password-assign", re.compile(r"password\s*[:=]\s*['\"][^'\"]{4,}['\"]", re.I)), + ("apikey-assign", re.compile(r"api[_-]?key\s*[:=]\s*['\"][A-Za-z0-9_~.\-]{12,}['\"]", re.I)), + ("secret-assign", re.compile(r"(client_)?secret\s*[:=]\s*['\"][A-Za-z0-9_~.\-]{12,}['\"]", re.I)), + ("token-assign", re.compile(r"token\s*[:=]\s*['\"][A-Za-z0-9_~.\-]{16,}['\"]", re.I)), + ("conn-string", re.compile(r"(postgres|mysql|mongodb)(\+\w+)?://[^/\s:]+:[^/\s@]+@", re.I)), +] + +NAME_RE = re.compile( + r"secret|credential|private|\.pem$|\.key$|\.p12$|\.pfx$|id_rsa|\.db$|\.sqlite|\.pyc$|__pycache__", + re.I, +) + + +def run_git(args): + p = subprocess.run(["git"] + args, capture_output=True, text=True) + if p.returncode != 0: + print(f"git {' '.join(args)} failed: {p.stderr.strip()}", file=sys.stderr) + sys.exit(2) + return p.stdout + + +print(f"range: {RANGE}") +diff = run_git(["diff", RANGE, "-U0"]) + +cur_file = "?" +added = 0 +hits = [] +for line in diff.splitlines(): + if line.startswith("+++ b/"): + cur_file = line[6:] + elif line.startswith("+") and not line.startswith("+++"): + added += 1 + body = line[1:] + for name, rx in PATTERNS: + if rx.search(body): + hits.append((name, cur_file, body[:200])) + +print(f"added lines scanned: {added}") +print(f"pattern hits: {len(hits)}") +for name, f, snippet in hits[:60]: + print(f"HIT [{name}] {f}: {snippet}") +if len(hits) > 60: + print(f"... and {len(hits) - 60} more") + +names = run_git(["diff", "--name-only", RANGE]).splitlines() +print(f"files in range: {len(names)}") +flagged = [n for n in names if NAME_RE.search(n)] +print(f"suspicious filenames: {len(flagged)}") +for n in flagged[:40]: + print(f"NAME: {n}") + +exts = {} +for n in names: + ext = n.rsplit(".", 1)[-1].lower() if "." in n else "(none)" + exts[ext] = exts.get(ext, 0) + 1 +print("extensions:") +for ext, c in sorted(exts.items(), key=lambda kv: -kv[1])[:25]: + print(f" {ext}: {c}")