feat(workflow): add scan_pushed tool and bring README up to date
- tools/scan_pushed.py: pre-push secret scan of a commit range, docs in README - README: document molo doos and version commands, fix scaffold listing, add specs and docs map lines, 18 plans, drop installed AppArmor gap - Verified: example suite 265 passed 0 failed, CLI rows against --help, all referenced files exist
This commit is contained in:
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scan a commit range for accidentally pushed secrets.
|
||||
|
||||
Usage:
|
||||
python3 tools/scan_pushed.py [range] # default range: origin/main..HEAD
|
||||
|
||||
Reports added lines matching secret patterns, suspicious filenames,
|
||||
and an extension histogram. Read-only: never modifies the repo.
|
||||
Exit 0 always; review HIT lines by hand since test fixtures match too.
|
||||
"""
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
RANGE = sys.argv[1] if len(sys.argv) > 1 else "origin/main..HEAD"
|
||||
|
||||
PATTERNS = [
|
||||
("private-key", re.compile(r"BEGIN.{0,20}PRIVATE KEY")),
|
||||
("github-token", re.compile(r"ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}")),
|
||||
("openai-key", re.compile(r"sk-(proj-|live-|test-)?[A-Za-z0-9]{10,}")),
|
||||
("aws-key", re.compile(r"AKIA[0-9A-Z]{16}")),
|
||||
("gcp-api-key", re.compile(r"AIza[0-9A-Za-z_-]{35}")),
|
||||
("slack-token", re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}")),
|
||||
("google-oauth", re.compile(r"ya29\.[A-Za-z0-9_-]{10,}")),
|
||||
("sendgrid", re.compile(r"SG\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")),
|
||||
("bearer", re.compile(r"Bearer [A-Za-z0-9._~+/-]{20,}")),
|
||||
("password-assign", re.compile(r"password\s*[:=]\s*['\"][^'\"]{4,}['\"]", re.I)),
|
||||
("apikey-assign", re.compile(r"api[_-]?key\s*[:=]\s*['\"][A-Za-z0-9_~.\-]{12,}['\"]", re.I)),
|
||||
("secret-assign", re.compile(r"(client_)?secret\s*[:=]\s*['\"][A-Za-z0-9_~.\-]{12,}['\"]", re.I)),
|
||||
("token-assign", re.compile(r"token\s*[:=]\s*['\"][A-Za-z0-9_~.\-]{16,}['\"]", re.I)),
|
||||
("conn-string", re.compile(r"(postgres|mysql|mongodb)(\+\w+)?://[^/\s:]+:[^/\s@]+@", re.I)),
|
||||
]
|
||||
|
||||
NAME_RE = re.compile(
|
||||
r"secret|credential|private|\.pem$|\.key$|\.p12$|\.pfx$|id_rsa|\.db$|\.sqlite|\.pyc$|__pycache__",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def run_git(args):
|
||||
p = subprocess.run(["git"] + args, capture_output=True, text=True)
|
||||
if p.returncode != 0:
|
||||
print(f"git {' '.join(args)} failed: {p.stderr.strip()}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
return p.stdout
|
||||
|
||||
|
||||
print(f"range: {RANGE}")
|
||||
diff = run_git(["diff", RANGE, "-U0"])
|
||||
|
||||
cur_file = "?"
|
||||
added = 0
|
||||
hits = []
|
||||
for line in diff.splitlines():
|
||||
if line.startswith("+++ b/"):
|
||||
cur_file = line[6:]
|
||||
elif line.startswith("+") and not line.startswith("+++"):
|
||||
added += 1
|
||||
body = line[1:]
|
||||
for name, rx in PATTERNS:
|
||||
if rx.search(body):
|
||||
hits.append((name, cur_file, body[:200]))
|
||||
|
||||
print(f"added lines scanned: {added}")
|
||||
print(f"pattern hits: {len(hits)}")
|
||||
for name, f, snippet in hits[:60]:
|
||||
print(f"HIT [{name}] {f}: {snippet}")
|
||||
if len(hits) > 60:
|
||||
print(f"... and {len(hits) - 60} more")
|
||||
|
||||
names = run_git(["diff", "--name-only", RANGE]).splitlines()
|
||||
print(f"files in range: {len(names)}")
|
||||
flagged = [n for n in names if NAME_RE.search(n)]
|
||||
print(f"suspicious filenames: {len(flagged)}")
|
||||
for n in flagged[:40]:
|
||||
print(f"NAME: {n}")
|
||||
|
||||
exts = {}
|
||||
for n in names:
|
||||
ext = n.rsplit(".", 1)[-1].lower() if "." in n else "(none)"
|
||||
exts[ext] = exts.get(ext, 0) + 1
|
||||
print("extensions:")
|
||||
for ext, c in sorted(exts.items(), key=lambda kv: -kv[1])[:25]:
|
||||
print(f" {ext}: {c}")
|
||||
Reference in New Issue
Block a user