220 lines
8.2 KiB
Bash
Executable File
220 lines
8.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# wa-api-inventory.sh: extract the mechanical transpilation scope from the pinned
|
|
# whatsmeow source (M0 harness; see docs/whatsapp.md section 17 and
|
|
# full_transpilation_and_integration.md prompt section 1).
|
|
#
|
|
# Scope, mechanically:
|
|
# 1. Every exported symbol (functions, methods, types, constants, vars) in
|
|
# whatsmeow's non-test, non-generated Go source. Skipped: *_test.go files,
|
|
# files whose header marks them generated (DO NOT EDIT / DO NOT MODIFY /
|
|
# "generated by": the *.pb.go files, internals.go, binary/proto/legacy.go),
|
|
# and files with "//go:build ignore" (the generator tool).
|
|
# 2. Every generated protobuf message the library uses in non-test code: a
|
|
# message struct from a *.pb.go file counts when "<pkg>.<Msg>" is
|
|
# referenced in an in-scope file. The proto.Marshal/proto.Unmarshal call
|
|
# sites that motivate this are listed separately as evidence.
|
|
#
|
|
# Output (all under build/, deterministic, sorted):
|
|
# build/wa-inventory.tsv package<tab>kind<tab>key<tab>file
|
|
# build/wa-marshal-sites.txt file:line of every proto.Marshal/Unmarshal call
|
|
# Kinds: func, method, type, const, var, proto. Key is package-qualified
|
|
# ("whatsmeow.Client.Connect", "types.JID", "waE2E.Message").
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
|
|
WA_VERSION="v0.0.0-20261027104441-1eec1136a50b"
|
|
WA_HASH="1eec1136a50b2ef148b05f1819e77a9d989e0457"
|
|
OUT="build/wa-inventory.tsv"
|
|
SITES="build/wa-marshal-sites.txt"
|
|
|
|
GOMODCACHE="$(go env GOMODCACHE)"
|
|
WA_SRC="$GOMODCACHE/go.mau.fi/whatsmeow@$WA_VERSION"
|
|
INFO="$GOMODCACHE/cache/download/go.mau.fi/whatsmeow/@v/$WA_VERSION.info"
|
|
test -d "$WA_SRC" || { echo "wa-api-inventory: missing $WA_SRC" >&2; exit 1; }
|
|
test -f "$INFO" || { echo "wa-api-inventory: missing $INFO" >&2; exit 1; }
|
|
GOT_HASH="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["Origin"]["Hash"])' "$INFO")"
|
|
test "$GOT_HASH" = "$WA_HASH" || {
|
|
echo "wa-api-inventory: pin drift: info says $GOT_HASH, want $WA_HASH" >&2; exit 1; }
|
|
|
|
mkdir -p build
|
|
WA_SRC="$WA_SRC" OUT="$OUT" SITES="$SITES" python3 - <<'EOF'
|
|
import os, re, sys
|
|
|
|
wa = os.environ["WA_SRC"]
|
|
out, sites = os.environ["OUT"], os.environ["SITES"]
|
|
|
|
def header_lines(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
return [f.readline() for _ in range(8)]
|
|
|
|
def is_generated(path):
|
|
return any("DO NOT EDIT" in ln or "DO NOT MODIFY" in ln or
|
|
"generated by" in ln.lower() for ln in header_lines(path))
|
|
|
|
def is_ignored(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
for ln in f:
|
|
if ln.startswith("//go:build ignore"):
|
|
return True
|
|
if ln.startswith("package "):
|
|
return False
|
|
return False
|
|
|
|
def pkg_of(text):
|
|
m = re.search(r"^package (\w+)", text, re.M)
|
|
return m.group(1) if m else "?"
|
|
|
|
def strip_line(ln):
|
|
ln = ln.split("//", 1)[0]
|
|
ln = re.sub(r"`[^`]*`", "", ln)
|
|
ln = re.sub(r'"(?:[^"\\]|\\.)*"', "", ln)
|
|
return ln
|
|
|
|
def exported(name):
|
|
return bool(name) and name[0].isupper() and name != "_"
|
|
|
|
# Collect files.
|
|
gofiles = []
|
|
for root, _, files in os.walk(wa):
|
|
for fn in files:
|
|
if fn.endswith(".go"):
|
|
gofiles.append(os.path.relpath(os.path.join(root, fn), wa))
|
|
gofiles.sort()
|
|
|
|
scoped, pbfiles = [], []
|
|
for rel in gofiles:
|
|
if rel.endswith("_test.go"):
|
|
continue
|
|
full = os.path.join(wa, rel)
|
|
if is_generated(full) or is_ignored(full):
|
|
if rel.endswith(".pb.go"):
|
|
pbfiles.append(rel)
|
|
elif not rel.endswith(".pb.go"):
|
|
sys.stderr.write("note: generated/ignored, out of scope: %s\n" % rel)
|
|
continue
|
|
scoped.append(rel)
|
|
|
|
rows = [] # (pkg, kind, key, file)
|
|
allsrc = []
|
|
for rel in scoped:
|
|
text = open(os.path.join(wa, rel), encoding="utf-8").read()
|
|
allsrc.append(text)
|
|
pkg = pkg_of(text)
|
|
# Blank multi-line raw-string bodies: SQL inside const blocks must not
|
|
# parse as constant names, and braces inside strings must not move depth.
|
|
rawlines = []
|
|
in_raw = False
|
|
for rln in text.split("\n"):
|
|
if in_raw:
|
|
if "`" in rln:
|
|
in_raw = False
|
|
rawlines.append("")
|
|
elif rln.count("`") % 2 == 1:
|
|
in_raw = True
|
|
rawlines.append(re.sub(r"`[^`]*$", "", rln))
|
|
else:
|
|
rawlines.append(rln)
|
|
lines = rawlines
|
|
i, n = 0, len(lines)
|
|
while i < n:
|
|
ln = lines[i]
|
|
m = re.match(r"^func \(([^)]*)\) ([A-Za-z_]\w*)", ln)
|
|
if m:
|
|
recv = m.group(1).strip().split()
|
|
rtype = recv[-1].lstrip("*").split("[")[0] if recv else "?"
|
|
meth = m.group(2)
|
|
if exported(meth):
|
|
rows.append((pkg, "method", "%s.%s.%s" % (pkg, rtype, meth), rel))
|
|
i += 1
|
|
continue
|
|
m = re.match(r"^func ([A-Za-z_]\w*)", ln)
|
|
if m and exported(m.group(1)):
|
|
rows.append((pkg, "func", "%s.%s" % (pkg, m.group(1)), rel))
|
|
i += 1
|
|
continue
|
|
m = re.match(r"^type ([A-Za-z_]\w*)", ln)
|
|
if m and exported(m.group(1)):
|
|
rows.append((pkg, "type", "%s.%s" % (pkg, m.group(1)), rel))
|
|
i += 1
|
|
continue
|
|
m = re.match(r"^type \($", ln.strip())
|
|
blk = None
|
|
if m:
|
|
blk = "type"
|
|
elif ln.strip() in ("const (", "var ("):
|
|
blk = "const" if ln.startswith("const") else "var"
|
|
elif re.match(r"^const ([A-Za-z_]\w*)", ln):
|
|
if exported(re.match(r"^const ([A-Za-z_]\w*)", ln).group(1)):
|
|
rows.append((pkg, "const", "%s.%s" % (pkg, re.match(r"^const ([A-Za-z_]\w*)", ln).group(1)), rel))
|
|
i += 1
|
|
continue
|
|
elif re.match(r"^var ([A-Za-z_]\w*)", ln):
|
|
if exported(re.match(r"^var ([A-Za-z_]\w*)", ln).group(1)):
|
|
rows.append((pkg, "var", "%s.%s" % (pkg, re.match(r"^var ([A-Za-z_]\w*)", ln).group(1)), rel))
|
|
i += 1
|
|
continue
|
|
if blk is None:
|
|
i += 1
|
|
continue
|
|
depth = 0
|
|
i += 1
|
|
while i < n:
|
|
bln = lines[i]
|
|
if bln.strip() == ")" and depth == 0:
|
|
break
|
|
code = strip_line(bln)
|
|
if depth == 0:
|
|
tok = re.match(r"\s*([A-Za-z_]\w*)", code)
|
|
if tok and exported(tok.group(1)):
|
|
rows.append((pkg, blk, "%s.%s" % (pkg, tok.group(1)), rel))
|
|
depth += code.count("{") - code.count("}")
|
|
i += 1
|
|
i += 1
|
|
|
|
# Proto messages used in non-test code.
|
|
blob = "\n".join(allsrc)
|
|
proto_count = 0
|
|
for rel in sorted(pbfiles):
|
|
text = open(os.path.join(wa, rel), encoding="utf-8").read()
|
|
pkg = pkg_of(text)
|
|
plines = text.split("\n")
|
|
for j, pln in enumerate(plines):
|
|
m = re.match(r"^type (\w+) struct \{$", pln)
|
|
if not m:
|
|
continue
|
|
window = "\n".join(plines[j:j + 3])
|
|
if "protoimpl.MessageState" not in window:
|
|
continue
|
|
msg = m.group(1)
|
|
if re.search(r"\b%s\.%s\b" % (re.escape(pkg), re.escape(msg)), blob):
|
|
rows.append((pkg, "proto", "%s.%s" % (pkg, msg), rel))
|
|
proto_count += 1
|
|
|
|
rows.sort(key=lambda r: (r[0], r[1], r[2], r[3]))
|
|
seen = {}
|
|
for pkg, kind, key, rel in rows:
|
|
if key in seen:
|
|
sys.stderr.write("DUPKEY %s first %s then %s:%s\n" % (key, seen[key], kind, rel))
|
|
sys.exit("wa-api-inventory: duplicate key, parser bug")
|
|
seen[key] = "%s:%s" % (kind, rel)
|
|
with open(out, "w", encoding="utf-8") as f:
|
|
for pkg, kind, key, rel in rows:
|
|
f.write("%s\t%s\t%s\t%s\n" % (pkg, kind, key, rel))
|
|
|
|
# Marshal/Unmarshal call sites (evidence, not coverable entries).
|
|
with open(sites, "w", encoding="utf-8") as f:
|
|
for rel in scoped:
|
|
full = os.path.join(wa, rel)
|
|
for j, ln in enumerate(open(full, encoding="utf-8"), 1):
|
|
if re.search(r"proto\.(Marshal|Unmarshal)\b", ln):
|
|
f.write("%s:%d: %s" % (rel, j, ln.strip()[:160] + "\n"))
|
|
|
|
kinds = {}
|
|
for _, k, _, _ in rows:
|
|
kinds[k] = kinds.get(k, 0) + 1
|
|
sys.stderr.write("scoped go files: %d, generated pb files: %d\n" % (len(scoped), len(pbfiles)))
|
|
sys.stderr.write("inventory entries: %d %s\n" % (len(rows), kinds))
|
|
sys.stderr.write("wrote %s\n" % out)
|
|
EOF
|
|
echo "inventory: $(wc -l < "$OUT") entries, $(wc -l < "$SITES") marshal sites"
|