tai/taicpp/tools/wa-api-inventory.sh

#!/usr/bin/env bash
# wa-api-inventory.sh: extract the mechanical transpilation scope from the pinned
# whatsmeow source (M0 harness; see docs/whatsapp.md section 17 and
# full_transpilation_and_integration.md prompt section 1).
#
# Scope, mechanically:
#   1. Every exported symbol (functions, methods, types, constants, vars) in
#      whatsmeow's non-test, non-generated Go source. Skipped: *_test.go files,
#      files whose header marks them generated (DO NOT EDIT / DO NOT MODIFY /
#      "generated by": the *.pb.go files, internals.go, binary/proto/legacy.go),
#      and files with "//go:build ignore" (the generator tool).
#   2. Every generated protobuf message the library uses in non-test code: a
#      message struct from a *.pb.go file counts when "<pkg>.<Msg>" is
#      referenced in an in-scope file. The proto.Marshal/proto.Unmarshal call
#      sites that motivate this are listed separately as evidence.
#
# Output (all under build/, deterministic, sorted):
#   build/wa-inventory.tsv      package<tab>kind<tab>key<tab>file
#   build/wa-marshal-sites.txt  file:line of every proto.Marshal/Unmarshal call
# Kinds: func, method, type, const, var, proto. Key is package-qualified
# ("whatsmeow.Client.Connect", "types.JID", "waE2E.Message").
set -euo pipefail
cd "$(dirname "$0")/.."

WA_VERSION="v0.0.0-20261027104441-1eec1136a50b"
WA_HASH="1eec1136a50b2ef148b05f1819e77a9d989e0457"
OUT="build/wa-inventory.tsv"
SITES="build/wa-marshal-sites.txt"

GOMODCACHE="$(go env GOMODCACHE)"
WA_SRC="$GOMODCACHE/go.mau.fi/whatsmeow@$WA_VERSION"
INFO="$GOMODCACHE/cache/download/go.mau.fi/whatsmeow/@v/$WA_VERSION.info"
test -d "$WA_SRC" || { echo "wa-api-inventory: missing $WA_SRC" >&2; exit 1; }
test -f "$INFO" || { echo "wa-api-inventory: missing $INFO" >&2; exit 1; }
GOT_HASH="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["Origin"]["Hash"])' "$INFO")"
test "$GOT_HASH" = "$WA_HASH" || {
  echo "wa-api-inventory: pin drift: info says $GOT_HASH, want $WA_HASH" >&2; exit 1; }

mkdir -p build
WA_SRC="$WA_SRC" OUT="$OUT" SITES="$SITES" python3 - <<'EOF'
import os, re, sys

wa = os.environ["WA_SRC"]
out, sites = os.environ["OUT"], os.environ["SITES"]

def header_lines(path):
    with open(path, encoding="utf-8") as f:
        return [f.readline() for _ in range(8)]

def is_generated(path):
    return any("DO NOT EDIT" in ln or "DO NOT MODIFY" in ln or
               "generated by" in ln.lower() for ln in header_lines(path))

def is_ignored(path):
    with open(path, encoding="utf-8") as f:
        for ln in f:
            if ln.startswith("//go:build ignore"):
                return True
            if ln.startswith("package "):
                return False
    return False

def pkg_of(text):
    m = re.search(r"^package (\w+)", text, re.M)
    return m.group(1) if m else "?"

def strip_line(ln):
    # Strings first, then the comment: a "//" inside a literal (e.g. the
    # "https://..." token in binary/token/token.go) must not truncate the
    # line, or brace-depth counting breaks and the block swallows the file.
    ln = re.sub(r"`[^`]*`", "", ln)
    ln = re.sub(r'"(?:[^"\\]|\\.)*"', "", ln)
    ln = ln.split("//", 1)[0]
    return ln

def exported(name):
    return bool(name) and name[0].isupper() and name != "_"

# Collect files.
gofiles = []
for root, _, files in os.walk(wa):
    for fn in files:
        if fn.endswith(".go"):
            gofiles.append(os.path.relpath(os.path.join(root, fn), wa))
gofiles.sort()

scoped, pbfiles = [], []
for rel in gofiles:
    if rel.endswith("_test.go"):
        continue
    full = os.path.join(wa, rel)
    if is_generated(full) or is_ignored(full):
        if rel.endswith(".pb.go"):
            pbfiles.append(rel)
        elif not rel.endswith(".pb.go"):
            sys.stderr.write("note: generated/ignored, out of scope: %s\n" % rel)
        continue
    scoped.append(rel)

rows = []  # (pkg, kind, key, file)
allsrc = []
for rel in scoped:
    text = open(os.path.join(wa, rel), encoding="utf-8").read()
    allsrc.append(text)
    pkg = pkg_of(text)
    # Blank multi-line raw-string bodies: SQL inside const blocks must not
    # parse as constant names, and braces inside strings must not move depth.
    rawlines = []
    in_raw = False
    for rln in text.split("\n"):
        if in_raw:
            if "`" in rln:
                in_raw = False
            rawlines.append("")
        elif rln.count("`") % 2 == 1:
            in_raw = True
            rawlines.append(re.sub(r"`[^`]*$", "", rln))
        else:
            rawlines.append(rln)
    lines = rawlines
    i, n = 0, len(lines)
    while i < n:
        ln = lines[i]
        m = re.match(r"^func \(([^)]*)\) ([A-Za-z_]\w*)", ln)
        if m:
            recv = m.group(1).strip().split()
            rtype = recv[-1].lstrip("*").split("[")[0] if recv else "?"
            meth = m.group(2)
            if exported(meth):
                rows.append((pkg, "method", "%s.%s.%s" % (pkg, rtype, meth), rel))
            i += 1
            continue
        m = re.match(r"^func ([A-Za-z_]\w*)", ln)
        if m and exported(m.group(1)):
            rows.append((pkg, "func", "%s.%s" % (pkg, m.group(1)), rel))
            i += 1
            continue
        m = re.match(r"^type ([A-Za-z_]\w*)", ln)
        if m and exported(m.group(1)):
            rows.append((pkg, "type", "%s.%s" % (pkg, m.group(1)), rel))
            i += 1
            continue
        m = re.match(r"^type \($", ln.strip())
        blk = None
        if m:
            blk = "type"
        elif ln.strip() in ("const (", "var ("):
            blk = "const" if ln.startswith("const") else "var"
        elif re.match(r"^const ([A-Za-z_]\w*)", ln):
            if exported(re.match(r"^const ([A-Za-z_]\w*)", ln).group(1)):
                rows.append((pkg, "const", "%s.%s" % (pkg, re.match(r"^const ([A-Za-z_]\w*)", ln).group(1)), rel))
            i += 1
            continue
        elif re.match(r"^var ([A-Za-z_]\w*)", ln):
            if exported(re.match(r"^var ([A-Za-z_]\w*)", ln).group(1)):
                rows.append((pkg, "var", "%s.%s" % (pkg, re.match(r"^var ([A-Za-z_]\w*)", ln).group(1)), rel))
            i += 1
            continue
        if blk is None:
            i += 1
            continue
        depth = 0
        i += 1
        while i < n:
            bln = lines[i]
            if bln.strip() == ")" and depth == 0:
                break
            code = strip_line(bln)
            if depth == 0:
                tok = re.match(r"\s*([A-Za-z_]\w*)", code)
                if tok and exported(tok.group(1)):
                    rows.append((pkg, blk, "%s.%s" % (pkg, tok.group(1)), rel))
            depth += code.count("{") - code.count("}")
            i += 1
        i += 1

# Proto messages used in non-test code.
blob = "\n".join(allsrc)
proto_count = 0
for rel in sorted(pbfiles):
    text = open(os.path.join(wa, rel), encoding="utf-8").read()
    pkg = pkg_of(text)
    plines = text.split("\n")
    for j, pln in enumerate(plines):
        m = re.match(r"^type (\w+) struct \{$", pln)
        if not m:
            continue
        window = "\n".join(plines[j:j + 3])
        if "protoimpl.MessageState" not in window:
            continue
        msg = m.group(1)
        if re.search(r"\b%s\.%s\b" % (re.escape(pkg), re.escape(msg)), blob):
            rows.append((pkg, "proto", "%s.%s" % (pkg, msg), rel))
            proto_count += 1

rows.sort(key=lambda r: (r[0], r[1], r[2], r[3]))
seen = {}
for pkg, kind, key, rel in rows:
    if key in seen:
        sys.stderr.write("DUPKEY %s first %s then %s:%s\n" % (key, seen[key], kind, rel))
        sys.exit("wa-api-inventory: duplicate key, parser bug")
    seen[key] = "%s:%s" % (kind, rel)
with open(out, "w", encoding="utf-8") as f:
    for pkg, kind, key, rel in rows:
        f.write("%s\t%s\t%s\t%s\n" % (pkg, kind, key, rel))

# Marshal/Unmarshal call sites (evidence, not coverable entries).
with open(sites, "w", encoding="utf-8") as f:
    for rel in scoped:
        full = os.path.join(wa, rel)
        for j, ln in enumerate(open(full, encoding="utf-8"), 1):
            if re.search(r"proto\.(Marshal|Unmarshal)\b", ln):
                f.write("%s:%d: %s" % (rel, j, ln.strip()[:160] + "\n"))

kinds = {}
for _, k, _, _ in rows:
    kinds[k] = kinds.get(k, 0) + 1
sys.stderr.write("scoped go files: %d, generated pb files: %d\n" % (len(scoped), len(pbfiles)))
sys.stderr.write("inventory entries: %d %s\n" % (len(rows), kinds))
sys.stderr.write("wrote %s\n" % out)
EOF
echo "inventory: $(wc -l < "$OUT") entries, $(wc -l < "$SITES") marshal sites"