feat(clients): discovery: generic engine for all Google Discovery APIs
Directory and document cache with caps, parameter and body validation before traffic, path/query/repeated/regional handling, key-or-OAuth choice, paging, long-running wait, whole-directory checker script, demo at /google/discovery. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Walk Google's whole Discovery directory and report what the engine cannot do.
|
||||
|
||||
python3 src/google/discovery/tools/check_discovery_directory.py [--cache DIR]
|
||||
[--no-fetch] [--json FILE] [--max-rejected N] [--workers N]
|
||||
|
||||
1. Fetches https://discovery.googleapis.com/discovery/v1/apis and the
|
||||
document of every preferred API into --cache (default
|
||||
build/google/discovery-cache). Fetch failures are listed as UNREACHABLE:
|
||||
Google lists them but does not serve them (not an engine fault).
|
||||
2. Builds build/google/discovery-audit (src/google/discovery/tools/audit_main.cc)
|
||||
against libgoogle.a and runs it on the saved files, so the verdict comes
|
||||
from the engine's own classification, never from a second implementation.
|
||||
3. Prints totals, a reason table and the APIs with rejected or limited
|
||||
methods. Exit 1 when a fetched document does not parse, or when rejected
|
||||
methods exceed --max-rejected (default: no ceiling).
|
||||
|
||||
python3 standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures as cf
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
|
||||
DIRECTORY = "https://discovery.googleapis.com/discovery/v1/apis"
|
||||
AUDIT_SRC = os.path.join(ROOT, "src/google/discovery/tools/audit_main.cc")
|
||||
AUDIT_BIN = os.path.join(ROOT, "build/google/discovery-audit")
|
||||
LIBS = ["build/google/libgoogle.a", "build/clients/libclients.a", "build/liboperability.a"]
|
||||
|
||||
|
||||
def get(url, tries=3):
|
||||
last = None
|
||||
for _ in range(tries):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=90) as r:
|
||||
return r.read()
|
||||
except Exception as e: # noqa: BLE001 - report whatever the network said
|
||||
last = e
|
||||
raise last
|
||||
|
||||
|
||||
def fetch_all(cache, workers):
|
||||
os.makedirs(cache, exist_ok=True)
|
||||
raw = get(DIRECTORY)
|
||||
with open(os.path.join(cache, "directory.json"), "wb") as f:
|
||||
f.write(raw)
|
||||
items = [i for i in json.loads(raw)["items"] if i.get("preferred")]
|
||||
unreachable = []
|
||||
|
||||
def one(item):
|
||||
path = os.path.join(cache, "%s_%s.json" % (item["name"], item["version"]))
|
||||
try:
|
||||
data = get(item["discoveryRestUrl"])
|
||||
with open(path, "wb") as f:
|
||||
f.write(data)
|
||||
return None
|
||||
except Exception as e: # noqa: BLE001
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
return (item["id"], str(e))
|
||||
|
||||
with cf.ThreadPoolExecutor(workers) as ex:
|
||||
for r in ex.map(one, items):
|
||||
if r:
|
||||
unreachable.append(r)
|
||||
return len(items), unreachable
|
||||
|
||||
|
||||
def build_audit():
|
||||
subprocess.run(["make", "-s", "-j8", "google", "build/clients/libclients.a",
|
||||
"build/liboperability.a"], cwd=ROOT, check=True)
|
||||
cmd = ["g++", "-std=c++23", "-Wall", "-Wextra", "-Werror", "-DMOLO_HAS_WOLFSSL",
|
||||
"-I" + os.path.join(ROOT, "src"), AUDIT_SRC] + [os.path.join(ROOT, l) for l in LIBS] + \
|
||||
["-lwolfssl", "-o", AUDIT_BIN]
|
||||
subprocess.run(cmd, cwd=ROOT, check=True)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--cache", default=os.path.join(ROOT, "build/google/discovery-cache"))
|
||||
ap.add_argument("--no-fetch", action="store_true", help="reuse the files already in --cache")
|
||||
ap.add_argument("--json", help="write the full audit report here")
|
||||
ap.add_argument("--max-rejected", type=int, default=None)
|
||||
ap.add_argument("--workers", type=int, default=16)
|
||||
ap.add_argument("--list", type=int, default=40, help="APIs to list with problems")
|
||||
args = ap.parse_args()
|
||||
|
||||
unreachable = []
|
||||
listed = None
|
||||
if not args.no_fetch:
|
||||
listed, unreachable = fetch_all(args.cache, args.workers)
|
||||
build_audit()
|
||||
out = subprocess.run([AUDIT_BIN, args.cache], capture_output=True, text=True)
|
||||
if out.returncode != 0:
|
||||
print(out.stderr, file=sys.stderr)
|
||||
return 1
|
||||
rep = json.loads(out.stdout)
|
||||
if args.json:
|
||||
with open(args.json, "w") as f:
|
||||
json.dump(rep, f, indent=1)
|
||||
|
||||
unreach_ids = {u[0] for u in unreachable}
|
||||
parse_failures = [f for f in rep["failed"] if f["api"] not in unreach_ids]
|
||||
gone = [f for f in rep["failed"] if f["api"] in unreach_ids or "NOT_FOUND" in f["why"]]
|
||||
print("preferred APIs in the directory : %d" % rep["preferred"])
|
||||
print("documents loaded by the engine : %d" % rep["loaded"])
|
||||
print("UNREACHABLE at the source : %d" % len(gone))
|
||||
for f in gone:
|
||||
print(" %-40s %s" % (f["api"], f["why"][:90]))
|
||||
parse_failures = [f for f in parse_failures if f not in gone]
|
||||
print("documents the engine REJECTS : %d" % len(parse_failures))
|
||||
for f in parse_failures:
|
||||
print(" %-40s %s" % (f["api"], f["why"][:160]))
|
||||
print("methods : %d" % rep["methods"])
|
||||
print(" full / limited / rejected : %d / %d / %d" % (rep["full"], rep["limited"], rep["rejected"]))
|
||||
print(" pageable (pageToken) : %d" % rep["pageable"])
|
||||
print(" long-running replies : %d (%d in APIs with no operations.get)" %
|
||||
(rep["longRunning"], rep["longRunningWithoutGetter"]))
|
||||
print("dry plans (required params only): %d planned, %d skipped (synthetic value vs exotic pattern), %d FAILED" %
|
||||
(rep["planned"], rep["planSkippedPattern"], rep["planFailed"]))
|
||||
for f in rep["planFailures"][:20]:
|
||||
print(" %s: %s" % (f["method"], f["why"][:140]))
|
||||
print("APIs with regional endpoints : %d" % rep["apisWithRegionalEndpoints"])
|
||||
print("APIs declaring a batchPath : %d (batch endpoints are not used)" % rep["apisDeclaringBatchPath"])
|
||||
print("\nreasons (methods):")
|
||||
for k, v in sorted(rep["reasons"].items(), key=lambda kv: -kv[1]):
|
||||
print(" %6d %s" % (v, k))
|
||||
bad = [a for a in rep["apis"] if a["rejected"] or a["limited"]]
|
||||
bad.sort(key=lambda a: -(a["rejected"] * 1000 + a["limited"]))
|
||||
print("\nAPIs with rejected or limited methods: %d of %d" % (len(bad), len(rep["apis"])))
|
||||
for a in bad[:args.list]:
|
||||
sample = a["problems"][0]
|
||||
print(" %-34s rejected %-3d limited %-3d e.g. %s: %s" %
|
||||
(a["api"], a["rejected"], a["limited"], sample["method"], sample["notes"][0][:70]))
|
||||
if len(bad) > args.list:
|
||||
print(" ... %d more (use --json)" % (len(bad) - args.list))
|
||||
|
||||
code = 0
|
||||
if parse_failures or rep["planFailed"]:
|
||||
code = 1
|
||||
if args.max_rejected is not None and rep["rejected"] > args.max_rejected:
|
||||
print("FAIL: %d rejected methods exceed --max-rejected %d" % (rep["rejected"], args.max_rejected))
|
||||
code = 1
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user