#!/usr/bin/env python3 """Fails when a call, request field, schema field or enum value declared in src/google/sandbox/calls.h is absent from Google's Discovery documents. Usage: python3 -I check_discovery.py [--cache DIR] Stdlib only, no credentials (Discovery documents are public). Exit 0: every declared row exists. Exit 1: at least one row is missing (listed). Exit 2: a document could not be fetched (loud, never a silent pass). """ import json import os import re import sys import urllib.request HERE = os.path.dirname(os.path.abspath(__file__)) CALLS_H = os.path.join(HERE, "..", "calls.h") ROW = re.compile(r'^\s*\{((?:"(?:[^"\\]|\\.)*"\s*,?\s*)+)\},?\s*$') STR = re.compile(r'"((?:[^"\\]|\\.)*)"') def rows(text, begin, end): block = text.split(begin, 1)[1].split(end, 1)[0] out = [] for line in block.splitlines(): m = ROW.match(line) if m: out.append([s for s in STR.findall(m.group(1))]) return out _docs = {} def discovery(service, version, cache): key = (service, version) if key in _docs: return _docs[key] path = os.path.join(cache, f"{service}_{version}.json") if cache else None if path and os.path.exists(path): doc = json.load(open(path)) else: url = f"https://{service}.googleapis.com/$discovery/rest?version={version}" try: with urllib.request.urlopen(url, timeout=120) as r: raw = r.read() except Exception as e: # noqa: BLE001 print(f"FETCH FAILED {url}: {e}") sys.exit(2) doc = json.loads(raw) if path: os.makedirs(cache, exist_ok=True) open(path, "wb").write(raw) _docs[key] = doc return doc def methods(doc): found = {} def walk(res): for r in res.values(): for m in r.get("methods", {}).values(): found[m["id"]] = m walk(r.get("resources", {})) walk(doc.get("resources", {})) return found def schema_id(service, version, name): prefix = {"aiplatform": "GoogleCloudAiplatform"}[service] return prefix + version[0].upper() + version[1:] + name def resolve(doc, prop): """Follow $ref and array items to the object schema of a property.""" if "items" in prop: prop = prop["items"] if "$ref" in prop: return doc["schemas"][prop["$ref"]] return prop def has_field_path(doc, schema, path): cur = schema for part in path.split("."): props = cur.get("properties", {}) if part not in props: return False cur = resolve(doc, props[part]) return True def main(): cache = None if "--cache" in sys.argv: cache = sys.argv[sys.argv.index("--cache") + 1] text = open(CALLS_H).read() calls = rows(text, "// CALLS-BEGIN", "// CALLS-END") schemas = rows(text, "// SCHEMAS-BEGIN", "// SCHEMAS-END") enums = rows(text, "// ENUMS-BEGIN", "// ENUMS-END") bad = [] checked = 0 for name, http, path, service, version, rpc, request in calls: doc = discovery(service, version, cache) m = methods(doc).get(rpc) if m is None: bad.append(f"{name}: rpc {rpc} missing in {service} {version}") continue if m["httpMethod"] != http: bad.append(f"{name}: http {m['httpMethod']} != declared {http}") if m["path"] != f"{version}/{path}": bad.append(f"{name}: path {m['path']} != declared {version}/{path}") if request: ref = m.get("request", {}).get("$ref") if not ref: bad.append(f"{name}: declares request fields but the method has no body") else: for f in request.split(","): if not has_field_path(doc, doc["schemas"][ref], f): bad.append(f"{name}: request field {f} missing in {ref}") checked += 1 else: checked += 1 elif m.get("request") and doc["schemas"][m["request"]["$ref"]].get("properties"): bad.append(f"{name}: method takes a body {m['request']['$ref']} but none is declared") checked += 3 for service, version, name, fields in schemas: doc = discovery(service, version, cache) s = doc["schemas"].get(schema_id(service, version, name)) if s is None: bad.append(f"schema {name} missing in {service} {version}") continue for f in fields.split(","): checked += 1 if f not in s.get("properties", {}): bad.append(f"schema {name}: field {f} missing in {service} {version}") for service, version, name, field, values in enums: doc = discovery(service, version, cache) s = doc["schemas"].get(schema_id(service, version, name)) prop = s.get("properties", {}).get(field) if s else None if prop is None: bad.append(f"enum {name}.{field} missing in {service} {version}") continue listed = set(prop.get("enum", [])) for v in values.split(","): checked += 1 if v not in listed: bad.append(f"enum {name}.{field}: value {v} not listed in {service} {version}") for b in bad: print("MISSING", b) print(f"{len(calls)} calls, {len(schemas)} schemas, {len(enums)} enums, " f"{checked} checks, {len(bad)} missing") sys.exit(1 if bad else 0) if __name__ == "__main__": main()