#!/usr/bin/env python3 """Checks src/google/sessions/calls.h against Google's Discovery document. Fetches https://aiplatform.googleapis.com/$discovery/rest?version= (no credential needed) and fails when a declared call, verb, path, request schema field, query parameter, enum value or response key is absent. python3 src/google/sessions/tools/check_discovery.py [--file discovery.json] Exit 0: everything declared exists. Exit 1: mismatches, listed. Exit 2: the discovery document could not be read. Python 3 stdlib only. Sends no secret. """ import datetime import json import os import re import sys import urllib.request HERE = os.path.dirname(os.path.abspath(__file__)) CALLS_H = os.path.join(HERE, "..", "calls.h") SCHEMA_PREFIX = "GoogleCloudAiplatform" # (schema suffix, property, expected enum values that the C++ code sends) ENUMS = [ ("MemoryTopicId", "managedMemoryTopic", ["USER_PERSONAL_INFO", "USER_PREFERENCES", "KEY_CONVERSATION_DETAILS", "EXPLICIT_INSTRUCTIONS"]), ("GenerateMemoriesRequest", "metadataMergeStrategy", ["OVERWRITE", "MERGE", "REQUIRE_EXACT_MATCH"]), ("IngestEventsRequest", "metadataMergeStrategy", ["OVERWRITE", "MERGE", "REQUIRE_EXACT_MATCH"]), ("MemoryFilter", "op", ["EQUAL", "GREATER_THAN", "LESS_THAN"]), ("Memory", "memoryType", ["NATURAL_LANGUAGE_COLLECTION", "STRUCTURED_PROFILE"]), ("CompactionConfigLlmSummarizationConfig", "mode", ["AUTO"]), ("CompactionConfigEventEditingConfig", "mode", ["AUTO"]), ] # (schema suffix, property) read by the response parsers. RESPONSE_KEYS = [ ("ListSessionsResponse", "sessions"), ("ListSessionsResponse", "nextPageToken"), ("ListEventsResponse", "sessionEvents"), ("ListEventsResponse", "nextPageToken"), ("ListMemoriesResponse", "memories"), ("ListMemoriesResponse", "nextPageToken"), ("ListMemoryRevisionsResponse", "memoryRevisions"), ("RetrieveMemoriesResponse", "retrievedMemories"), ("RetrieveMemoriesResponse", "nextPageToken"), ("RetrieveMemoriesResponseRetrievedMemory", "memory"), ("RetrieveMemoriesResponseRetrievedMemory", "distance"), ("RetrieveProfilesResponse", "profiles"), ("MemoryProfile", "profile"), ("MemoryProfile", "schemaId"), ("MemoryRevision", "fact"), ("MemoryRevision", "context"), ("MemoryRevision", "labels"), ("MemoryRevision", "createTime"), ("MemoryRevision", "expireTime"), ("Session", "name"), ("Session", "createTime"), ("Session", "updateTime"), ("Session", "expireTime"), ("Memory", "name"), ("Memory", "expireTime"), ] # Nested request shapes built by the C++ code: (schema suffix, fields). NESTED = [ ("GenerateMemoriesRequestVertexSessionSource", "session startTime endTime"), ("GenerateMemoriesRequestDirectContentsSource", "events"), ("GenerateMemoriesRequestDirectContentsSourceEvent", "content"), ("GenerateMemoriesRequestDirectMemoriesSource", "directMemories"), ("GenerateMemoriesRequestDirectMemoriesSourceDirectMemory", "fact topics"), ("RetrieveMemoriesRequestSimilaritySearchParams", "searchQuery topK"), ("RetrieveMemoriesRequestSimpleRetrievalParams", "pageSize pageToken"), ("IngestionDirectContentsSource", "events"), ("IngestionDirectContentsSourceEvent", "eventId eventTime content"), ("MemoryGenerationTriggerConfig", "generationRule"), ("MemoryGenerationTriggerConfigGenerationTriggerRule", "eventCount overlapEventCount fixedInterval idleDuration"), ("MemoryConjunctionFilter", "filters"), ("MemoryFilter", "key op negate value"), ("MemoryMetadataValue", "stringValue doubleValue boolValue timestampValue"), ("MemoryTopicId", "customMemoryTopicLabel managedMemoryTopic"), ("CompactionConfig", "summarization eventEditing"), ("CompactionConfigLlmSummarizationConfig", "mode"), ("CompactionConfigEventEditingConfig", "mode"), ] def load(path): if path: with open(path, "r", encoding="utf-8") as f: return json.load(f) version = parse_header()["version"] url = "https://aiplatform.googleapis.com/$discovery/rest?version=" + version with urllib.request.urlopen(url, timeout=120) as r: return json.load(r) def parse_header(): text = open(CALLS_H, "r", encoding="utf-8").read() version = re.search(r'kDiscoveryVersion\s*=\s*"([^"]+)"', text).group(1) prefix = re.search(r'kDiscoveryPrefix\s*=\s*"([^"]+)"', text).group(1) rows = [] row_re = re.compile( r'^\{Call::(k\w+),\s*"(\w+)",\s*"([^"]*)",\s*"([^"]*)",\s*"([^"]*)",' r'\s*"([^"]*)",\s*"([^"]*)"\},\s*$', re.M) for m in row_re.finditer(text): rows.append({"call": m.group(1), "verb": m.group(2), "path": m.group(3), "method": m.group(4), "request": m.group(5), "fields": m.group(6).split(), "query": m.group(7).split()}) enum_names = re.search(r"enum class Call \{(.*?)kCount", text, re.S).group(1) declared = re.findall(r"\bk\w+", enum_names) return {"version": version, "prefix": prefix, "rows": rows, "declared": declared} def find_method(doc, method_id): found = [] def walk(res): for m in res.get("methods", {}).values(): if m["id"] == method_id: found.append(m) for r in res.get("resources", {}).values(): walk(r) walk(doc) return found[0] if found else None def normalize(path, version): path = re.sub(r"\{\+(\w+)\}", r"{\1}", path) return "/" + path[len(version) + 1:] if path.startswith(version + "/") else path def main(): path = None if "--file" in sys.argv: path = sys.argv[sys.argv.index("--file") + 1] header = parse_header() try: doc = load(path) except Exception as exc: # network or parse failure is not a mismatch print("cannot read discovery document: %s" % exc) return 2 version, prefix = header["version"], header["prefix"] schemas = doc["schemas"] schema_prefix = SCHEMA_PREFIX + ("V1beta1" if version == "v1beta1" else "V1") errors = [] rows = header["rows"] if sorted(r["call"] for r in rows) != sorted(header["declared"]): errors.append("calls.h: enum Call and table rows differ") for row in rows: mid = prefix + row["method"] m = find_method(doc, mid) if m is None: errors.append("%s: method %s absent" % (row["call"], mid)) continue if m["httpMethod"] != row["verb"]: errors.append("%s: verb %s != %s" % (row["call"], row["verb"], m["httpMethod"])) got = normalize(m["path"], version) if got != row["path"]: errors.append("%s: path %s != %s" % (row["call"], row["path"], got)) for q in row["query"]: p = m.get("parameters", {}).get(q) if not p or p.get("location") != "query": errors.append("%s: query parameter %s absent" % (row["call"], q)) if row["request"]: want = schema_prefix + row["request"] ref = m.get("request", {}).get("$ref") if ref != want: errors.append("%s: request %s != %s" % (row["call"], want, ref)) props = schemas.get(want, {}).get("properties", {}) for field in row["fields"]: if field not in props: errors.append("%s: field %s absent from %s" % (row["call"], field, want)) elif m.get("request"): errors.append("%s: discovery has a request body %s the table omits" % (row["call"], m["request"]["$ref"])) for suffix, prop, values in ENUMS: s = schemas.get(schema_prefix + suffix, {}).get("properties", {}).get(prop) if s is None: errors.append("enum %s.%s absent" % (suffix, prop)) continue allowed = s.get("enum") if allowed is None: # Free string in discovery: documented, not enforced. continue for v in values: if v not in allowed: errors.append("enum value %s absent from %s.%s" % (v, suffix, prop)) for suffix, prop in RESPONSE_KEYS: if prop not in schemas.get(schema_prefix + suffix, {}).get("properties", {}): errors.append("response key %s.%s absent" % (suffix, prop)) for suffix, fields in NESTED: props = schemas.get(schema_prefix + suffix, {}).get("properties", {}) for field in fields.split(): if field not in props: errors.append("nested %s.%s absent" % (suffix, field)) stamp = datetime.date.today().isoformat() if errors: print("FAIL (%s, discovery %s): %d mismatches" % (stamp, version, len(errors))) for e in errors: print(" - " + e) return 1 print("OK %s: %d calls, %d enums, %d response keys, %d nested schemas match " "aiplatform %s discovery" % (stamp, len(rows), len(ENUMS), len(RESPONSE_KEYS), len(NESTED), version)) return 0 if __name__ == "__main__": sys.exit(main())