#!/usr/bin/env python3 """Check the xAI platform call tables against xAI's published specs. Fails (exit 1) when a declared call is absent from its source, a request field or query parameter we use is not in the spec, a WebSocket message, query parameter or session field in ws_events.cpp differs from the three docs.x.ai .ws.json documents, or a spec operation in our areas is neither in the table nor in an explicit exclusion list. Python 3 stdlib only, no credentials. python3 -I src/xai/platform/tools/check_spec.py Sources (all public, read at run time): openapi https://api.x.ai/api-docs/openapi.json (also docs.x.ai/openapi.json) docs https://docs.x.ai/developers/rest-api-reference/.md guide https://docs.x.ai/developers/.md (prose, cross-checks) ws https://docs.x.ai/{voice-realtime,tts-streaming,stt-streaming}.ws.json proto xai-org/xai-proto billing.proto and auth.proto (second source for billing) """ import json import os import re import sys import urllib.request HERE = os.path.dirname(os.path.abspath(__file__)) CALLS = os.path.join(HERE, "..", "calls.cpp") WSCPP = os.path.join(HERE, "..", "ws_events.cpp") DOCS = "https://docs.x.ai/developers/" OPENAPI = "https://api.x.ai/api-docs/openapi.json" PROTO = "https://raw.githubusercontent.com/xai-org/xai-proto/main/proto/xai/" # Operations that belong to other packages or are deliberately not wrapped. EXCLUDED_OPENAPI_PREFIXES = ( "/v1/chat/", "/v1/complete", "/v1/completions","/v1/embedding", "/v1/image", "/v1/language-models", "/v1/messages", "/v1/models", "/v1/responses", "/v1/tokenize-text", "/v1/video", "/v1/skills", ) EXCLUDED_DOCS = { # Documented without a request or response shape: nothing to verify against. ("POST", "/v1/files:initialize"), ("POST", "/v1/files:uploadChunks"), ("PUT", "/v1/files/{file_id}"), # Core package (src/xai/core). ("POST", "/v1/tokenize-text"), } DOC_PAGES = ( "inference/voice", "collections/collection", "collections/search", "management/auth", "management/billing", "management/audit", "files/upload", "files/manage", "files/download", "inference/other", ) GUIDES = ( "model-capabilities/audio/text-to-speech", "model-capabilities/audio/speech-to-text", "model-capabilities/audio/custom-voices", "model-capabilities/audio/speech-to-speech", "files/collections/api", ) VOICE_GUIDES = GUIDES[:3] ROW = re.compile( r'^\s*\{Call::(\w+), "([A-Z]+)", "(\w+)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)"\},?\s*$') WSROW = re.compile(r'^\s*\{"(\w+)", "(\w+)", "([^"]*)"\},?\s*$') _cache = {} def fetch(url): if url not in _cache: last = None for _ in range(3): try: with urllib.request.urlopen(url, timeout=60) as r: _cache[url] = r.read().decode("utf-8") break except Exception as e: # noqa: BLE001 last = e else: raise SystemExit("cannot fetch %s: %s" % (url, last)) return _cache[url] def block(text, begin, end): a = text.index(begin) b = text.index(end) return text[a:b].splitlines()[1:] def parse_doc_page(slug): """-> {(METHOD, path): {'body': set, 'query': set}} from a reference .md page.""" md = fetch(DOCS + "rest-api-reference/" + slug + ".md").replace("\\", "") out = {} for sec in re.split(r"(?m)^## ", md)[1:]: head, _, rest = sec.partition("\n") m = re.match(r"(GET|POST|PUT|PATCH|DELETE) (\S+)$", head.strip()) if not m: continue info = {"body": set(), "query": set()} sub = None for line in rest.split("\n"): h = re.match(r"### (.*)", line) if h: sub = h.group(1).strip() continue b = re.match(r"\* `([^`]+)` \(", line) if b and sub == "Request Body": info["body"].add(b.group(1)) elif b and sub == "Query Parameters": info["query"].add(b.group(1)) out[(m.group(1), m.group(2))] = info return out def ws_expected(doc): """Rebuild the (kind, name) set the way the table is generated.""" out = set() for q in doc["queryParameters"]: out.add(("query", q["name"])) for m in doc["clientMessages"]: out.add(("client", m["type"])) for m in doc["serverMessages"]: out.add(("server", m["type"])) return out def session_paths(schema, prefix, out): for k, v in (schema.get("properties") or {}).items(): if k == "tools": continue out.add(("session", prefix + k)) if v.get("type") == "object": session_paths(v, prefix + k + ".", out) def main(): text = open(CALLS).read() rows = [] for line in block(text, "// CALLS-BEGIN", "// CALLS-END"): m = ROW.match(line) if m: rows.append(m.groups()) bad = [] if not rows: bad.append("no rows parsed from calls.cpp") openapi = json.loads(fetch(OPENAPI)) docs = {slug: parse_doc_page(slug) for slug in DOC_PAGES} guides = {g: fetch(DOCS + g + ".md").replace("\\", "") for g in GUIDES} n_openapi = n_docs = n_guide = 0 table_keys = set() for name, http, host, path, source, body, query, guide in rows: table_keys.add((http, path)) kind, _, ref = source.partition(":") fields = [f for f in body.split(",") if f] qparams = [q for q in query.split(",") if q] if kind == "openapi": n_openapi += 1 op = (openapi["paths"].get(path) or {}).get(http.lower()) if not op: bad.append("%s: %s %s absent from openapi" % (name, http, path)) continue if op.get("operationId") != ref: bad.append("%s: operationId %s != %s" % (name, ref, op.get("operationId"))) params = {p["name"] for p in op.get("parameters", []) if p.get("in") == "query"} for q in qparams: if q not in params: bad.append("%s: query param %s absent" % (name, q)) if fields: content = op.get("requestBody", {}).get("content", {}) schema = next(iter(content.values()), {}).get("schema", {}) if "$ref" in schema: schema = openapi["components"]["schemas"][schema["$ref"].split("/")[-1]] props = set((schema.get("properties") or {}).keys()) for f in fields: if f not in props: bad.append("%s: body field %s absent" % (name, f)) elif kind == "docs": n_docs += 1 ent = docs.get(ref, {}).get((http, path)) if ent is None: bad.append("%s: %s %s absent from docs page %s" % (name, http, path, ref)) continue for f in fields: if f not in ent["body"]: bad.append("%s: body field %s absent from docs" % (name, f)) for q in qparams: if q not in ent["query"]: bad.append("%s: query param %s absent from docs" % (name, q)) if ref == "inference/voice" and not path.startswith("/v2/") and "realtime" not in path: blob = "\n".join(guides[g] for g in VOICE_GUIDES) for f in fields + [x for x in guide.split(",") if x]: if "`%s`" % f not in blob: bad.append("%s: field %s not in the voice guides" % (name, f)) elif kind == "guide": n_guide += 1 g = guides.get(ref, "") for f in [x for x in guide.split(",") if x]: if '"%s=' % f not in g and "%s=" % f not in g: bad.append("%s: guide field %s not in %s" % (name, f, ref)) if "/documents " not in g: bad.append("%s: documents upload example absent from guide" % name) else: bad.append("%s: unknown source %s" % (name, source)) # Hosts: collection and management paths live on the management host. for name, http, host, path, source, *_ in rows: if host not in ("api", "management"): bad.append("%s: bad host %s" % (name, host)) # Coverage: every openapi operation is a row or an exclusion. for path, item in openapi["paths"].items(): for method in item: if method not in ("get", "post", "put", "patch", "delete"): continue if (method.upper(), path) in table_keys: continue if path.startswith(EXCLUDED_OPENAPI_PREFIXES): continue bad.append("openapi %s %s is neither a row nor excluded" % (method.upper(), path)) # Coverage: every docs reference operation is a row or an exclusion. for slug, ops in docs.items(): for key in ops: if key in table_keys or key in EXCLUDED_DOCS: continue bad.append("docs %s %s %s is neither a row nor excluded" % (slug, key[0], key[1])) # Second source for billing and key info: xai-proto. billing = fetch(PROTO + "management_api/v1/billing.proto") rpcs = re.findall(r"(?m)^\s*rpc (\w+)\(", billing) billing_rows = [r for r in rows if r[3].startswith("/v1/billing/")] if len(rpcs) != len(billing_rows): bad.append("billing.proto has %d rpcs, table has %d billing rows" % (len(rpcs), len(billing_rows))) if "rpc get_api_key_info" not in fetch(PROTO + "api/v1/auth.proto"): bad.append("auth.proto has no get_api_key_info") # WebSocket vocabulary. wsrows = set() for line in block(open(WSCPP).read(), "// WS-BEGIN", "// WS-END"): m = WSROW.match(line) if m: wsrows.add(m.groups()) n_ws = len(wsrows) files = (("realtime", "voice-realtime"), ("tts", "tts-streaming"), ("stt", "stt-streaming")) for stream, fname in files: doc = json.loads(fetch("https://docs.x.ai/%s.ws.json" % fname)) want = ws_expected(doc) if stream == "realtime": su = [m for m in doc["clientMessages"] if m["type"] == "session.update"][0] sess = su["schema"]["properties"]["session"] session_paths(sess, "", want) for t in sess["properties"]["tools"]["items"]["properties"]["type"]["enum"]: want.add(("tool", t)) have = {(k, n) for s, k, n in wsrows if s == stream and not k.startswith("guide_")} for k, n in sorted(want - have): bad.append("ws %s: %s %s in spec but not in table" % (stream, k, n)) for k, n in sorted(have - want): bad.append("ws %s: %s %s in table but not in spec" % (stream, k, n)) sts = guides["model-capabilities/audio/speech-to-speech"] for s, k, n in wsrows: if k.startswith("guide_") and n not in sts and n.replace("_", "\\_") not in sts: bad.append("ws guide row %s %s %s not in the speech-to-speech guide" % (s, k, n)) if bad: print("FAIL: %d problems" % len(bad)) for b in bad: print(" " + b) return 1 print("OK: %d call rows (%d openapi, %d docs, %d guide), %d ws rows, " "%d billing rpcs cross-checked, openapi %d paths, docs %d pages" % (len(rows), n_openapi, n_docs, n_guide, n_ws, len(rpcs), len(openapi["paths"]), len(DOC_PAGES))) return 0 if __name__ == "__main__": sys.exit(main())