Files
moloapp/src/xai/platform/tools/check_spec.py
T

276 lines
11 KiB
Python

#!/usr/bin/env python3
"""Check the xAI platform call tables against xAI's published specs.
Fails (exit 1) when a declared call is absent from its source, a request
field or query parameter we use is not in the spec, a WebSocket message,
query parameter or session field in ws_events.cpp differs from the three
docs.x.ai .ws.json documents, or a spec operation in our areas is neither in
the table nor in an explicit exclusion list. Python 3 stdlib only, no
credentials.
python3 -I src/xai/platform/tools/check_spec.py
Sources (all public, read at run time):
openapi https://api.x.ai/api-docs/openapi.json (also docs.x.ai/openapi.json)
docs https://docs.x.ai/developers/rest-api-reference/<page>.md
guide https://docs.x.ai/developers/<page>.md (prose, cross-checks)
ws https://docs.x.ai/{voice-realtime,tts-streaming,stt-streaming}.ws.json
proto xai-org/xai-proto billing.proto and auth.proto (second source for billing)
"""
import json
import os
import re
import sys
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
CALLS = os.path.join(HERE, "..", "calls.cpp")
WSCPP = os.path.join(HERE, "..", "ws_events.cpp")
DOCS = "https://docs.x.ai/developers/"
OPENAPI = "https://api.x.ai/api-docs/openapi.json"
PROTO = "https://raw.githubusercontent.com/xai-org/xai-proto/main/proto/xai/"
# Operations that belong to other packages or are deliberately not wrapped.
EXCLUDED_OPENAPI_PREFIXES = (
"/v1/chat/", "/v1/complete", "/v1/completions","/v1/embedding", "/v1/image", "/v1/language-models",
"/v1/messages", "/v1/models", "/v1/responses", "/v1/tokenize-text",
"/v1/video", "/v1/skills",
)
EXCLUDED_DOCS = {
# Documented without a request or response shape: nothing to verify against.
("POST", "/v1/files:initialize"),
("POST", "/v1/files:uploadChunks"),
("PUT", "/v1/files/{file_id}"),
# Core package (src/xai/core).
("POST", "/v1/tokenize-text"),
}
DOC_PAGES = (
"inference/voice", "collections/collection", "collections/search",
"management/auth", "management/billing", "management/audit",
"files/upload", "files/manage", "files/download", "inference/other",
)
GUIDES = (
"model-capabilities/audio/text-to-speech", "model-capabilities/audio/speech-to-text",
"model-capabilities/audio/custom-voices", "model-capabilities/audio/speech-to-speech",
"files/collections/api",
)
VOICE_GUIDES = GUIDES[:3]
ROW = re.compile(
r'^\s*\{Call::(\w+), "([A-Z]+)", "(\w+)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)"\},?\s*$')
WSROW = re.compile(r'^\s*\{"(\w+)", "(\w+)", "([^"]*)"\},?\s*$')
_cache = {}
def fetch(url):
if url not in _cache:
last = None
for _ in range(3):
try:
with urllib.request.urlopen(url, timeout=60) as r:
_cache[url] = r.read().decode("utf-8")
break
except Exception as e: # noqa: BLE001
last = e
else:
raise SystemExit("cannot fetch %s: %s" % (url, last))
return _cache[url]
def block(text, begin, end):
a = text.index(begin)
b = text.index(end)
return text[a:b].splitlines()[1:]
def parse_doc_page(slug):
"""-> {(METHOD, path): {'body': set, 'query': set}} from a reference .md page."""
md = fetch(DOCS + "rest-api-reference/" + slug + ".md").replace("\\", "")
out = {}
for sec in re.split(r"(?m)^## ", md)[1:]:
head, _, rest = sec.partition("\n")
m = re.match(r"(GET|POST|PUT|PATCH|DELETE) (\S+)$", head.strip())
if not m:
continue
info = {"body": set(), "query": set()}
sub = None
for line in rest.split("\n"):
h = re.match(r"### (.*)", line)
if h:
sub = h.group(1).strip()
continue
b = re.match(r"\* `([^`]+)` \(", line)
if b and sub == "Request Body":
info["body"].add(b.group(1))
elif b and sub == "Query Parameters":
info["query"].add(b.group(1))
out[(m.group(1), m.group(2))] = info
return out
def ws_expected(doc):
"""Rebuild the (kind, name) set the way the table is generated."""
out = set()
for q in doc["queryParameters"]:
out.add(("query", q["name"]))
for m in doc["clientMessages"]:
out.add(("client", m["type"]))
for m in doc["serverMessages"]:
out.add(("server", m["type"]))
return out
def session_paths(schema, prefix, out):
for k, v in (schema.get("properties") or {}).items():
if k == "tools":
continue
out.add(("session", prefix + k))
if v.get("type") == "object":
session_paths(v, prefix + k + ".", out)
def main():
text = open(CALLS).read()
rows = []
for line in block(text, "// CALLS-BEGIN", "// CALLS-END"):
m = ROW.match(line)
if m:
rows.append(m.groups())
bad = []
if not rows:
bad.append("no rows parsed from calls.cpp")
openapi = json.loads(fetch(OPENAPI))
docs = {slug: parse_doc_page(slug) for slug in DOC_PAGES}
guides = {g: fetch(DOCS + g + ".md").replace("\\", "") for g in GUIDES}
n_openapi = n_docs = n_guide = 0
table_keys = set()
for name, http, host, path, source, body, query, guide in rows:
table_keys.add((http, path))
kind, _, ref = source.partition(":")
fields = [f for f in body.split(",") if f]
qparams = [q for q in query.split(",") if q]
if kind == "openapi":
n_openapi += 1
op = (openapi["paths"].get(path) or {}).get(http.lower())
if not op:
bad.append("%s: %s %s absent from openapi" % (name, http, path))
continue
if op.get("operationId") != ref:
bad.append("%s: operationId %s != %s" % (name, ref, op.get("operationId")))
params = {p["name"] for p in op.get("parameters", []) if p.get("in") == "query"}
for q in qparams:
if q not in params:
bad.append("%s: query param %s absent" % (name, q))
if fields:
content = op.get("requestBody", {}).get("content", {})
schema = next(iter(content.values()), {}).get("schema", {})
if "$ref" in schema:
schema = openapi["components"]["schemas"][schema["$ref"].split("/")[-1]]
props = set((schema.get("properties") or {}).keys())
for f in fields:
if f not in props:
bad.append("%s: body field %s absent" % (name, f))
elif kind == "docs":
n_docs += 1
ent = docs.get(ref, {}).get((http, path))
if ent is None:
bad.append("%s: %s %s absent from docs page %s" % (name, http, path, ref))
continue
for f in fields:
if f not in ent["body"]:
bad.append("%s: body field %s absent from docs" % (name, f))
for q in qparams:
if q not in ent["query"]:
bad.append("%s: query param %s absent from docs" % (name, q))
if ref == "inference/voice" and not path.startswith("/v2/") and "realtime" not in path:
blob = "\n".join(guides[g] for g in VOICE_GUIDES)
for f in fields + [x for x in guide.split(",") if x]:
if "`%s`" % f not in blob:
bad.append("%s: field %s not in the voice guides" % (name, f))
elif kind == "guide":
n_guide += 1
g = guides.get(ref, "")
for f in [x for x in guide.split(",") if x]:
if '"%s=' % f not in g and "%s=" % f not in g:
bad.append("%s: guide field %s not in %s" % (name, f, ref))
if "/documents " not in g:
bad.append("%s: documents upload example absent from guide" % name)
else:
bad.append("%s: unknown source %s" % (name, source))
# Hosts: collection and management paths live on the management host.
for name, http, host, path, source, *_ in rows:
if host not in ("api", "management"):
bad.append("%s: bad host %s" % (name, host))
# Coverage: every openapi operation is a row or an exclusion.
for path, item in openapi["paths"].items():
for method in item:
if method not in ("get", "post", "put", "patch", "delete"):
continue
if (method.upper(), path) in table_keys:
continue
if path.startswith(EXCLUDED_OPENAPI_PREFIXES):
continue
bad.append("openapi %s %s is neither a row nor excluded" % (method.upper(), path))
# Coverage: every docs reference operation is a row or an exclusion.
for slug, ops in docs.items():
for key in ops:
if key in table_keys or key in EXCLUDED_DOCS:
continue
bad.append("docs %s %s %s is neither a row nor excluded" % (slug, key[0], key[1]))
# Second source for billing and key info: xai-proto.
billing = fetch(PROTO + "management_api/v1/billing.proto")
rpcs = re.findall(r"(?m)^\s*rpc (\w+)\(", billing)
billing_rows = [r for r in rows if r[3].startswith("/v1/billing/")]
if len(rpcs) != len(billing_rows):
bad.append("billing.proto has %d rpcs, table has %d billing rows" % (len(rpcs), len(billing_rows)))
if "rpc get_api_key_info" not in fetch(PROTO + "api/v1/auth.proto"):
bad.append("auth.proto has no get_api_key_info")
# WebSocket vocabulary.
wsrows = set()
for line in block(open(WSCPP).read(), "// WS-BEGIN", "// WS-END"):
m = WSROW.match(line)
if m:
wsrows.add(m.groups())
n_ws = len(wsrows)
files = (("realtime", "voice-realtime"), ("tts", "tts-streaming"), ("stt", "stt-streaming"))
for stream, fname in files:
doc = json.loads(fetch("https://docs.x.ai/%s.ws.json" % fname))
want = ws_expected(doc)
if stream == "realtime":
su = [m for m in doc["clientMessages"] if m["type"] == "session.update"][0]
sess = su["schema"]["properties"]["session"]
session_paths(sess, "", want)
for t in sess["properties"]["tools"]["items"]["properties"]["type"]["enum"]:
want.add(("tool", t))
have = {(k, n) for s, k, n in wsrows if s == stream and not k.startswith("guide_")}
for k, n in sorted(want - have):
bad.append("ws %s: %s %s in spec but not in table" % (stream, k, n))
for k, n in sorted(have - want):
bad.append("ws %s: %s %s in table but not in spec" % (stream, k, n))
sts = guides["model-capabilities/audio/speech-to-speech"]
for s, k, n in wsrows:
if k.startswith("guide_") and n not in sts and n.replace("_", "\\_") not in sts:
bad.append("ws guide row %s %s %s not in the speech-to-speech guide" % (s, k, n))
if bad:
print("FAIL: %d problems" % len(bad))
for b in bad:
print(" " + b)
return 1
print("OK: %d call rows (%d openapi, %d docs, %d guide), %d ws rows, "
"%d billing rpcs cross-checked, openapi %d paths, docs %d pages"
% (len(rows), n_openapi, n_docs, n_guide, n_ws, len(rpcs), len(openapi["paths"]),
len(DOC_PAGES)))
return 0
if __name__ == "__main__":
sys.exit(main())