276 lines
11 KiB
Python
276 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Check the xAI platform call tables against xAI's published specs.
|
|
|
|
Fails (exit 1) when a declared call is absent from its source, a request
|
|
field or query parameter we use is not in the spec, a WebSocket message,
|
|
query parameter or session field in ws_events.cpp differs from the three
|
|
docs.x.ai .ws.json documents, or a spec operation in our areas is neither in
|
|
the table nor in an explicit exclusion list. Python 3 stdlib only, no
|
|
credentials.
|
|
|
|
python3 -I src/xai/platform/tools/check_spec.py
|
|
|
|
Sources (all public, read at run time):
|
|
openapi https://api.x.ai/api-docs/openapi.json (also docs.x.ai/openapi.json)
|
|
docs https://docs.x.ai/developers/rest-api-reference/<page>.md
|
|
guide https://docs.x.ai/developers/<page>.md (prose, cross-checks)
|
|
ws https://docs.x.ai/{voice-realtime,tts-streaming,stt-streaming}.ws.json
|
|
proto xai-org/xai-proto billing.proto and auth.proto (second source for billing)
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import urllib.request
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
CALLS = os.path.join(HERE, "..", "calls.cpp")
|
|
WSCPP = os.path.join(HERE, "..", "ws_events.cpp")
|
|
DOCS = "https://docs.x.ai/developers/"
|
|
OPENAPI = "https://api.x.ai/api-docs/openapi.json"
|
|
PROTO = "https://raw.githubusercontent.com/xai-org/xai-proto/main/proto/xai/"
|
|
|
|
# Operations that belong to other packages or are deliberately not wrapped.
|
|
EXCLUDED_OPENAPI_PREFIXES = (
|
|
"/v1/chat/", "/v1/complete", "/v1/completions","/v1/embedding", "/v1/image", "/v1/language-models",
|
|
"/v1/messages", "/v1/models", "/v1/responses", "/v1/tokenize-text",
|
|
"/v1/video", "/v1/skills",
|
|
)
|
|
EXCLUDED_DOCS = {
|
|
# Documented without a request or response shape: nothing to verify against.
|
|
("POST", "/v1/files:initialize"),
|
|
("POST", "/v1/files:uploadChunks"),
|
|
("PUT", "/v1/files/{file_id}"),
|
|
# Core package (src/xai/core).
|
|
("POST", "/v1/tokenize-text"),
|
|
}
|
|
DOC_PAGES = (
|
|
"inference/voice", "collections/collection", "collections/search",
|
|
"management/auth", "management/billing", "management/audit",
|
|
"files/upload", "files/manage", "files/download", "inference/other",
|
|
)
|
|
GUIDES = (
|
|
"model-capabilities/audio/text-to-speech", "model-capabilities/audio/speech-to-text",
|
|
"model-capabilities/audio/custom-voices", "model-capabilities/audio/speech-to-speech",
|
|
"files/collections/api",
|
|
)
|
|
VOICE_GUIDES = GUIDES[:3]
|
|
|
|
ROW = re.compile(
|
|
r'^\s*\{Call::(\w+), "([A-Z]+)", "(\w+)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)"\},?\s*$')
|
|
WSROW = re.compile(r'^\s*\{"(\w+)", "(\w+)", "([^"]*)"\},?\s*$')
|
|
|
|
_cache = {}
|
|
|
|
|
|
def fetch(url):
|
|
if url not in _cache:
|
|
last = None
|
|
for _ in range(3):
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=60) as r:
|
|
_cache[url] = r.read().decode("utf-8")
|
|
break
|
|
except Exception as e: # noqa: BLE001
|
|
last = e
|
|
else:
|
|
raise SystemExit("cannot fetch %s: %s" % (url, last))
|
|
return _cache[url]
|
|
|
|
|
|
def block(text, begin, end):
|
|
a = text.index(begin)
|
|
b = text.index(end)
|
|
return text[a:b].splitlines()[1:]
|
|
|
|
|
|
def parse_doc_page(slug):
|
|
"""-> {(METHOD, path): {'body': set, 'query': set}} from a reference .md page."""
|
|
md = fetch(DOCS + "rest-api-reference/" + slug + ".md").replace("\\", "")
|
|
out = {}
|
|
for sec in re.split(r"(?m)^## ", md)[1:]:
|
|
head, _, rest = sec.partition("\n")
|
|
m = re.match(r"(GET|POST|PUT|PATCH|DELETE) (\S+)$", head.strip())
|
|
if not m:
|
|
continue
|
|
info = {"body": set(), "query": set()}
|
|
sub = None
|
|
for line in rest.split("\n"):
|
|
h = re.match(r"### (.*)", line)
|
|
if h:
|
|
sub = h.group(1).strip()
|
|
continue
|
|
b = re.match(r"\* `([^`]+)` \(", line)
|
|
if b and sub == "Request Body":
|
|
info["body"].add(b.group(1))
|
|
elif b and sub == "Query Parameters":
|
|
info["query"].add(b.group(1))
|
|
out[(m.group(1), m.group(2))] = info
|
|
return out
|
|
|
|
|
|
def ws_expected(doc):
|
|
"""Rebuild the (kind, name) set the way the table is generated."""
|
|
out = set()
|
|
for q in doc["queryParameters"]:
|
|
out.add(("query", q["name"]))
|
|
for m in doc["clientMessages"]:
|
|
out.add(("client", m["type"]))
|
|
for m in doc["serverMessages"]:
|
|
out.add(("server", m["type"]))
|
|
return out
|
|
|
|
|
|
def session_paths(schema, prefix, out):
|
|
for k, v in (schema.get("properties") or {}).items():
|
|
if k == "tools":
|
|
continue
|
|
out.add(("session", prefix + k))
|
|
if v.get("type") == "object":
|
|
session_paths(v, prefix + k + ".", out)
|
|
|
|
|
|
def main():
|
|
text = open(CALLS).read()
|
|
rows = []
|
|
for line in block(text, "// CALLS-BEGIN", "// CALLS-END"):
|
|
m = ROW.match(line)
|
|
if m:
|
|
rows.append(m.groups())
|
|
bad = []
|
|
if not rows:
|
|
bad.append("no rows parsed from calls.cpp")
|
|
|
|
openapi = json.loads(fetch(OPENAPI))
|
|
docs = {slug: parse_doc_page(slug) for slug in DOC_PAGES}
|
|
guides = {g: fetch(DOCS + g + ".md").replace("\\", "") for g in GUIDES}
|
|
|
|
n_openapi = n_docs = n_guide = 0
|
|
table_keys = set()
|
|
for name, http, host, path, source, body, query, guide in rows:
|
|
table_keys.add((http, path))
|
|
kind, _, ref = source.partition(":")
|
|
fields = [f for f in body.split(",") if f]
|
|
qparams = [q for q in query.split(",") if q]
|
|
if kind == "openapi":
|
|
n_openapi += 1
|
|
op = (openapi["paths"].get(path) or {}).get(http.lower())
|
|
if not op:
|
|
bad.append("%s: %s %s absent from openapi" % (name, http, path))
|
|
continue
|
|
if op.get("operationId") != ref:
|
|
bad.append("%s: operationId %s != %s" % (name, ref, op.get("operationId")))
|
|
params = {p["name"] for p in op.get("parameters", []) if p.get("in") == "query"}
|
|
for q in qparams:
|
|
if q not in params:
|
|
bad.append("%s: query param %s absent" % (name, q))
|
|
if fields:
|
|
content = op.get("requestBody", {}).get("content", {})
|
|
schema = next(iter(content.values()), {}).get("schema", {})
|
|
if "$ref" in schema:
|
|
schema = openapi["components"]["schemas"][schema["$ref"].split("/")[-1]]
|
|
props = set((schema.get("properties") or {}).keys())
|
|
for f in fields:
|
|
if f not in props:
|
|
bad.append("%s: body field %s absent" % (name, f))
|
|
elif kind == "docs":
|
|
n_docs += 1
|
|
ent = docs.get(ref, {}).get((http, path))
|
|
if ent is None:
|
|
bad.append("%s: %s %s absent from docs page %s" % (name, http, path, ref))
|
|
continue
|
|
for f in fields:
|
|
if f not in ent["body"]:
|
|
bad.append("%s: body field %s absent from docs" % (name, f))
|
|
for q in qparams:
|
|
if q not in ent["query"]:
|
|
bad.append("%s: query param %s absent from docs" % (name, q))
|
|
if ref == "inference/voice" and not path.startswith("/v2/") and "realtime" not in path:
|
|
blob = "\n".join(guides[g] for g in VOICE_GUIDES)
|
|
for f in fields + [x for x in guide.split(",") if x]:
|
|
if "`%s`" % f not in blob:
|
|
bad.append("%s: field %s not in the voice guides" % (name, f))
|
|
elif kind == "guide":
|
|
n_guide += 1
|
|
g = guides.get(ref, "")
|
|
for f in [x for x in guide.split(",") if x]:
|
|
if '"%s=' % f not in g and "%s=" % f not in g:
|
|
bad.append("%s: guide field %s not in %s" % (name, f, ref))
|
|
if "/documents " not in g:
|
|
bad.append("%s: documents upload example absent from guide" % name)
|
|
else:
|
|
bad.append("%s: unknown source %s" % (name, source))
|
|
|
|
# Hosts: collection and management paths live on the management host.
|
|
for name, http, host, path, source, *_ in rows:
|
|
if host not in ("api", "management"):
|
|
bad.append("%s: bad host %s" % (name, host))
|
|
|
|
# Coverage: every openapi operation is a row or an exclusion.
|
|
for path, item in openapi["paths"].items():
|
|
for method in item:
|
|
if method not in ("get", "post", "put", "patch", "delete"):
|
|
continue
|
|
if (method.upper(), path) in table_keys:
|
|
continue
|
|
if path.startswith(EXCLUDED_OPENAPI_PREFIXES):
|
|
continue
|
|
bad.append("openapi %s %s is neither a row nor excluded" % (method.upper(), path))
|
|
# Coverage: every docs reference operation is a row or an exclusion.
|
|
for slug, ops in docs.items():
|
|
for key in ops:
|
|
if key in table_keys or key in EXCLUDED_DOCS:
|
|
continue
|
|
bad.append("docs %s %s %s is neither a row nor excluded" % (slug, key[0], key[1]))
|
|
|
|
# Second source for billing and key info: xai-proto.
|
|
billing = fetch(PROTO + "management_api/v1/billing.proto")
|
|
rpcs = re.findall(r"(?m)^\s*rpc (\w+)\(", billing)
|
|
billing_rows = [r for r in rows if r[3].startswith("/v1/billing/")]
|
|
if len(rpcs) != len(billing_rows):
|
|
bad.append("billing.proto has %d rpcs, table has %d billing rows" % (len(rpcs), len(billing_rows)))
|
|
if "rpc get_api_key_info" not in fetch(PROTO + "api/v1/auth.proto"):
|
|
bad.append("auth.proto has no get_api_key_info")
|
|
|
|
# WebSocket vocabulary.
|
|
wsrows = set()
|
|
for line in block(open(WSCPP).read(), "// WS-BEGIN", "// WS-END"):
|
|
m = WSROW.match(line)
|
|
if m:
|
|
wsrows.add(m.groups())
|
|
n_ws = len(wsrows)
|
|
files = (("realtime", "voice-realtime"), ("tts", "tts-streaming"), ("stt", "stt-streaming"))
|
|
for stream, fname in files:
|
|
doc = json.loads(fetch("https://docs.x.ai/%s.ws.json" % fname))
|
|
want = ws_expected(doc)
|
|
if stream == "realtime":
|
|
su = [m for m in doc["clientMessages"] if m["type"] == "session.update"][0]
|
|
sess = su["schema"]["properties"]["session"]
|
|
session_paths(sess, "", want)
|
|
for t in sess["properties"]["tools"]["items"]["properties"]["type"]["enum"]:
|
|
want.add(("tool", t))
|
|
have = {(k, n) for s, k, n in wsrows if s == stream and not k.startswith("guide_")}
|
|
for k, n in sorted(want - have):
|
|
bad.append("ws %s: %s %s in spec but not in table" % (stream, k, n))
|
|
for k, n in sorted(have - want):
|
|
bad.append("ws %s: %s %s in table but not in spec" % (stream, k, n))
|
|
sts = guides["model-capabilities/audio/speech-to-speech"]
|
|
for s, k, n in wsrows:
|
|
if k.startswith("guide_") and n not in sts and n.replace("_", "\\_") not in sts:
|
|
bad.append("ws guide row %s %s %s not in the speech-to-speech guide" % (s, k, n))
|
|
|
|
if bad:
|
|
print("FAIL: %d problems" % len(bad))
|
|
for b in bad:
|
|
print(" " + b)
|
|
return 1
|
|
print("OK: %d call rows (%d openapi, %d docs, %d guide), %d ws rows, "
|
|
"%d billing rpcs cross-checked, openapi %d paths, docs %d pages"
|
|
% (len(rows), n_openapi, n_docs, n_guide, n_ws, len(rpcs), len(openapi["paths"]),
|
|
len(DOC_PAGES)))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|