#!/usr/bin/env python3 """Check src/vendors/compat against OpenAI's OpenAPI document and its own SPEC. Every compat call is an OpenAI operation (providers differ only in base URL, auth and quirks). Fails (exit 1) when call a row of calls.cpp has a wrong method, path, operation id, request schema, request body field or response field (fields are resolved through $ref, allOf, oneOf and anyOf); coverage an operation of the spec in the families this module touches (chat completions, embeddings, models, images, audio) is neither in the table nor in EXCLUDED; registry the provider ids / base URLs / credential names of registry.cpp and the table of SPEC.md section 4 differ in either direction. The registry check needs no network: pass --local to run only it. Python 3 stdlib only. python3 -I src/vendors/compat/tools/check_spec.py [--local] [path/to/openapi.json] """ import json import os import re import sys import urllib.request HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.join(HERE, "..") SPEC_SHA = "8243cc8fa6f04430fb1f1a6a2cb926fbeb4f7f26" # same pin as src/openai/core URL = "https://raw.githubusercontent.com/openai/openai-openapi/%s/openapi.json" % SPEC_SHA FAMILIES = ("/chat/completions", "/embeddings", "/models", "/images", "/audio", "/completions", "/moderations") EXCLUDED = { "listChatCompletions": "stored completions are an OpenAI product feature, not portable", "getChatCompletion": "stored completions", "updateChatCompletion": "stored completions", "deleteChatCompletion": "stored completions", "getChatCompletionMessages": "stored completions", "createCompletion": "legacy completions; not claimed by any registry row", "createModeration": "no registry row documents a moderation endpoint", "createImageEdit": "multipart image editing; no registry row documents it", "createImageVariation": "multipart image variations; no registry row documents it", "createTranslation": "audio translation; no registry row documents it", "createVoiceConsent": "OpenAI custom voices only", "listVoiceConsents": "OpenAI custom voices only", "getVoiceConsent": "OpenAI custom voices only", "updateVoiceConsent": "OpenAI custom voices only", "deleteVoiceConsent": "OpenAI custom voices only", "createVoice": "OpenAI custom voices only", "deleteVoice": "OpenAI custom voices only", "retrieveModel": "no registry row documents GET /models/{id}; models are listed", "deleteModel": "OpenAI fine-tune deletion only", } STR = re.compile(r'"((?:[^"\\]|\\.)*)"') def string_runs(block): """String literals of a C++ block with adjacent literals concatenated.""" out, last_end, cur = [], None, None for m in STR.finditer(block): if last_end is not None and block[last_end:m.start()].strip() == "": cur += m.group(1) else: if cur is not None: out.append(cur) cur = m.group(1) last_end = m.end() if cur is not None: out.append(cur) return out def calls_rows(): text = open(os.path.join(ROOT, "calls.cpp")).read() a = text.index("const CallSpec kCalls[]") b = text.index("};", a) rows = [] for blk in re.findall(r"\{Call::k\w+.*?Feature::k\w+, \"\w+\"\}", text[a:b], re.S): rows.append(string_runs(blk)) return rows # method, path, op, request, fields, response, fields_read, feature_name class Spec: def __init__(self, doc): self.doc = doc def schema(self, name): return self.doc["components"]["schemas"][name] def deref(self, node): while isinstance(node, dict) and "$ref" in node: node = self.schema(node["$ref"].rsplit("/", 1)[1]) return node def props(self, node, seen=None): seen = seen or set() out = set() if isinstance(node, dict) and "$ref" in node: ref = node["$ref"] if ref in seen: return out seen.add(ref) return self.props(self.deref(node), seen) if not isinstance(node, dict): return out out |= set(node.get("properties", {})) for key in ("allOf", "oneOf", "anyOf"): for sub in node.get(key, []): out |= self.props(sub, seen) return out def schema_name_of(self, content, want=""): ref = content.get("schema", {}) if "$ref" in ref: return ref["$ref"].rsplit("/", 1)[1] # a union (anyOf / oneOf): the wanted name may be one member for key in ("anyOf", "oneOf"): for sub in ref.get(key, []): if "$ref" in sub and sub["$ref"].rsplit("/", 1)[1] == want: return want return "" def check_calls(spec, bad): rows = calls_rows() declared = set() for method, path, op, req, fields, resp, read, _feat in rows: node = spec.doc["paths"].get(path, {}).get(method.lower()) if not node: bad.append("%s %s: not in the spec" % (method, path)) continue declared.add(op) if node["operationId"] != op: bad.append("%s %s: operationId %s != %s" % (method, path, node["operationId"], op)) body = node.get("requestBody", {}).get("content", {}) if req: content = body.get("application/json") or body.get("multipart/form-data") or {} if spec.schema_name_of(content) != req: bad.append("%s %s: request schema %s != %s" % (method, path, spec.schema_name_of(content), req)) known = spec.props(content.get("schema", {})) for f in filter(None, fields.split(",")): if f not in known: bad.append("%s %s: body field %s not in %s" % (method, path, f, req)) if resp: ok = node["responses"]["200"]["content"]["application/json"] if spec.schema_name_of(ok, resp) != resp: bad.append("%s %s: response schema %s != %s" % (method, path, spec.schema_name_of(ok), resp)) known = spec.props(ok.get("schema", {})) if not known: known = spec.props({"$ref": "#/components/schemas/" + resp}) for f in filter(None, read.split(",")): if f not in known: bad.append("%s %s: response field %s not in %s" % (method, path, f, resp)) for path, item in spec.doc["paths"].items(): if not path.startswith(FAMILIES): continue for m, node in item.items(): if m not in ("get", "post", "delete"): continue op = node["operationId"] if op not in declared and op not in EXCLUDED: bad.append("spec operation %s %s (%s) is neither in the table nor excluded" % (m.upper(), path, op)) for op in EXCLUDED: if op in declared: bad.append("%s is both declared and excluded" % op) return len(rows) def registry_rows(): text = open(os.path.join(ROOT, "registry.cpp")).read() a = text.index("const ProviderRow kRows[]") b = text.index("constexpr std::size_t kRowCount") out = [] for blk in re.findall(r'\n \{"[a-z0-9]+",.*?(?=\n \{"|\n //|\n\};)', text[a:b], re.S): s = string_runs(blk) out.append({"id": s[0], "name": s[1], "base": s[2], "key": s[3], "base_env": s[4], "extra_env": s[5], "models": s[6], "doc": s[7]}) return out def spec_md_rows(): text = open(os.path.join(ROOT, "SPEC.md")).read() a = text.index("## 4. Provider table") b = text.index("\n## ", a + 5) rows = {} for line in text[a:b].splitlines(): cells = [c.strip().strip("`") for c in line.strip().strip("|").split("|")] if line.startswith("|") and len(cells) >= 5 and re.fullmatch(r"[a-z0-9]+", cells[0]) and cells[0] != "id": rows[cells[0]] = cells return rows def check_registry(bad): reg = registry_rows() md = spec_md_rows() ids = [r["id"] for r in reg] if len(ids) != len(set(ids)): bad.append("registry.cpp has duplicate ids") for r in reg: cells = md.get(r["id"]) if not cells: bad.append("provider %s is in registry.cpp but not in SPEC.md section 4" % r["id"]) continue line = " ".join(cells) for what in ("base", "key", "doc"): if r[what] not in line: bad.append("provider %s: %s '%s' differs between registry.cpp and SPEC.md" % (r["id"], what, r[what])) for i in md: if i not in ids: bad.append("provider %s is in SPEC.md section 4 but not in registry.cpp" % i) return len(reg) def main(): args = [a for a in sys.argv[1:] if not a.startswith("--")] local = "--local" in sys.argv bad = [] n_reg = check_registry(bad) n_calls = 0 if not local: if args: doc = json.load(open(args[0])) else: req = urllib.request.Request(URL, headers={"User-Agent": "molo-compat-spec-check/1"}) with urllib.request.urlopen(req, timeout=120) as r: doc = json.loads(r.read().decode("utf-8")) n_calls = check_calls(Spec(doc), bad) print("openapi %s (%s)" % (doc["info"]["version"], SPEC_SHA[:12])) print("registry rows: %d, call rows checked: %d" % (n_reg, n_calls)) if bad: print("FAIL") for b in bad: print(" " + b) sys.exit(1) print("OK") if __name__ == "__main__": main()