Files
moloapp/src/vendors/compat/tools/check_spec.py
T

240 lines
9.4 KiB
Python

#!/usr/bin/env python3
"""Check src/vendors/compat against OpenAI's OpenAPI document and its own SPEC.
Every compat call is an OpenAI operation (providers differ only in base URL,
auth and quirks). Fails (exit 1) when
call a row of calls.cpp has a wrong method, path, operation id, request
schema, request body field or response field (fields are resolved
through $ref, allOf, oneOf and anyOf);
coverage an operation of the spec in the families this module touches
(chat completions, embeddings, models, images, audio) is neither
in the table nor in EXCLUDED;
registry the provider ids / base URLs / credential names of registry.cpp and
the table of SPEC.md section 4 differ in either direction.
The registry check needs no network: pass --local to run only it.
Python 3 stdlib only.
python3 -I src/vendors/compat/tools/check_spec.py [--local] [path/to/openapi.json]
"""
import json
import os
import re
import sys
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.join(HERE, "..")
SPEC_SHA = "8243cc8fa6f04430fb1f1a6a2cb926fbeb4f7f26" # same pin as src/openai/core
URL = "https://raw.githubusercontent.com/openai/openai-openapi/%s/openapi.json" % SPEC_SHA
FAMILIES = ("/chat/completions", "/embeddings", "/models", "/images", "/audio", "/completions", "/moderations")
EXCLUDED = {
"listChatCompletions": "stored completions are an OpenAI product feature, not portable",
"getChatCompletion": "stored completions",
"updateChatCompletion": "stored completions",
"deleteChatCompletion": "stored completions",
"getChatCompletionMessages": "stored completions",
"createCompletion": "legacy completions; not claimed by any registry row",
"createModeration": "no registry row documents a moderation endpoint",
"createImageEdit": "multipart image editing; no registry row documents it",
"createImageVariation": "multipart image variations; no registry row documents it",
"createTranslation": "audio translation; no registry row documents it",
"createVoiceConsent": "OpenAI custom voices only",
"listVoiceConsents": "OpenAI custom voices only",
"getVoiceConsent": "OpenAI custom voices only",
"updateVoiceConsent": "OpenAI custom voices only",
"deleteVoiceConsent": "OpenAI custom voices only",
"createVoice": "OpenAI custom voices only",
"deleteVoice": "OpenAI custom voices only",
"retrieveModel": "no registry row documents GET /models/{id}; models are listed",
"deleteModel": "OpenAI fine-tune deletion only",
}
STR = re.compile(r'"((?:[^"\\]|\\.)*)"')
def string_runs(block):
"""String literals of a C++ block with adjacent literals concatenated."""
out, last_end, cur = [], None, None
for m in STR.finditer(block):
if last_end is not None and block[last_end:m.start()].strip() == "":
cur += m.group(1)
else:
if cur is not None:
out.append(cur)
cur = m.group(1)
last_end = m.end()
if cur is not None:
out.append(cur)
return out
def calls_rows():
text = open(os.path.join(ROOT, "calls.cpp")).read()
a = text.index("const CallSpec kCalls[]")
b = text.index("};", a)
rows = []
for blk in re.findall(r"\{Call::k\w+.*?Feature::k\w+, \"\w+\"\}", text[a:b], re.S):
rows.append(string_runs(blk))
return rows # method, path, op, request, fields, response, fields_read, feature_name
class Spec:
def __init__(self, doc):
self.doc = doc
def schema(self, name):
return self.doc["components"]["schemas"][name]
def deref(self, node):
while isinstance(node, dict) and "$ref" in node:
node = self.schema(node["$ref"].rsplit("/", 1)[1])
return node
def props(self, node, seen=None):
seen = seen or set()
out = set()
if isinstance(node, dict) and "$ref" in node:
ref = node["$ref"]
if ref in seen:
return out
seen.add(ref)
return self.props(self.deref(node), seen)
if not isinstance(node, dict):
return out
out |= set(node.get("properties", {}))
for key in ("allOf", "oneOf", "anyOf"):
for sub in node.get(key, []):
out |= self.props(sub, seen)
return out
def schema_name_of(self, content, want=""):
ref = content.get("schema", {})
if "$ref" in ref:
return ref["$ref"].rsplit("/", 1)[1]
# a union (anyOf / oneOf): the wanted name may be one member
for key in ("anyOf", "oneOf"):
for sub in ref.get(key, []):
if "$ref" in sub and sub["$ref"].rsplit("/", 1)[1] == want:
return want
return ""
def check_calls(spec, bad):
rows = calls_rows()
declared = set()
for method, path, op, req, fields, resp, read, _feat in rows:
node = spec.doc["paths"].get(path, {}).get(method.lower())
if not node:
bad.append("%s %s: not in the spec" % (method, path))
continue
declared.add(op)
if node["operationId"] != op:
bad.append("%s %s: operationId %s != %s" % (method, path, node["operationId"], op))
body = node.get("requestBody", {}).get("content", {})
if req:
content = body.get("application/json") or body.get("multipart/form-data") or {}
if spec.schema_name_of(content) != req:
bad.append("%s %s: request schema %s != %s" % (method, path, spec.schema_name_of(content), req))
known = spec.props(content.get("schema", {}))
for f in filter(None, fields.split(",")):
if f not in known:
bad.append("%s %s: body field %s not in %s" % (method, path, f, req))
if resp:
ok = node["responses"]["200"]["content"]["application/json"]
if spec.schema_name_of(ok, resp) != resp:
bad.append("%s %s: response schema %s != %s" % (method, path, spec.schema_name_of(ok), resp))
known = spec.props(ok.get("schema", {}))
if not known:
known = spec.props({"$ref": "#/components/schemas/" + resp})
for f in filter(None, read.split(",")):
if f not in known:
bad.append("%s %s: response field %s not in %s" % (method, path, f, resp))
for path, item in spec.doc["paths"].items():
if not path.startswith(FAMILIES):
continue
for m, node in item.items():
if m not in ("get", "post", "delete"):
continue
op = node["operationId"]
if op not in declared and op not in EXCLUDED:
bad.append("spec operation %s %s (%s) is neither in the table nor excluded" % (m.upper(), path, op))
for op in EXCLUDED:
if op in declared:
bad.append("%s is both declared and excluded" % op)
return len(rows)
def registry_rows():
text = open(os.path.join(ROOT, "registry.cpp")).read()
a = text.index("const ProviderRow kRows[]")
b = text.index("constexpr std::size_t kRowCount")
out = []
for blk in re.findall(r'\n \{"[a-z0-9]+",.*?(?=\n \{"|\n //|\n\};)', text[a:b], re.S):
s = string_runs(blk)
out.append({"id": s[0], "name": s[1], "base": s[2], "key": s[3], "base_env": s[4], "extra_env": s[5],
"models": s[6], "doc": s[7]})
return out
def spec_md_rows():
text = open(os.path.join(ROOT, "SPEC.md")).read()
a = text.index("## 4. Provider table")
b = text.index("\n## ", a + 5)
rows = {}
for line in text[a:b].splitlines():
cells = [c.strip().strip("`") for c in line.strip().strip("|").split("|")]
if line.startswith("|") and len(cells) >= 5 and re.fullmatch(r"[a-z0-9]+", cells[0]) and cells[0] != "id":
rows[cells[0]] = cells
return rows
def check_registry(bad):
reg = registry_rows()
md = spec_md_rows()
ids = [r["id"] for r in reg]
if len(ids) != len(set(ids)):
bad.append("registry.cpp has duplicate ids")
for r in reg:
cells = md.get(r["id"])
if not cells:
bad.append("provider %s is in registry.cpp but not in SPEC.md section 4" % r["id"])
continue
line = " ".join(cells)
for what in ("base", "key", "doc"):
if r[what] not in line:
bad.append("provider %s: %s '%s' differs between registry.cpp and SPEC.md" % (r["id"], what, r[what]))
for i in md:
if i not in ids:
bad.append("provider %s is in SPEC.md section 4 but not in registry.cpp" % i)
return len(reg)
def main():
args = [a for a in sys.argv[1:] if not a.startswith("--")]
local = "--local" in sys.argv
bad = []
n_reg = check_registry(bad)
n_calls = 0
if not local:
if args:
doc = json.load(open(args[0]))
else:
req = urllib.request.Request(URL, headers={"User-Agent": "molo-compat-spec-check/1"})
with urllib.request.urlopen(req, timeout=120) as r:
doc = json.loads(r.read().decode("utf-8"))
n_calls = check_calls(Spec(doc), bad)
print("openapi %s (%s)" % (doc["info"]["version"], SPEC_SHA[:12]))
print("registry rows: %d, call rows checked: %d" % (n_reg, n_calls))
if bad:
print("FAIL")
for b in bad:
print(" " + b)
sys.exit(1)
print("OK")
if __name__ == "__main__":
main()