240 lines
9.4 KiB
Python
240 lines
9.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Check src/vendors/compat against OpenAI's OpenAPI document and its own SPEC.
|
|
|
|
Every compat call is an OpenAI operation (providers differ only in base URL,
|
|
auth and quirks). Fails (exit 1) when
|
|
|
|
call a row of calls.cpp has a wrong method, path, operation id, request
|
|
schema, request body field or response field (fields are resolved
|
|
through $ref, allOf, oneOf and anyOf);
|
|
coverage an operation of the spec in the families this module touches
|
|
(chat completions, embeddings, models, images, audio) is neither
|
|
in the table nor in EXCLUDED;
|
|
registry the provider ids / base URLs / credential names of registry.cpp and
|
|
the table of SPEC.md section 4 differ in either direction.
|
|
|
|
The registry check needs no network: pass --local to run only it.
|
|
Python 3 stdlib only.
|
|
|
|
python3 -I src/vendors/compat/tools/check_spec.py [--local] [path/to/openapi.json]
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import urllib.request
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = os.path.join(HERE, "..")
|
|
SPEC_SHA = "8243cc8fa6f04430fb1f1a6a2cb926fbeb4f7f26" # same pin as src/openai/core
|
|
URL = "https://raw.githubusercontent.com/openai/openai-openapi/%s/openapi.json" % SPEC_SHA
|
|
|
|
FAMILIES = ("/chat/completions", "/embeddings", "/models", "/images", "/audio", "/completions", "/moderations")
|
|
EXCLUDED = {
|
|
"listChatCompletions": "stored completions are an OpenAI product feature, not portable",
|
|
"getChatCompletion": "stored completions",
|
|
"updateChatCompletion": "stored completions",
|
|
"deleteChatCompletion": "stored completions",
|
|
"getChatCompletionMessages": "stored completions",
|
|
"createCompletion": "legacy completions; not claimed by any registry row",
|
|
"createModeration": "no registry row documents a moderation endpoint",
|
|
"createImageEdit": "multipart image editing; no registry row documents it",
|
|
"createImageVariation": "multipart image variations; no registry row documents it",
|
|
"createTranslation": "audio translation; no registry row documents it",
|
|
"createVoiceConsent": "OpenAI custom voices only",
|
|
"listVoiceConsents": "OpenAI custom voices only",
|
|
"getVoiceConsent": "OpenAI custom voices only",
|
|
"updateVoiceConsent": "OpenAI custom voices only",
|
|
"deleteVoiceConsent": "OpenAI custom voices only",
|
|
"createVoice": "OpenAI custom voices only",
|
|
"deleteVoice": "OpenAI custom voices only",
|
|
"retrieveModel": "no registry row documents GET /models/{id}; models are listed",
|
|
"deleteModel": "OpenAI fine-tune deletion only",
|
|
}
|
|
STR = re.compile(r'"((?:[^"\\]|\\.)*)"')
|
|
|
|
|
|
def string_runs(block):
|
|
"""String literals of a C++ block with adjacent literals concatenated."""
|
|
out, last_end, cur = [], None, None
|
|
for m in STR.finditer(block):
|
|
if last_end is not None and block[last_end:m.start()].strip() == "":
|
|
cur += m.group(1)
|
|
else:
|
|
if cur is not None:
|
|
out.append(cur)
|
|
cur = m.group(1)
|
|
last_end = m.end()
|
|
if cur is not None:
|
|
out.append(cur)
|
|
return out
|
|
|
|
|
|
def calls_rows():
|
|
text = open(os.path.join(ROOT, "calls.cpp")).read()
|
|
a = text.index("const CallSpec kCalls[]")
|
|
b = text.index("};", a)
|
|
rows = []
|
|
for blk in re.findall(r"\{Call::k\w+.*?Feature::k\w+, \"\w+\"\}", text[a:b], re.S):
|
|
rows.append(string_runs(blk))
|
|
return rows # method, path, op, request, fields, response, fields_read, feature_name
|
|
|
|
|
|
class Spec:
|
|
def __init__(self, doc):
|
|
self.doc = doc
|
|
|
|
def schema(self, name):
|
|
return self.doc["components"]["schemas"][name]
|
|
|
|
def deref(self, node):
|
|
while isinstance(node, dict) and "$ref" in node:
|
|
node = self.schema(node["$ref"].rsplit("/", 1)[1])
|
|
return node
|
|
|
|
def props(self, node, seen=None):
|
|
seen = seen or set()
|
|
out = set()
|
|
if isinstance(node, dict) and "$ref" in node:
|
|
ref = node["$ref"]
|
|
if ref in seen:
|
|
return out
|
|
seen.add(ref)
|
|
return self.props(self.deref(node), seen)
|
|
if not isinstance(node, dict):
|
|
return out
|
|
out |= set(node.get("properties", {}))
|
|
for key in ("allOf", "oneOf", "anyOf"):
|
|
for sub in node.get(key, []):
|
|
out |= self.props(sub, seen)
|
|
return out
|
|
|
|
def schema_name_of(self, content, want=""):
|
|
ref = content.get("schema", {})
|
|
if "$ref" in ref:
|
|
return ref["$ref"].rsplit("/", 1)[1]
|
|
# a union (anyOf / oneOf): the wanted name may be one member
|
|
for key in ("anyOf", "oneOf"):
|
|
for sub in ref.get(key, []):
|
|
if "$ref" in sub and sub["$ref"].rsplit("/", 1)[1] == want:
|
|
return want
|
|
return ""
|
|
|
|
|
|
def check_calls(spec, bad):
|
|
rows = calls_rows()
|
|
declared = set()
|
|
for method, path, op, req, fields, resp, read, _feat in rows:
|
|
node = spec.doc["paths"].get(path, {}).get(method.lower())
|
|
if not node:
|
|
bad.append("%s %s: not in the spec" % (method, path))
|
|
continue
|
|
declared.add(op)
|
|
if node["operationId"] != op:
|
|
bad.append("%s %s: operationId %s != %s" % (method, path, node["operationId"], op))
|
|
body = node.get("requestBody", {}).get("content", {})
|
|
if req:
|
|
content = body.get("application/json") or body.get("multipart/form-data") or {}
|
|
if spec.schema_name_of(content) != req:
|
|
bad.append("%s %s: request schema %s != %s" % (method, path, spec.schema_name_of(content), req))
|
|
known = spec.props(content.get("schema", {}))
|
|
for f in filter(None, fields.split(",")):
|
|
if f not in known:
|
|
bad.append("%s %s: body field %s not in %s" % (method, path, f, req))
|
|
if resp:
|
|
ok = node["responses"]["200"]["content"]["application/json"]
|
|
if spec.schema_name_of(ok, resp) != resp:
|
|
bad.append("%s %s: response schema %s != %s" % (method, path, spec.schema_name_of(ok), resp))
|
|
known = spec.props(ok.get("schema", {}))
|
|
if not known:
|
|
known = spec.props({"$ref": "#/components/schemas/" + resp})
|
|
for f in filter(None, read.split(",")):
|
|
if f not in known:
|
|
bad.append("%s %s: response field %s not in %s" % (method, path, f, resp))
|
|
for path, item in spec.doc["paths"].items():
|
|
if not path.startswith(FAMILIES):
|
|
continue
|
|
for m, node in item.items():
|
|
if m not in ("get", "post", "delete"):
|
|
continue
|
|
op = node["operationId"]
|
|
if op not in declared and op not in EXCLUDED:
|
|
bad.append("spec operation %s %s (%s) is neither in the table nor excluded" % (m.upper(), path, op))
|
|
for op in EXCLUDED:
|
|
if op in declared:
|
|
bad.append("%s is both declared and excluded" % op)
|
|
return len(rows)
|
|
|
|
|
|
def registry_rows():
|
|
text = open(os.path.join(ROOT, "registry.cpp")).read()
|
|
a = text.index("const ProviderRow kRows[]")
|
|
b = text.index("constexpr std::size_t kRowCount")
|
|
out = []
|
|
for blk in re.findall(r'\n \{"[a-z0-9]+",.*?(?=\n \{"|\n //|\n\};)', text[a:b], re.S):
|
|
s = string_runs(blk)
|
|
out.append({"id": s[0], "name": s[1], "base": s[2], "key": s[3], "base_env": s[4], "extra_env": s[5],
|
|
"models": s[6], "doc": s[7]})
|
|
return out
|
|
|
|
|
|
def spec_md_rows():
|
|
text = open(os.path.join(ROOT, "SPEC.md")).read()
|
|
a = text.index("## 4. Provider table")
|
|
b = text.index("\n## ", a + 5)
|
|
rows = {}
|
|
for line in text[a:b].splitlines():
|
|
cells = [c.strip().strip("`") for c in line.strip().strip("|").split("|")]
|
|
if line.startswith("|") and len(cells) >= 5 and re.fullmatch(r"[a-z0-9]+", cells[0]) and cells[0] != "id":
|
|
rows[cells[0]] = cells
|
|
return rows
|
|
|
|
|
|
def check_registry(bad):
|
|
reg = registry_rows()
|
|
md = spec_md_rows()
|
|
ids = [r["id"] for r in reg]
|
|
if len(ids) != len(set(ids)):
|
|
bad.append("registry.cpp has duplicate ids")
|
|
for r in reg:
|
|
cells = md.get(r["id"])
|
|
if not cells:
|
|
bad.append("provider %s is in registry.cpp but not in SPEC.md section 4" % r["id"])
|
|
continue
|
|
line = " ".join(cells)
|
|
for what in ("base", "key", "doc"):
|
|
if r[what] not in line:
|
|
bad.append("provider %s: %s '%s' differs between registry.cpp and SPEC.md" % (r["id"], what, r[what]))
|
|
for i in md:
|
|
if i not in ids:
|
|
bad.append("provider %s is in SPEC.md section 4 but not in registry.cpp" % i)
|
|
return len(reg)
|
|
|
|
|
|
def main():
|
|
args = [a for a in sys.argv[1:] if not a.startswith("--")]
|
|
local = "--local" in sys.argv
|
|
bad = []
|
|
n_reg = check_registry(bad)
|
|
n_calls = 0
|
|
if not local:
|
|
if args:
|
|
doc = json.load(open(args[0]))
|
|
else:
|
|
req = urllib.request.Request(URL, headers={"User-Agent": "molo-compat-spec-check/1"})
|
|
with urllib.request.urlopen(req, timeout=120) as r:
|
|
doc = json.loads(r.read().decode("utf-8"))
|
|
n_calls = check_calls(Spec(doc), bad)
|
|
print("openapi %s (%s)" % (doc["info"]["version"], SPEC_SHA[:12]))
|
|
print("registry rows: %d, call rows checked: %d" % (n_reg, n_calls))
|
|
if bad:
|
|
print("FAIL")
|
|
for b in bad:
|
|
print(" " + b)
|
|
sys.exit(1)
|
|
print("OK")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|