feat(clients): openai admin: organization admin API, usage, costs, audit logs, webhooks
121-call table checked against the OpenAI OpenAPI document, admin-key-only client, destructive guards, WebhookVerifier (Standard Webhooks HMAC-SHA256), demo routes under /ai/openai/admin, local and opt-in live tests. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check src/openai/admin against OpenAI's published OpenAPI document.
|
||||
|
||||
Fails (exit 1) when a declared call is absent from the spec, its HTTP method,
|
||||
path, operationId, auth scheme, query parameters, request body fields,
|
||||
required fields or response schema differ, when a schema field we read is
|
||||
absent, when the audit event or webhook event catalogs differ, or when a spec
|
||||
operation in scope (organization administration, webhook endpoints) is neither
|
||||
in the call table nor in the explicit EXCLUDED list below. Python 3 stdlib
|
||||
only, no credentials.
|
||||
|
||||
python3 -I src/openai/admin/tools/check_spec.py # pinned revision
|
||||
python3 -I src/openai/admin/tools/check_spec.py --head # newest main
|
||||
python3 -I src/openai/admin/tools/check_spec.py --file x.json
|
||||
|
||||
Second source: the official openai-go SDK api.md must list every table row's
|
||||
HTTP method and path (cross-check, rule 2 of plan 17).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SRC = os.path.join(HERE, "..")
|
||||
PINNED = "8243cc8fa6f04430fb1f1a6a2cb926fbeb4f7f26"
|
||||
SPEC_URL = "https://raw.githubusercontent.com/openai/openai-openapi/%s/openapi.json"
|
||||
GO_API = "https://raw.githubusercontent.com/openai/openai-go/main/api.md"
|
||||
GO_WEBHOOKS_API = "https://raw.githubusercontent.com/openai/openai-go/main/webhooks/api.md"
|
||||
|
||||
# Spec operations in scope that this package deliberately does not wrap:
|
||||
# method path -> reason. Empty: every in-scope operation is wrapped.
|
||||
EXCLUDED = {}
|
||||
|
||||
ROW = re.compile(r'^\s*\{(.*)\},?\s*$')
|
||||
STR = re.compile(r'"((?:[^"\\]|\\.)*)"')
|
||||
|
||||
|
||||
def fetch(url):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "molo-check-spec"})
|
||||
with urllib.request.urlopen(req, timeout=300) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
def block(text, begin, end):
|
||||
a = text.index(begin)
|
||||
b = text.index(end)
|
||||
rows = []
|
||||
for line in text[a:b].splitlines()[1:]:
|
||||
m = ROW.match(line)
|
||||
if m and '"' in line:
|
||||
rows.append(STR.findall(m.group(1)))
|
||||
return rows
|
||||
|
||||
|
||||
def strings_block(text, begin, end):
|
||||
a = text.index(begin)
|
||||
b = text.index(end)
|
||||
return STR.findall(text[a:b].split("\n", 1)[1])
|
||||
|
||||
|
||||
def in_scope(path):
|
||||
return path.startswith("/organization") or path.startswith("/webhook_endpoints") \
|
||||
or path == "/webhook_event_types"
|
||||
|
||||
|
||||
def main():
|
||||
args = sys.argv[1:]
|
||||
if "--file" in args:
|
||||
raw = open(args[args.index("--file") + 1], "rb").read()
|
||||
label = args[args.index("--file") + 1]
|
||||
else:
|
||||
sha = PINNED
|
||||
if "--head" in args:
|
||||
meta = json.loads(fetch("https://api.github.com/repos/openai/openai-openapi/commits/main"))
|
||||
sha = meta["sha"]
|
||||
raw = fetch(SPEC_URL % sha)
|
||||
label = sha
|
||||
spec = json.loads(raw)
|
||||
schemas = spec["components"]["schemas"]
|
||||
paths = spec["paths"]
|
||||
|
||||
def deref(x):
|
||||
while isinstance(x, dict) and "$ref" in x:
|
||||
x = schemas[x["$ref"].split("/")[-1]] if x["$ref"].startswith("#/components/schemas/") \
|
||||
else resolve(x["$ref"])
|
||||
return x
|
||||
|
||||
def resolve(ref):
|
||||
node = spec
|
||||
for part in ref.lstrip("#/").split("/"):
|
||||
node = node[part]
|
||||
return node
|
||||
|
||||
def flat(sch):
|
||||
sch = deref(sch)
|
||||
props, req = dict(sch.get("properties", {})), list(sch.get("required", []))
|
||||
for a in sch.get("allOf", []):
|
||||
p, r = flat(a)
|
||||
props.update(p)
|
||||
req += r
|
||||
return props, req
|
||||
|
||||
calls_src = open(os.path.join(SRC, "calls.cpp")).read()
|
||||
calls = block(calls_src, "// CALLS-BEGIN", "// CALLS-END")
|
||||
fields = block(calls_src, "// SCHEMAS-BEGIN", "// SCHEMAS-END")
|
||||
bad = []
|
||||
default_security = spec.get("security", [])
|
||||
|
||||
for key, http, path, op_id, auth, body, req, query, rschema, resp, flags in \
|
||||
[tuple(r[:11]) for r in calls]:
|
||||
node = paths.get(path, {}).get(http.lower())
|
||||
if not node:
|
||||
bad.append("%s: %s %s absent from the spec" % (key, http, path))
|
||||
continue
|
||||
if node.get("operationId") != op_id:
|
||||
bad.append("%s: operationId %s != %s" % (key, op_id, node.get("operationId")))
|
||||
sec = node.get("security", default_security)
|
||||
scheme = list(sec[0].keys())[0] if sec else ""
|
||||
want = "AdminApiKeyAuth" if auth == "admin" else "ApiKeyAuth"
|
||||
if scheme != want:
|
||||
bad.append("%s: auth %s but the spec says %s" % (key, auth, scheme))
|
||||
if auth == "project" and "AdminApiKeyAuth" in json.dumps(sec):
|
||||
bad.append("%s: project call declared with admin security" % key)
|
||||
spec_query = []
|
||||
for prm in node.get("parameters", []):
|
||||
prm = deref(prm)
|
||||
if prm["in"] == "query":
|
||||
spec_query.append(prm["name"])
|
||||
mine = [q for q in query.split(",") if q]
|
||||
for q in mine:
|
||||
if q not in spec_query:
|
||||
bad.append("%s: query parameter %s absent" % (key, q))
|
||||
for q in spec_query:
|
||||
if q not in mine:
|
||||
bad.append("%s: spec query parameter %s not in the table" % (key, q))
|
||||
want_path_params = re.findall(r"\{([^}]+)\}", path)
|
||||
spec_path_params = [deref(p)["name"] for p in node.get("parameters", []) if deref(p)["in"] == "path"]
|
||||
if sorted(want_path_params) != sorted(spec_path_params):
|
||||
bad.append("%s: path parameters %s != %s" % (key, want_path_params, spec_path_params))
|
||||
rb = node.get("requestBody")
|
||||
if rb:
|
||||
sch = rb["content"]["application/json"]["schema"]
|
||||
ref = sch["$ref"].split("/")[-1] if "$ref" in sch else ""
|
||||
if ref != rschema:
|
||||
bad.append("%s: request schema %r != %r" % (key, rschema, ref))
|
||||
props, reqd = flat(sch)
|
||||
for f in [f for f in body.split(",") if f]:
|
||||
if f not in props:
|
||||
bad.append("%s: body field %s absent" % (key, f))
|
||||
for f in props:
|
||||
if f not in body.split(","):
|
||||
bad.append("%s: spec body field %s not in the table" % (key, f))
|
||||
if sorted(set(reqd)) != sorted(f for f in req.split(",") if f):
|
||||
bad.append("%s: required %s != %s" % (key, sorted(set(reqd)), req))
|
||||
elif body or req or rschema:
|
||||
bad.append("%s: table has a body but the spec has none" % key)
|
||||
ok = node["responses"].get("200") or node["responses"].get("201")
|
||||
content = deref(ok).get("content", {}).get("application/json") if ok else None
|
||||
got = content["schema"]["$ref"].split("/")[-1] if content and "$ref" in content["schema"] else ""
|
||||
if got != resp:
|
||||
bad.append("%s: response schema %r != %r" % (key, resp, got))
|
||||
if (http == "DELETE") != (flags == "destructive") and key not in (
|
||||
"projects.archive", "certificates.deactivate", "projects.certificates.deactivate",
|
||||
"data_retention.update", "projects.data_retention.update",
|
||||
"webhook_endpoints.rotate_secret"):
|
||||
bad.append("%s: destructive flag disagrees with the HTTP method" % key)
|
||||
|
||||
for name, flds in fields:
|
||||
sch = schemas.get(name)
|
||||
if sch is None:
|
||||
bad.append("missing schema %s" % name)
|
||||
continue
|
||||
props, _ = flat(sch)
|
||||
for f in flds.split(","):
|
||||
if f not in props:
|
||||
bad.append("schema %s.%s absent" % (name, f))
|
||||
|
||||
declared = {(r[1], r[2]) for r in calls}
|
||||
in_scope_ops = 0
|
||||
for path, item in paths.items():
|
||||
if not in_scope(path):
|
||||
continue
|
||||
for method in ("get", "post", "put", "patch", "delete"):
|
||||
if method in item:
|
||||
in_scope_ops += 1
|
||||
k = (method.upper(), path)
|
||||
if k not in declared and ("%s %s" % k) not in EXCLUDED:
|
||||
bad.append("unwrapped spec operation %s %s" % k)
|
||||
for ex in EXCLUDED:
|
||||
m, p = ex.split(" ", 1)
|
||||
if (m, p) in declared:
|
||||
bad.append("excluded operation %s is also in the table" % ex)
|
||||
|
||||
# Audit event type catalog.
|
||||
audit_src = open(os.path.join(SRC, "audit_events.cpp")).read()
|
||||
mine = strings_block(audit_src, "// AUDIT-EVENTS-BEGIN", "// AUDIT-EVENTS-END")
|
||||
theirs = schemas["AuditLogEventType"]["enum"]
|
||||
if mine != theirs:
|
||||
bad.append("audit event catalog differs: missing %s extra %s" % (
|
||||
[e for e in theirs if e not in mine], [e for e in mine if e not in theirs]))
|
||||
|
||||
# Webhook catalogs.
|
||||
wh_src = open(os.path.join(SRC, "webhook_events.cpp")).read()
|
||||
sent = strings_block(wh_src, "// WEBHOOK-EVENTS-BEGIN", "// WEBHOOK-EVENTS-END")
|
||||
spec_sent = []
|
||||
for name, item in spec.get("webhooks", {}).items():
|
||||
sch = item["post"]["requestBody"]["content"]["application/json"]["schema"]
|
||||
props, _ = flat(sch)
|
||||
enum = deref(props["type"]).get("enum")
|
||||
spec_sent += enum or []
|
||||
if sorted(sent) != sorted(spec_sent):
|
||||
bad.append("webhook event types differ: missing %s extra %s" % (
|
||||
sorted(set(spec_sent) - set(sent)), sorted(set(sent) - set(spec_sent))))
|
||||
sub = strings_block(wh_src, "// SUBSCRIBABLE-BEGIN", "// SUBSCRIBABLE-END")
|
||||
if sub != schemas["ProjectEventTypeEnum"]["enum"]:
|
||||
bad.append("subscribable webhook event types differ from ProjectEventTypeEnum")
|
||||
usage_src = open(os.path.join(SRC, "usage.cpp")).read()
|
||||
ub = usage_src[usage_src.index("// USAGE-ENDPOINTS-BEGIN"):usage_src.index("// USAGE-ENDPOINTS-END")]
|
||||
for name, gb in re.findall(r'"(\w+)", Call::\w+,\s*"([^"]*)"\}', ub):
|
||||
node = paths["/organization/usage/" + name]["get"]
|
||||
enum = []
|
||||
for prm in node["parameters"]:
|
||||
prm = deref(prm)
|
||||
if prm["name"] == "group_by":
|
||||
it = deref(prm["schema"]).get("items")
|
||||
enum = deref(it).get("enum", []) if it else []
|
||||
if enum != [g for g in gb.split(",") if g]:
|
||||
bad.append("usage %s group_by %s != spec %s" % (name, gb, enum))
|
||||
spec_res = [n for n in schemas if n.startswith("Usage") and n.endswith("Result")] + ["CostsResult"]
|
||||
for n in spec_res:
|
||||
if n not in [f[0] for f in fields]:
|
||||
bad.append("usage result schema %s is not in the SCHEMAS table" % n)
|
||||
evb = wh_src[wh_src.index("// EVENT-SPECS"):wh_src.index("// WEBHOOK-EVENTS-BEGIN")]
|
||||
rows_ev = re.findall(r'\{"([^"]+)", "([^"]+)", "([^"]*)", "([^"]*)"\}', evb)
|
||||
for t, key, flds, rq in rows_ev:
|
||||
item = spec["webhooks"].get(key)
|
||||
if not item:
|
||||
bad.append("webhook %s absent" % key)
|
||||
continue
|
||||
props, _ = flat(item["post"]["requestBody"]["content"]["application/json"]["schema"])
|
||||
dp, dr = flat(props["data"])
|
||||
if sorted(dp) != sorted(flds.split(",")) or sorted(set(dr)) != sorted(x for x in rq.split(",") if x):
|
||||
bad.append("webhook %s data fields differ" % t)
|
||||
|
||||
# Second source: openai-go api.md lists every method and path.
|
||||
sdk = fetch(GO_API).decode() + fetch(GO_WEBHOOKS_API).decode()
|
||||
listed = set(re.findall(r'title="(get|post|delete) (/[^"]+)"', sdk))
|
||||
miss = 0
|
||||
for r in calls:
|
||||
http, path = r[1].lower(), r[2]
|
||||
sdk_path = path[1:] if (http, path[1:]) in listed else path
|
||||
if (http, sdk_path) not in listed and (http, path.lstrip("/")) not in listed:
|
||||
miss += 1
|
||||
bad.append("openai-go api.md does not list %s %s" % (http.upper(), path))
|
||||
|
||||
if bad:
|
||||
print("SPEC CHECK FAILED (%s)" % label)
|
||||
for b in bad:
|
||||
print(" " + b)
|
||||
return 1
|
||||
print("spec ok: %d calls, %d schemas, %d in-scope spec operations covered, "
|
||||
"%d audit event types, %d sent webhook event types, openai-go lists all %d rows, "
|
||||
"openapi %s revision %s, info.version %s"
|
||||
% (len(calls), len(fields), in_scope_ops, len(theirs), len(spec_sent), len(calls) - miss,
|
||||
"openapi.json", label, spec["info"]["version"]))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Unauthenticated probe: every GET row of calls.cpp must answer 401 (path
|
||||
exists, auth required) and a made-up sibling must answer 404. Sends an
|
||||
obviously invalid key, never a real credential. Python 3 stdlib only.
|
||||
|
||||
python3 -I src/openai/admin/tools/probe_paths.py
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
BASE = "https://api.openai.com/v1"
|
||||
|
||||
|
||||
def status(path):
|
||||
req = urllib.request.Request(BASE + path, headers={"Authorization": "Bearer sk-admin-invalid"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code
|
||||
|
||||
|
||||
def main():
|
||||
src = open(os.path.join(HERE, "..", "calls.cpp")).read()
|
||||
src = src[src.index("// CALLS-BEGIN"):src.index("// CALLS-END")]
|
||||
rows = [re.findall(r'"([^"]*)"', l) for l in src.splitlines() if l.strip().startswith("{Call::")]
|
||||
bad, n = [], 0
|
||||
for r in rows:
|
||||
if r[1] != "GET":
|
||||
continue
|
||||
n += 1
|
||||
path = re.sub(r"\{[^}]+\}", "probe_id", r[2])
|
||||
code = status(path)
|
||||
if code != 401:
|
||||
bad.append("%s -> %s (want 401)" % (path, code))
|
||||
code = status("/organization/nonexistent_probe_path")
|
||||
if code != 404:
|
||||
bad.append("made-up path -> %s (want 404)" % code)
|
||||
if bad:
|
||||
print("PROBE FAILED")
|
||||
for b in bad:
|
||||
print(" " + b)
|
||||
return 1
|
||||
print("probe ok: %d GET paths answered 401, made-up path answered 404" % n)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user