feat(clients): openai admin: organization admin API, usage, costs, audit logs, webhooks

121-call table checked against the OpenAI OpenAPI document, admin-key-only
client, destructive guards, WebhookVerifier (Standard Webhooks HMAC-SHA256),
demo routes under /ai/openai/admin, local and opt-in live tests.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-24 07:43:17 +02:00
co-authored by Claude Sonnet 5.5
parent 2e0270c043
commit 9e550889cc
47 changed files with 5533 additions and 3 deletions
+272
View File
@@ -0,0 +1,272 @@
#!/usr/bin/env python3
"""Check src/openai/admin against OpenAI's published OpenAPI document.
Fails (exit 1) when a declared call is absent from the spec, its HTTP method,
path, operationId, auth scheme, query parameters, request body fields,
required fields or response schema differ, when a schema field we read is
absent, when the audit event or webhook event catalogs differ, or when a spec
operation in scope (organization administration, webhook endpoints) is neither
in the call table nor in the explicit EXCLUDED list below. Python 3 stdlib
only, no credentials.
python3 -I src/openai/admin/tools/check_spec.py # pinned revision
python3 -I src/openai/admin/tools/check_spec.py --head # newest main
python3 -I src/openai/admin/tools/check_spec.py --file x.json
Second source: the official openai-go SDK api.md must list every table row's
HTTP method and path (cross-check, rule 2 of plan 17).
"""
import json
import os
import re
import sys
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
SRC = os.path.join(HERE, "..")
PINNED = "8243cc8fa6f04430fb1f1a6a2cb926fbeb4f7f26"
SPEC_URL = "https://raw.githubusercontent.com/openai/openai-openapi/%s/openapi.json"
GO_API = "https://raw.githubusercontent.com/openai/openai-go/main/api.md"
GO_WEBHOOKS_API = "https://raw.githubusercontent.com/openai/openai-go/main/webhooks/api.md"
# Spec operations in scope that this package deliberately does not wrap:
# method path -> reason. Empty: every in-scope operation is wrapped.
EXCLUDED = {}
ROW = re.compile(r'^\s*\{(.*)\},?\s*$')
STR = re.compile(r'"((?:[^"\\]|\\.)*)"')
def fetch(url):
req = urllib.request.Request(url, headers={"User-Agent": "molo-check-spec"})
with urllib.request.urlopen(req, timeout=300) as r:
return r.read()
def block(text, begin, end):
a = text.index(begin)
b = text.index(end)
rows = []
for line in text[a:b].splitlines()[1:]:
m = ROW.match(line)
if m and '"' in line:
rows.append(STR.findall(m.group(1)))
return rows
def strings_block(text, begin, end):
a = text.index(begin)
b = text.index(end)
return STR.findall(text[a:b].split("\n", 1)[1])
def in_scope(path):
return path.startswith("/organization") or path.startswith("/webhook_endpoints") \
or path == "/webhook_event_types"
def main():
args = sys.argv[1:]
if "--file" in args:
raw = open(args[args.index("--file") + 1], "rb").read()
label = args[args.index("--file") + 1]
else:
sha = PINNED
if "--head" in args:
meta = json.loads(fetch("https://api.github.com/repos/openai/openai-openapi/commits/main"))
sha = meta["sha"]
raw = fetch(SPEC_URL % sha)
label = sha
spec = json.loads(raw)
schemas = spec["components"]["schemas"]
paths = spec["paths"]
def deref(x):
while isinstance(x, dict) and "$ref" in x:
x = schemas[x["$ref"].split("/")[-1]] if x["$ref"].startswith("#/components/schemas/") \
else resolve(x["$ref"])
return x
def resolve(ref):
node = spec
for part in ref.lstrip("#/").split("/"):
node = node[part]
return node
def flat(sch):
sch = deref(sch)
props, req = dict(sch.get("properties", {})), list(sch.get("required", []))
for a in sch.get("allOf", []):
p, r = flat(a)
props.update(p)
req += r
return props, req
calls_src = open(os.path.join(SRC, "calls.cpp")).read()
calls = block(calls_src, "// CALLS-BEGIN", "// CALLS-END")
fields = block(calls_src, "// SCHEMAS-BEGIN", "// SCHEMAS-END")
bad = []
default_security = spec.get("security", [])
for key, http, path, op_id, auth, body, req, query, rschema, resp, flags in \
[tuple(r[:11]) for r in calls]:
node = paths.get(path, {}).get(http.lower())
if not node:
bad.append("%s: %s %s absent from the spec" % (key, http, path))
continue
if node.get("operationId") != op_id:
bad.append("%s: operationId %s != %s" % (key, op_id, node.get("operationId")))
sec = node.get("security", default_security)
scheme = list(sec[0].keys())[0] if sec else ""
want = "AdminApiKeyAuth" if auth == "admin" else "ApiKeyAuth"
if scheme != want:
bad.append("%s: auth %s but the spec says %s" % (key, auth, scheme))
if auth == "project" and "AdminApiKeyAuth" in json.dumps(sec):
bad.append("%s: project call declared with admin security" % key)
spec_query = []
for prm in node.get("parameters", []):
prm = deref(prm)
if prm["in"] == "query":
spec_query.append(prm["name"])
mine = [q for q in query.split(",") if q]
for q in mine:
if q not in spec_query:
bad.append("%s: query parameter %s absent" % (key, q))
for q in spec_query:
if q not in mine:
bad.append("%s: spec query parameter %s not in the table" % (key, q))
want_path_params = re.findall(r"\{([^}]+)\}", path)
spec_path_params = [deref(p)["name"] for p in node.get("parameters", []) if deref(p)["in"] == "path"]
if sorted(want_path_params) != sorted(spec_path_params):
bad.append("%s: path parameters %s != %s" % (key, want_path_params, spec_path_params))
rb = node.get("requestBody")
if rb:
sch = rb["content"]["application/json"]["schema"]
ref = sch["$ref"].split("/")[-1] if "$ref" in sch else ""
if ref != rschema:
bad.append("%s: request schema %r != %r" % (key, rschema, ref))
props, reqd = flat(sch)
for f in [f for f in body.split(",") if f]:
if f not in props:
bad.append("%s: body field %s absent" % (key, f))
for f in props:
if f not in body.split(","):
bad.append("%s: spec body field %s not in the table" % (key, f))
if sorted(set(reqd)) != sorted(f for f in req.split(",") if f):
bad.append("%s: required %s != %s" % (key, sorted(set(reqd)), req))
elif body or req or rschema:
bad.append("%s: table has a body but the spec has none" % key)
ok = node["responses"].get("200") or node["responses"].get("201")
content = deref(ok).get("content", {}).get("application/json") if ok else None
got = content["schema"]["$ref"].split("/")[-1] if content and "$ref" in content["schema"] else ""
if got != resp:
bad.append("%s: response schema %r != %r" % (key, resp, got))
if (http == "DELETE") != (flags == "destructive") and key not in (
"projects.archive", "certificates.deactivate", "projects.certificates.deactivate",
"data_retention.update", "projects.data_retention.update",
"webhook_endpoints.rotate_secret"):
bad.append("%s: destructive flag disagrees with the HTTP method" % key)
for name, flds in fields:
sch = schemas.get(name)
if sch is None:
bad.append("missing schema %s" % name)
continue
props, _ = flat(sch)
for f in flds.split(","):
if f not in props:
bad.append("schema %s.%s absent" % (name, f))
declared = {(r[1], r[2]) for r in calls}
in_scope_ops = 0
for path, item in paths.items():
if not in_scope(path):
continue
for method in ("get", "post", "put", "patch", "delete"):
if method in item:
in_scope_ops += 1
k = (method.upper(), path)
if k not in declared and ("%s %s" % k) not in EXCLUDED:
bad.append("unwrapped spec operation %s %s" % k)
for ex in EXCLUDED:
m, p = ex.split(" ", 1)
if (m, p) in declared:
bad.append("excluded operation %s is also in the table" % ex)
# Audit event type catalog.
audit_src = open(os.path.join(SRC, "audit_events.cpp")).read()
mine = strings_block(audit_src, "// AUDIT-EVENTS-BEGIN", "// AUDIT-EVENTS-END")
theirs = schemas["AuditLogEventType"]["enum"]
if mine != theirs:
bad.append("audit event catalog differs: missing %s extra %s" % (
[e for e in theirs if e not in mine], [e for e in mine if e not in theirs]))
# Webhook catalogs.
wh_src = open(os.path.join(SRC, "webhook_events.cpp")).read()
sent = strings_block(wh_src, "// WEBHOOK-EVENTS-BEGIN", "// WEBHOOK-EVENTS-END")
spec_sent = []
for name, item in spec.get("webhooks", {}).items():
sch = item["post"]["requestBody"]["content"]["application/json"]["schema"]
props, _ = flat(sch)
enum = deref(props["type"]).get("enum")
spec_sent += enum or []
if sorted(sent) != sorted(spec_sent):
bad.append("webhook event types differ: missing %s extra %s" % (
sorted(set(spec_sent) - set(sent)), sorted(set(sent) - set(spec_sent))))
sub = strings_block(wh_src, "// SUBSCRIBABLE-BEGIN", "// SUBSCRIBABLE-END")
if sub != schemas["ProjectEventTypeEnum"]["enum"]:
bad.append("subscribable webhook event types differ from ProjectEventTypeEnum")
usage_src = open(os.path.join(SRC, "usage.cpp")).read()
ub = usage_src[usage_src.index("// USAGE-ENDPOINTS-BEGIN"):usage_src.index("// USAGE-ENDPOINTS-END")]
for name, gb in re.findall(r'"(\w+)", Call::\w+,\s*"([^"]*)"\}', ub):
node = paths["/organization/usage/" + name]["get"]
enum = []
for prm in node["parameters"]:
prm = deref(prm)
if prm["name"] == "group_by":
it = deref(prm["schema"]).get("items")
enum = deref(it).get("enum", []) if it else []
if enum != [g for g in gb.split(",") if g]:
bad.append("usage %s group_by %s != spec %s" % (name, gb, enum))
spec_res = [n for n in schemas if n.startswith("Usage") and n.endswith("Result")] + ["CostsResult"]
for n in spec_res:
if n not in [f[0] for f in fields]:
bad.append("usage result schema %s is not in the SCHEMAS table" % n)
evb = wh_src[wh_src.index("// EVENT-SPECS"):wh_src.index("// WEBHOOK-EVENTS-BEGIN")]
rows_ev = re.findall(r'\{"([^"]+)", "([^"]+)", "([^"]*)", "([^"]*)"\}', evb)
for t, key, flds, rq in rows_ev:
item = spec["webhooks"].get(key)
if not item:
bad.append("webhook %s absent" % key)
continue
props, _ = flat(item["post"]["requestBody"]["content"]["application/json"]["schema"])
dp, dr = flat(props["data"])
if sorted(dp) != sorted(flds.split(",")) or sorted(set(dr)) != sorted(x for x in rq.split(",") if x):
bad.append("webhook %s data fields differ" % t)
# Second source: openai-go api.md lists every method and path.
sdk = fetch(GO_API).decode() + fetch(GO_WEBHOOKS_API).decode()
listed = set(re.findall(r'title="(get|post|delete) (/[^"]+)"', sdk))
miss = 0
for r in calls:
http, path = r[1].lower(), r[2]
sdk_path = path[1:] if (http, path[1:]) in listed else path
if (http, sdk_path) not in listed and (http, path.lstrip("/")) not in listed:
miss += 1
bad.append("openai-go api.md does not list %s %s" % (http.upper(), path))
if bad:
print("SPEC CHECK FAILED (%s)" % label)
for b in bad:
print(" " + b)
return 1
print("spec ok: %d calls, %d schemas, %d in-scope spec operations covered, "
"%d audit event types, %d sent webhook event types, openai-go lists all %d rows, "
"openapi %s revision %s, info.version %s"
% (len(calls), len(fields), in_scope_ops, len(theirs), len(spec_sent), len(calls) - miss,
"openapi.json", label, spec["info"]["version"]))
return 0
if __name__ == "__main__":
sys.exit(main())
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env python3
"""Unauthenticated probe: every GET row of calls.cpp must answer 401 (path
exists, auth required) and a made-up sibling must answer 404. Sends an
obviously invalid key, never a real credential. Python 3 stdlib only.
python3 -I src/openai/admin/tools/probe_paths.py
"""
import os
import re
import sys
import urllib.error
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
BASE = "https://api.openai.com/v1"
def status(path):
req = urllib.request.Request(BASE + path, headers={"Authorization": "Bearer sk-admin-invalid"})
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status
except urllib.error.HTTPError as e:
return e.code
def main():
src = open(os.path.join(HERE, "..", "calls.cpp")).read()
src = src[src.index("// CALLS-BEGIN"):src.index("// CALLS-END")]
rows = [re.findall(r'"([^"]*)"', l) for l in src.splitlines() if l.strip().startswith("{Call::")]
bad, n = [], 0
for r in rows:
if r[1] != "GET":
continue
n += 1
path = re.sub(r"\{[^}]+\}", "probe_id", r[2])
code = status(path)
if code != 401:
bad.append("%s -> %s (want 401)" % (path, code))
code = status("/organization/nonexistent_probe_path")
if code != 404:
bad.append("made-up path -> %s (want 404)" % code)
if bad:
print("PROBE FAILED")
for b in bad:
print(" " + b)
return 1
print("probe ok: %d GET paths answered 401, made-up path answered 404" % n)
return 0
if __name__ == "__main__":
sys.exit(main())