feat(clients): huggingface: Hub, Inference Providers, Endpoints, TGI, TEI, chat model, demo
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
+315
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Checks calls.cpp (the module's one call table) against Hugging Face's
|
||||
published documents. Python 3 stdlib only, no credentials.
|
||||
|
||||
python3 -I src/vendors/huggingface/tools/check_spec.py [--cache DIR] [--probe]
|
||||
|
||||
Documents (all fetched unless found in --cache):
|
||||
hub https://huggingface.co/.well-known/openapi.json (Hub OpenAPI 3.1)
|
||||
ie https://api.endpoints.huggingface.cloud/openapi.json (Inference Endpoints 2.0.0)
|
||||
tgi text-generation-inference docs/openapi.json (main)
|
||||
tei text-embeddings-inference docs/openapi.json (main)
|
||||
src huggingface_hub hf_api.py, hf_inference.py, constants.py (main; the Hub OpenAPI
|
||||
omits the repo list / info routes, so those rows cite the SDK source)
|
||||
task huggingface.js packages/tasks/src/tasks/<task>/spec/input.json
|
||||
|
||||
Per row it requires: the HTTP method and path exist (a {repoType} row is
|
||||
expanded to models, datasets, spaces; {typePrefix} to "", /datasets, /spaces),
|
||||
the operationId matches when the document has one, every query parameter and
|
||||
body field in the row is present in the document. Coverage: for ie, tgi and tei
|
||||
every document operation, query parameter and body field must be in the table;
|
||||
for hub every operation must be in the table or match an explicit EXCLUDED rule
|
||||
(counted per reason in the output); for hub the unlisted query parameters and
|
||||
body fields are reported (not failed: the Hub document is large and
|
||||
the table lists what the module sends). With --probe, rows whose spec is
|
||||
"src" or "none" are probed without a credential: a 404 fails.
|
||||
Exit 0: all rows verified. Exit 1: at least one mismatch (printed).
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
TABLE = os.path.join(HERE, "..", "calls.cpp")
|
||||
RAW = "https://raw.githubusercontent.com/huggingface/"
|
||||
URLS = {
|
||||
"hub.json": "https://huggingface.co/.well-known/openapi.json",
|
||||
"ie.json": "https://api.endpoints.huggingface.cloud/openapi.json",
|
||||
"tgi.json": RAW + "text-generation-inference/main/docs/openapi.json",
|
||||
"tei.json": RAW + "text-embeddings-inference/main/docs/openapi.json",
|
||||
"hf_api.py": RAW + "huggingface_hub/main/src/huggingface_hub/hf_api.py",
|
||||
"hf_inference.py": RAW + "huggingface_hub/main/src/huggingface_hub/inference/_providers/hf_inference.py",
|
||||
"constants.py": RAW + "huggingface_hub/main/src/huggingface_hub/constants.py",
|
||||
}
|
||||
TASK_URL = RAW + "huggingface.js/main/packages/tasks/src/tasks/%s/spec/input.json"
|
||||
HOSTS = {
|
||||
"Hub": "https://huggingface.co",
|
||||
"Router": "https://router.huggingface.co",
|
||||
"Endpoints": "https://api.endpoints.huggingface.cloud",
|
||||
}
|
||||
S = r'"((?:[^"\\]|\\.)*)"'
|
||||
ROW = re.compile(r"^\s*\{Call::(\w+), Surface::k(\w+), " + ", ".join([S] * 8) + r"\},?\s*$")
|
||||
METHODS = ("get", "post", "put", "patch", "delete")
|
||||
REPO_TYPES = ["models", "datasets", "spaces"]
|
||||
PREFIXES = ["", "/datasets", "/spaces"]
|
||||
|
||||
# Hub operations deliberately not wrapped: (reason, regex over "METHOD /path").
|
||||
EXCLUDED = [
|
||||
("repository writes (commit, upload, LFS, branches, tags, squash, duplicate, settings, move, create)",
|
||||
r"^(POST|PUT|DELETE) /api/(models|datasets|spaces)/\{namespace\}/\{repo\}/(commit|preupload|lfs-files|tag|branch|super-squash|settings|resource-group|volumes|hardware|sleeptime|dev-mode|custom-domain|secrets|variables)"),
|
||||
("repository writes (commit, upload, LFS, branches, tags, squash, duplicate, settings, move, create)",
|
||||
r"^(POST|PUT|DELETE|GET) /api/(repos|\{repoType\})/.*(create|move|duplicate)"),
|
||||
("repository admin reads (LFS file list, xet tokens, jwt, scan, notebook, leaderboard, resource group)",
|
||||
r"^GET /api/(models|datasets|spaces)/\{namespace\}/\{repo\}/(lfs-files|xet-write-token|xet-read-token|jwt|scan|notebook|leaderboard|resource-group|logs|events|metrics)"),
|
||||
("gated repo access requests", r"(ask-access|user-access|access-request)"),
|
||||
("resolve-cache and viewer helpers", r"^GET /api/resolve-cache/"),
|
||||
("secrets and variables of a Space (credentials)", r"^(GET|POST|DELETE) /api/spaces/\{namespace\}/\{repo\}/(secrets|variables)"),
|
||||
("rarely used discussion moderation (pin, merge, ignore, storage, ref delete, comment edit, hide, reaction)",
|
||||
r"^(GET|POST|DELETE) /api/\{repoType\}/\{namespace\}/\{repo\}/discussions/\{num\}/(pin|merge|ignore|storage|ref|comment/)"),
|
||||
("collection item update and delete (need the slug-id split, see SPEC Gaps) and resource groups",
|
||||
r"^(PATCH|DELETE|POST|GET) /api/collections/.*(items/|items/batch|resource-group|\{slug\}-\{id\})"),
|
||||
("comments on blogs, papers, posts", r"^(POST|DELETE) /api/(blog|papers|posts)/"),
|
||||
("paper curation (index, links, media, claim)", r"^POST /api/(papers/index|papers/\{paperId\}/(links|media)|settings/papers/claim)"),
|
||||
("account settings, billing and notifications", r"^(GET|POST|PUT|PATCH|DELETE) /api/(settings/(?!webhooks)|notifications|credentials)"),
|
||||
("organizations, users, avatars, SCIM, service accounts, resource groups",
|
||||
r"^[A-Z]+ /api/(orgs|organizations|users|avatars|scim|service-accounts|resource-groups)"),
|
||||
("Hub Jobs and scheduled jobs (separate product)", r"^[A-Z]+ /api/(jobs|scheduled-jobs)"),
|
||||
("buckets, containers, kernels (separate repo kinds)", r"^[A-Z]+ /(api/)?(buckets|containers|kernels)"),
|
||||
("Spaces runtime control", r"^[A-Z]+ /api/spaces/(zero-gpu|\{namespace\}/\{repo\}/(runtime|restart|pause))"),
|
||||
("partner provider mapping writes and inference-provider account settings",
|
||||
r"^(POST|PUT) /api/(partners|settings/inference-providers)"),
|
||||
("Inference Endpoints auth-check helper (internal)", r"/api/inference-endpoints/"),
|
||||
("OAuth, docs search, agents, sql console, registry token, repo search POST variant",
|
||||
r"^[A-Z]+ /(oauth|api/(docs|agent-harnesses|registry|sql-console|\{repoType\}/\{namespace\}/\{repo\}/sql-console|quicksearch$))"),
|
||||
("blog, MCP settings, likes", r"^[A-Z]+ /api/(blog|settings|trending-.*)"),
|
||||
]
|
||||
|
||||
|
||||
def load(cache, name):
|
||||
if cache:
|
||||
p = os.path.join(cache, name)
|
||||
if os.path.exists(p):
|
||||
return open(p, "rb").read()
|
||||
url = URLS.get(name) or name
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "molo-check-spec/1"})
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
data = r.read()
|
||||
if cache:
|
||||
os.makedirs(os.path.dirname(os.path.join(cache, name)), exist_ok=True)
|
||||
open(os.path.join(cache, name), "wb").write(data)
|
||||
return data
|
||||
|
||||
|
||||
def resolve(doc, node):
|
||||
seen = 0
|
||||
while isinstance(node, dict) and "$ref" in node and seen < 20:
|
||||
cur = doc
|
||||
for p in node["$ref"].lstrip("#/").split("/"):
|
||||
cur = cur[p]
|
||||
node = cur
|
||||
seen += 1
|
||||
return node
|
||||
|
||||
|
||||
def props(doc, schema):
|
||||
schema = resolve(doc, schema)
|
||||
if not isinstance(schema, dict):
|
||||
return {}
|
||||
out = dict(schema.get("properties", {}))
|
||||
for key in ("allOf", "anyOf", "oneOf"):
|
||||
for sub in schema.get(key, []):
|
||||
out.update(props(doc, sub))
|
||||
return out
|
||||
|
||||
|
||||
def op_fields(doc, op):
|
||||
q = set()
|
||||
for p in op.get("parameters", []):
|
||||
p = resolve(doc, p)
|
||||
if p.get("in") == "query":
|
||||
q.add(p["name"])
|
||||
body = set()
|
||||
rb = resolve(doc, op.get("requestBody", {}))
|
||||
if rb:
|
||||
content = rb.get("content", {})
|
||||
ct = next(iter(content), None)
|
||||
if ct:
|
||||
body = set(props(doc, content[ct].get("schema")).keys())
|
||||
return q, body
|
||||
|
||||
|
||||
def unesc(s):
|
||||
return s.replace('\\"', '"').replace("\\\\", "\\")
|
||||
|
||||
|
||||
def variants(path, doc_paths=()):
|
||||
# The document may carry {repoType} literally (discussions) or one path per
|
||||
# repo type (everything else); a literal match wins.
|
||||
if path in doc_paths:
|
||||
return [path]
|
||||
if "{repoType}" in path:
|
||||
return [path.replace("{repoType}", t) for t in REPO_TYPES]
|
||||
if "{typePrefix}" in path:
|
||||
return [path.replace("{typePrefix}", p) for p in PREFIXES]
|
||||
return [path]
|
||||
|
||||
|
||||
def csv(s):
|
||||
return [x for x in s.split(",") if x]
|
||||
|
||||
|
||||
def probe_url(surface, path):
|
||||
subs = {"{namespace}": "openai-community", "{repo}": "gpt2", "{rev}": "main", "{path}": "config.json",
|
||||
"{model}": "openai-community/gpt2", "{task}": "feature-extraction", "{repoType}": "models"}
|
||||
for k, v in subs.items():
|
||||
path = path.replace(k, v)
|
||||
return HOSTS[surface] + path
|
||||
|
||||
|
||||
def main():
|
||||
cache = None
|
||||
if "--cache" in sys.argv:
|
||||
cache = sys.argv[sys.argv.index("--cache") + 1]
|
||||
text = open(TABLE).read()
|
||||
begin, end = text.index("// CALLS-BEGIN"), text.index("// CALLS-END")
|
||||
rows = []
|
||||
for line in text[begin:end].splitlines():
|
||||
m = ROW.match(line)
|
||||
if m:
|
||||
g = m.groups()
|
||||
rows.append(dict(name=g[0], surface=g[1], method=unesc(g[2]), path=unesc(g[3]), spec=unesc(g[4]),
|
||||
ref=unesc(g[5]), auth=unesc(g[6]), flags=unesc(g[7]), body=csv(unesc(g[8])),
|
||||
query=csv(unesc(g[9]))))
|
||||
if not rows:
|
||||
print("no rows parsed from calls.cpp")
|
||||
return 1
|
||||
docs = {}
|
||||
shas = {}
|
||||
for key in ("hub", "ie", "tgi", "tei"):
|
||||
raw = load(cache, key + ".json")
|
||||
shas[key] = hashlib.sha256(raw).hexdigest()[:12]
|
||||
docs[key] = json.loads(raw)
|
||||
sources = ""
|
||||
for name in ("hf_api.py", "hf_inference.py", "constants.py"):
|
||||
raw = load(cache, name)
|
||||
shas[name] = hashlib.sha256(raw).hexdigest()[:12]
|
||||
sources += raw.decode("utf-8", "replace")
|
||||
ops = {}
|
||||
for key, doc in docs.items():
|
||||
ops[key] = {}
|
||||
for path, item in doc["paths"].items():
|
||||
for method, op in item.items():
|
||||
if method in METHODS:
|
||||
ops[key][(method.upper(), path)] = op
|
||||
bad = []
|
||||
names = set()
|
||||
covered = {k: set() for k in docs}
|
||||
notes = {"hub_unlisted_query": 0, "hub_unlisted_body": 0}
|
||||
for r in rows:
|
||||
if r["name"] in names:
|
||||
bad.append("duplicate row %s" % r["name"])
|
||||
names.add(r["name"])
|
||||
spec = r["spec"]
|
||||
if spec in docs:
|
||||
union_q, union_b, found = set(), set(), False
|
||||
for v in variants(r["path"], docs[spec]["paths"]):
|
||||
op = ops[spec].get((r["method"], v))
|
||||
if op is None:
|
||||
bad.append("%s: %s %s not in the %s document" % (r["name"], r["method"], v, spec))
|
||||
continue
|
||||
found = True
|
||||
covered[spec].add((r["method"], v))
|
||||
if spec != "hub" and r["ref"] != "-" and op.get("operationId") != r["ref"]:
|
||||
bad.append("%s: operationId %s, table says %s" % (r["name"], op.get("operationId"), r["ref"]))
|
||||
q, b = op_fields(docs[spec], op)
|
||||
union_q |= q
|
||||
union_b |= b
|
||||
if not found:
|
||||
continue
|
||||
for f in r["query"]:
|
||||
if f not in union_q:
|
||||
bad.append("%s: query parameter %s absent from the document" % (r["name"], f))
|
||||
for f in r["body"]:
|
||||
if f not in union_b:
|
||||
bad.append("%s: body field %s absent from the document" % (r["name"], f))
|
||||
if spec == "hub":
|
||||
notes["hub_unlisted_query"] += len(union_q - set(r["query"]))
|
||||
notes["hub_unlisted_body"] += len(union_b - set(r["body"]))
|
||||
else:
|
||||
for f in union_q - set(r["query"]):
|
||||
bad.append("%s: document query parameter %s not in the table" % (r["name"], f))
|
||||
for f in union_b - set(r["body"]):
|
||||
bad.append("%s: document body field %s not in the table" % (r["name"], f))
|
||||
elif spec == "src":
|
||||
if r["ref"] not in sources:
|
||||
bad.append("%s: source needle %r not found in huggingface_hub" % (r["name"], r["ref"]))
|
||||
elif spec == "task":
|
||||
try:
|
||||
schema = json.loads(load(cache, "tasks/%s.json" % r["ref"]) if cache and os.path.exists(os.path.join(cache, "tasks/%s.json" % r["ref"])) else load(None, TASK_URL % r["ref"]))
|
||||
except Exception as e: # noqa
|
||||
bad.append("%s: cannot load the %s task schema (%s)" % (r["name"], r["ref"], e))
|
||||
continue
|
||||
allowed = set(schema.get("properties", {}).keys())
|
||||
for f in r["body"]:
|
||||
if f not in allowed:
|
||||
bad.append("%s: body field %s absent from the %s task schema" % (r["name"], f, r["ref"]))
|
||||
for f in allowed - set(r["body"]):
|
||||
bad.append("%s: task schema field %s not in the table" % (r["name"], f))
|
||||
elif spec != "none":
|
||||
bad.append("%s: unknown spec kind %s" % (r["name"], spec))
|
||||
# Coverage.
|
||||
for key in ("ie", "tgi", "tei"):
|
||||
for k in ops[key]:
|
||||
if k not in covered[key]:
|
||||
bad.append("unwrapped %s operation %s %s" % (key, k[0], k[1]))
|
||||
excluded_count = {}
|
||||
unwrapped_hub = 0
|
||||
for (method, path), op in sorted(ops["hub"].items()):
|
||||
if (method, path) in covered["hub"]:
|
||||
continue
|
||||
label = "%s %s" % (method, path)
|
||||
reason = None
|
||||
for why, rx in EXCLUDED:
|
||||
if re.search(rx, label):
|
||||
reason = why
|
||||
break
|
||||
if reason is None:
|
||||
bad.append("unwrapped hub operation %s (tags %s)" % (label, op.get("tags")))
|
||||
else:
|
||||
excluded_count[reason] = excluded_count.get(reason, 0) + 1
|
||||
unwrapped_hub += 1
|
||||
if "--probe" in sys.argv:
|
||||
for r in rows:
|
||||
if r["spec"] not in ("src", "none") or r["surface"] not in HOSTS or r["method"] != "GET":
|
||||
continue
|
||||
if "{" in r["path"].replace("{namespace}", "").replace("{repo}", "").replace("{rev}", "").replace("{path}", "").replace("{model}", ""):
|
||||
continue
|
||||
url = probe_url(r["surface"], r["path"])
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "molo-check-spec/1"})
|
||||
try:
|
||||
status = urllib.request.urlopen(req, timeout=60).status
|
||||
except urllib.error.HTTPError as e:
|
||||
status = e.code
|
||||
print("probe GET %s -> %d" % (url, status))
|
||||
if status == 404:
|
||||
bad.append("%s: probe answered 404" % r["name"])
|
||||
for b in bad:
|
||||
print("FAIL " + b)
|
||||
print("sources: " + ", ".join("%s=%s" % kv for kv in sorted(shas.items())))
|
||||
print("hub operations excluded (%d):" % unwrapped_hub)
|
||||
for why, n in sorted(excluded_count.items()):
|
||||
print(" %3d %s" % (n, why))
|
||||
print("hub columns the module does not send: %d query parameters, %d body fields (information)"
|
||||
% (notes["hub_unlisted_query"], notes["hub_unlisted_body"]))
|
||||
counts = {k: len(v) for k, v in ops.items()}
|
||||
print("%d rows checked; spec operations hub=%d ie=%d tgi=%d tei=%d; %d problems"
|
||||
% (len(rows), counts["hub"], counts["ie"], counts["tgi"], counts["tei"], len(bad)))
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user