Files
moloapp/src/google/optimize/tools/check_discovery.py
T
retoorandClaude Sonnet 5.5 a3396bfc68 feat(clients): sessions+registry+optimize+armor: close gaps, loud list truncation, shared Gemini adapter
sessions: operations get/list/cancel/delete/wait for engine, session, memory and
serving profile; servingProfiles CRUD. registry: ListResult with truncated flag,
v1alpha twins, gateway/policy operations, authz IAM, deny policy update, boundary
patch/bindings. optimize: make_gemini_model over models::VertexModel (DirectModel
removed), generateSyntheticData, online evaluator operations, trace/log writers.
armor: SecurityAdmin for all SCC v2 methods (attack exposure excluded), documented
agent threat categories, floor settings fixed to location global after live run.
check_discovery.py of all four enforce coverage.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-24 05:43:13 +02:00

203 lines
11 KiB
Python

#!/usr/bin/env python3
"""Verify the optimize call table against Google's Discovery documents.
Fails (exit 1) when a declared call is absent, differs in HTTP method or path,
or declares a request field or query parameter Discovery does not list, and
when a response/request schema field the builders and parsers rely on is
absent. Python 3 stdlib only; needs network, no credential.
python3 -I src/google/optimize/tools/check_discovery.py
"""
import json
import os
import re
import sys
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
TABLE = os.path.join(HERE, '..', 'calls_table.inc')
DISCOVERY = {
('aiplatform', 'v1'): 'https://aiplatform.googleapis.com/$discovery/rest?version=v1',
('cloudtrace', 'v1'): 'https://cloudtrace.googleapis.com/$discovery/rest?version=v1',
('cloudtrace', 'v2'): 'https://cloudtrace.googleapis.com/$discovery/rest?version=v2',
('logging', 'v2'): 'https://logging.googleapis.com/$discovery/rest?version=v2',
}
AI = 'GoogleCloudAiplatformV1'
# (service, version, schema, fields the C++ code reads or writes)
SCHEMA_FIELDS = [
('aiplatform', 'v1', AI + 'EvaluateInstancesRequest',
'safetyInput exactMatchInput pointwiseMetricInput pairwiseMetricInput autoraterConfig '
'trajectoryExactMatchInput trajectoryInOrderMatchInput trajectoryAnyOrderMatchInput '
'trajectoryPrecisionInput trajectoryRecallInput trajectorySingleToolUseInput'),
('aiplatform', 'v1', AI + 'EvaluateInstancesResponse',
'exactMatchResults safetyResult pointwiseMetricResult pairwiseMetricResult '
'trajectoryExactMatchResults trajectoryInOrderMatchResults trajectoryAnyOrderMatchResults '
'trajectoryPrecisionResults trajectoryRecallResults trajectorySingleToolUseResults'),
('aiplatform', 'v1', AI + 'SafetyInput', 'metricSpec instance'),
('aiplatform', 'v1', AI + 'SafetyInstance', 'prediction'),
('aiplatform', 'v1', AI + 'SafetyResult', 'score explanation'),
('aiplatform', 'v1', AI + 'ExactMatchInput', 'metricSpec instances'),
('aiplatform', 'v1', AI + 'ExactMatchInstance', 'prediction reference'),
('aiplatform', 'v1', AI + 'ExactMatchResults', 'exactMatchMetricValues'),
('aiplatform', 'v1', AI + 'ExactMatchMetricValue', 'score'),
('aiplatform', 'v1', AI + 'PointwiseMetricInput', 'metricSpec instance'),
('aiplatform', 'v1', AI + 'PointwiseMetricSpec', 'metricPromptTemplate systemInstruction'),
('aiplatform', 'v1', AI + 'PointwiseMetricInstance', 'jsonInstance'),
('aiplatform', 'v1', AI + 'PointwiseMetricResult', 'score explanation'),
('aiplatform', 'v1', AI + 'PairwiseMetricInput', 'metricSpec instance'),
('aiplatform', 'v1', AI + 'PairwiseMetricSpec',
'metricPromptTemplate baselineResponseFieldName candidateResponseFieldName systemInstruction'),
('aiplatform', 'v1', AI + 'PairwiseMetricInstance', 'jsonInstance'),
('aiplatform', 'v1', AI + 'PairwiseMetricResult', 'pairwiseChoice explanation'),
('aiplatform', 'v1', AI + 'TrajectoryExactMatchInput', 'metricSpec instances'),
('aiplatform', 'v1', AI + 'TrajectoryExactMatchInstance', 'predictedTrajectory referenceTrajectory'),
('aiplatform', 'v1', AI + 'TrajectoryExactMatchResults', 'trajectoryExactMatchMetricValues'),
('aiplatform', 'v1', AI + 'TrajectoryInOrderMatchResults', 'trajectoryInOrderMatchMetricValues'),
('aiplatform', 'v1', AI + 'TrajectoryAnyOrderMatchResults', 'trajectoryAnyOrderMatchMetricValues'),
('aiplatform', 'v1', AI + 'TrajectoryPrecisionResults', 'trajectoryPrecisionMetricValues'),
('aiplatform', 'v1', AI + 'TrajectoryRecallResults', 'trajectoryRecallMetricValues'),
('aiplatform', 'v1', AI + 'TrajectorySingleToolUseResults', 'trajectorySingleToolUseMetricValues'),
('aiplatform', 'v1', AI + 'TrajectorySingleToolUseSpec', 'toolName'),
('aiplatform', 'v1', AI + 'TrajectorySingleToolUseInstance', 'predictedTrajectory'),
('aiplatform', 'v1', AI + 'Trajectory', 'toolCalls'),
('aiplatform', 'v1', AI + 'ToolCall', 'toolName toolInput'),
('aiplatform', 'v1', AI + 'AutoraterConfig', 'autoraterModel samplingCount flipEnabled'),
('aiplatform', 'v1', AI + 'UserScenarioGenerationConfig',
'userScenarioCount simulationInstruction environmentData modelName'),
('aiplatform', 'v1', AI + 'GenerateUserScenariosResponse', 'userScenarios'),
('aiplatform', 'v1', AI + 'UserScenario', 'testCaseTitle startingPrompt conversationPlan'),
('aiplatform', 'v1', AI + 'GenerateLossClustersRequest', 'configs evaluationSet'),
('aiplatform', 'v1', AI + 'LossAnalysisConfig', 'metric candidate'),
('aiplatform', 'v1', AI + 'EvaluationRun',
'displayName dataSource evaluationConfig inferenceConfigs state'),
('aiplatform', 'v1', AI + 'EvaluationRunDataSource', 'evaluationSet'),
('aiplatform', 'v1', AI + 'EvaluationRunEvaluationConfig', 'metrics autoraterConfig'),
('aiplatform', 'v1', AI + 'EvaluationRunMetric', 'predefinedMetricSpec'),
('aiplatform', 'v1', AI + 'EvaluationRunMetricPredefinedMetricSpec', 'metricSpecName'),
('aiplatform', 'v1', AI + 'EvaluationRunEvaluationConfigAutoraterConfig', 'autoraterModel sampleCount'),
('aiplatform', 'v1', AI + 'EvaluationRunInferenceConfig', 'model agentRunConfig'),
('aiplatform', 'v1', AI + 'EvaluationRunInferenceConfigAgentRunConfig', 'agentEngine userSimulatorConfig'),
('aiplatform', 'v1', AI + 'EvaluationRunInferenceConfigAgentRunConfigUserSimulatorConfig', 'maxTurn modelName'),
('aiplatform', 'v1', AI + 'OnlineEvaluator',
'displayName agentResource metricSources config cloudObservability'),
('aiplatform', 'v1', AI + 'MetricSource', 'metricResourceName'),
('aiplatform', 'v1', AI + 'OnlineEvaluatorConfig', 'randomSampling maxEvaluatedSamplesPerRun'),
('aiplatform', 'v1', AI + 'OnlineEvaluatorConfigRandomSampling', 'percentage'),
('aiplatform', 'v1', AI + 'OnlineEvaluatorCloudObservability', 'openTelemetry'),
('aiplatform', 'v1', AI + 'OnlineEvaluatorCloudObservabilityOpenTelemetry', 'semconvVersion'),
('aiplatform', 'v1', AI + 'EvaluationItem', 'displayName evaluationItemType evaluationRequest'),
('aiplatform', 'v1', AI + 'EvaluationRequest', 'prompt'),
('aiplatform', 'v1', AI + 'EvaluationPrompt', 'text userScenario'),
('aiplatform', 'v1', AI + 'EvaluationPromptUserScenario', 'startingPrompt conversationPlan'),
('aiplatform', 'v1', AI + 'ListEvaluationRunsResponse', 'evaluationRuns nextPageToken'),
('aiplatform', 'v1', AI + 'ListEvaluationSetsResponse', 'evaluationSets nextPageToken'),
('aiplatform', 'v1', AI + 'ListEvaluationItemsResponse', 'evaluationItems nextPageToken'),
('aiplatform', 'v1', AI + 'ListEvaluationMetricsResponse', 'evaluationMetrics nextPageToken'),
('aiplatform', 'v1', AI + 'ListOnlineEvaluatorsResponse', 'onlineEvaluators nextPageToken'),
('cloudtrace', 'v1', 'ListTracesResponse', 'traces nextPageToken'),
('cloudtrace', 'v1', 'Trace', 'traceId projectId spans'),
('cloudtrace', 'v1', 'TraceSpan', 'spanId parentSpanId name kind startTime endTime labels'),
('logging', 'v2', 'ListLogEntriesResponse', 'entries nextPageToken'),
('logging', 'v2', 'LogEntry',
'logName timestamp severity trace spanId textPayload jsonPayload labels'),
]
# (service, version, schema, property, enum values the code compares against)
ENUM_VALUES = [
('aiplatform', 'v1', AI + 'EvaluationRun', 'state', 'SUCCEEDED FAILED CANCELLED'),
('aiplatform', 'v1', AI + 'EvaluationItem', 'evaluationItemType', 'REQUEST'),
('aiplatform', 'v1', AI + 'PairwiseMetricResult', 'pairwiseChoice', 'BASELINE CANDIDATE TIE'),
]
# Coverage: every Discovery method with one of these id prefixes must be a
# table row or be listed in EXCLUDED with its reason.
COVERED = [
('aiplatform', 'v1', ['aiplatform.projects.locations.evaluat',
'aiplatform.projects.locations.generate',
'aiplatform.projects.locations.onlineEvaluators']),
('cloudtrace', 'v1', ['']),
('cloudtrace', 'v2', ['']),
('logging', 'v2', ['logging.entries']),
]
EXCLUDED = {
'logging.entries.tail': 'server-streaming RPC with no end; this module has no unbounded stream (hard caps rule)',
}
def fetch(url):
with urllib.request.urlopen(url, timeout=60) as r:
return json.load(r)
def methods(node):
for m in node.get('methods', {}).values():
yield m
for child in node.get('resources', {}).values():
yield from methods(child)
def parse_table():
text = open(TABLE).read()
rows = re.findall(r'\{"([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)",\s*"([^"]*)",\s*"([^"]*)"\}', text)
if not rows:
sys.exit('no rows parsed from calls_table.inc')
return rows
def main():
docs = {k: fetch(u) for k, u in DISCOVERY.items()}
errors = []
rows = parse_table()
for rpc_id, http, path, svc, ver, body_csv, query_csv in rows:
doc = docs.get((svc, ver))
if doc is None:
errors.append('%s: no discovery doc for %s %s' % (rpc_id, svc, ver))
continue
found = {m['id']: m for m in methods(doc)}
m = found.get(rpc_id)
if m is None:
errors.append('%s: absent from discovery' % rpc_id)
continue
if m['httpMethod'] != http:
errors.append('%s: verb %s != %s' % (rpc_id, http, m['httpMethod']))
if m['path'] != ver + path:
errors.append('%s: path %s != %s' % (rpc_id, ver + path, m['path']))
ref = (m.get('request') or {}).get('$ref')
want_body = sorted(doc['schemas'][ref]['properties']) if ref else []
if sorted(x for x in body_csv.split(',') if x) != want_body:
errors.append('%s: body fields differ from discovery' % rpc_id)
want_query = sorted(k for k, v in m.get('parameters', {}).items() if v['location'] == 'query')
if sorted(x for x in query_csv.split(',') if x) != want_query:
errors.append('%s: query params differ from discovery' % rpc_id)
declared = {(r[3], r[4], r[0]) for r in rows}
for svc, ver, prefixes in COVERED:
for m in methods(docs[(svc, ver)]):
if any(m['id'].startswith(p) for p in prefixes) and (svc, ver, m['id']) not in declared \
and m['id'] not in EXCLUDED:
errors.append('discovery method %s is not in the table' % m['id'])
for svc, ver, schema, fields in SCHEMA_FIELDS:
s = docs[(svc, ver)]['schemas'].get(schema)
if s is None:
errors.append('schema %s absent' % schema)
continue
for f in fields.split():
if f not in s.get('properties', {}):
errors.append('%s.%s absent' % (schema, f))
for svc, ver, schema, prop, values in ENUM_VALUES:
p = docs[(svc, ver)]['schemas'][schema]['properties'][prop]
for v in values.split():
if v not in p.get('enum', []):
errors.append('%s.%s lacks enum %s' % (schema, prop, v))
if errors:
print('FAIL: %d discovery mismatches' % len(errors))
for e in errors:
print(' ' + e)
sys.exit(1)
print('OK: %d calls and %d schemas match discovery' % (len(rows), len(SCHEMA_FIELDS)))
if __name__ == '__main__':
main()