sessions: operations get/list/cancel/delete/wait for engine, session, memory and serving profile; servingProfiles CRUD. registry: ListResult with truncated flag, v1alpha twins, gateway/policy operations, authz IAM, deny policy update, boundary patch/bindings. optimize: make_gemini_model over models::VertexModel (DirectModel removed), generateSyntheticData, online evaluator operations, trace/log writers. armor: SecurityAdmin for all SCC v2 methods (attack exposure excluded), documented agent threat categories, floor settings fixed to location global after live run. check_discovery.py of all four enforce coverage. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
203 lines
11 KiB
Python
203 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Verify the optimize call table against Google's Discovery documents.
|
|
|
|
Fails (exit 1) when a declared call is absent, differs in HTTP method or path,
|
|
or declares a request field or query parameter Discovery does not list, and
|
|
when a response/request schema field the builders and parsers rely on is
|
|
absent. Python 3 stdlib only; needs network, no credential.
|
|
|
|
python3 -I src/google/optimize/tools/check_discovery.py
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import urllib.request
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
TABLE = os.path.join(HERE, '..', 'calls_table.inc')
|
|
|
|
DISCOVERY = {
|
|
('aiplatform', 'v1'): 'https://aiplatform.googleapis.com/$discovery/rest?version=v1',
|
|
('cloudtrace', 'v1'): 'https://cloudtrace.googleapis.com/$discovery/rest?version=v1',
|
|
('cloudtrace', 'v2'): 'https://cloudtrace.googleapis.com/$discovery/rest?version=v2',
|
|
('logging', 'v2'): 'https://logging.googleapis.com/$discovery/rest?version=v2',
|
|
}
|
|
|
|
AI = 'GoogleCloudAiplatformV1'
|
|
# (service, version, schema, fields the C++ code reads or writes)
|
|
SCHEMA_FIELDS = [
|
|
('aiplatform', 'v1', AI + 'EvaluateInstancesRequest',
|
|
'safetyInput exactMatchInput pointwiseMetricInput pairwiseMetricInput autoraterConfig '
|
|
'trajectoryExactMatchInput trajectoryInOrderMatchInput trajectoryAnyOrderMatchInput '
|
|
'trajectoryPrecisionInput trajectoryRecallInput trajectorySingleToolUseInput'),
|
|
('aiplatform', 'v1', AI + 'EvaluateInstancesResponse',
|
|
'exactMatchResults safetyResult pointwiseMetricResult pairwiseMetricResult '
|
|
'trajectoryExactMatchResults trajectoryInOrderMatchResults trajectoryAnyOrderMatchResults '
|
|
'trajectoryPrecisionResults trajectoryRecallResults trajectorySingleToolUseResults'),
|
|
('aiplatform', 'v1', AI + 'SafetyInput', 'metricSpec instance'),
|
|
('aiplatform', 'v1', AI + 'SafetyInstance', 'prediction'),
|
|
('aiplatform', 'v1', AI + 'SafetyResult', 'score explanation'),
|
|
('aiplatform', 'v1', AI + 'ExactMatchInput', 'metricSpec instances'),
|
|
('aiplatform', 'v1', AI + 'ExactMatchInstance', 'prediction reference'),
|
|
('aiplatform', 'v1', AI + 'ExactMatchResults', 'exactMatchMetricValues'),
|
|
('aiplatform', 'v1', AI + 'ExactMatchMetricValue', 'score'),
|
|
('aiplatform', 'v1', AI + 'PointwiseMetricInput', 'metricSpec instance'),
|
|
('aiplatform', 'v1', AI + 'PointwiseMetricSpec', 'metricPromptTemplate systemInstruction'),
|
|
('aiplatform', 'v1', AI + 'PointwiseMetricInstance', 'jsonInstance'),
|
|
('aiplatform', 'v1', AI + 'PointwiseMetricResult', 'score explanation'),
|
|
('aiplatform', 'v1', AI + 'PairwiseMetricInput', 'metricSpec instance'),
|
|
('aiplatform', 'v1', AI + 'PairwiseMetricSpec',
|
|
'metricPromptTemplate baselineResponseFieldName candidateResponseFieldName systemInstruction'),
|
|
('aiplatform', 'v1', AI + 'PairwiseMetricInstance', 'jsonInstance'),
|
|
('aiplatform', 'v1', AI + 'PairwiseMetricResult', 'pairwiseChoice explanation'),
|
|
('aiplatform', 'v1', AI + 'TrajectoryExactMatchInput', 'metricSpec instances'),
|
|
('aiplatform', 'v1', AI + 'TrajectoryExactMatchInstance', 'predictedTrajectory referenceTrajectory'),
|
|
('aiplatform', 'v1', AI + 'TrajectoryExactMatchResults', 'trajectoryExactMatchMetricValues'),
|
|
('aiplatform', 'v1', AI + 'TrajectoryInOrderMatchResults', 'trajectoryInOrderMatchMetricValues'),
|
|
('aiplatform', 'v1', AI + 'TrajectoryAnyOrderMatchResults', 'trajectoryAnyOrderMatchMetricValues'),
|
|
('aiplatform', 'v1', AI + 'TrajectoryPrecisionResults', 'trajectoryPrecisionMetricValues'),
|
|
('aiplatform', 'v1', AI + 'TrajectoryRecallResults', 'trajectoryRecallMetricValues'),
|
|
('aiplatform', 'v1', AI + 'TrajectorySingleToolUseResults', 'trajectorySingleToolUseMetricValues'),
|
|
('aiplatform', 'v1', AI + 'TrajectorySingleToolUseSpec', 'toolName'),
|
|
('aiplatform', 'v1', AI + 'TrajectorySingleToolUseInstance', 'predictedTrajectory'),
|
|
('aiplatform', 'v1', AI + 'Trajectory', 'toolCalls'),
|
|
('aiplatform', 'v1', AI + 'ToolCall', 'toolName toolInput'),
|
|
('aiplatform', 'v1', AI + 'AutoraterConfig', 'autoraterModel samplingCount flipEnabled'),
|
|
('aiplatform', 'v1', AI + 'UserScenarioGenerationConfig',
|
|
'userScenarioCount simulationInstruction environmentData modelName'),
|
|
('aiplatform', 'v1', AI + 'GenerateUserScenariosResponse', 'userScenarios'),
|
|
('aiplatform', 'v1', AI + 'UserScenario', 'testCaseTitle startingPrompt conversationPlan'),
|
|
('aiplatform', 'v1', AI + 'GenerateLossClustersRequest', 'configs evaluationSet'),
|
|
('aiplatform', 'v1', AI + 'LossAnalysisConfig', 'metric candidate'),
|
|
('aiplatform', 'v1', AI + 'EvaluationRun',
|
|
'displayName dataSource evaluationConfig inferenceConfigs state'),
|
|
('aiplatform', 'v1', AI + 'EvaluationRunDataSource', 'evaluationSet'),
|
|
('aiplatform', 'v1', AI + 'EvaluationRunEvaluationConfig', 'metrics autoraterConfig'),
|
|
('aiplatform', 'v1', AI + 'EvaluationRunMetric', 'predefinedMetricSpec'),
|
|
('aiplatform', 'v1', AI + 'EvaluationRunMetricPredefinedMetricSpec', 'metricSpecName'),
|
|
('aiplatform', 'v1', AI + 'EvaluationRunEvaluationConfigAutoraterConfig', 'autoraterModel sampleCount'),
|
|
('aiplatform', 'v1', AI + 'EvaluationRunInferenceConfig', 'model agentRunConfig'),
|
|
('aiplatform', 'v1', AI + 'EvaluationRunInferenceConfigAgentRunConfig', 'agentEngine userSimulatorConfig'),
|
|
('aiplatform', 'v1', AI + 'EvaluationRunInferenceConfigAgentRunConfigUserSimulatorConfig', 'maxTurn modelName'),
|
|
('aiplatform', 'v1', AI + 'OnlineEvaluator',
|
|
'displayName agentResource metricSources config cloudObservability'),
|
|
('aiplatform', 'v1', AI + 'MetricSource', 'metricResourceName'),
|
|
('aiplatform', 'v1', AI + 'OnlineEvaluatorConfig', 'randomSampling maxEvaluatedSamplesPerRun'),
|
|
('aiplatform', 'v1', AI + 'OnlineEvaluatorConfigRandomSampling', 'percentage'),
|
|
('aiplatform', 'v1', AI + 'OnlineEvaluatorCloudObservability', 'openTelemetry'),
|
|
('aiplatform', 'v1', AI + 'OnlineEvaluatorCloudObservabilityOpenTelemetry', 'semconvVersion'),
|
|
('aiplatform', 'v1', AI + 'EvaluationItem', 'displayName evaluationItemType evaluationRequest'),
|
|
('aiplatform', 'v1', AI + 'EvaluationRequest', 'prompt'),
|
|
('aiplatform', 'v1', AI + 'EvaluationPrompt', 'text userScenario'),
|
|
('aiplatform', 'v1', AI + 'EvaluationPromptUserScenario', 'startingPrompt conversationPlan'),
|
|
('aiplatform', 'v1', AI + 'ListEvaluationRunsResponse', 'evaluationRuns nextPageToken'),
|
|
('aiplatform', 'v1', AI + 'ListEvaluationSetsResponse', 'evaluationSets nextPageToken'),
|
|
('aiplatform', 'v1', AI + 'ListEvaluationItemsResponse', 'evaluationItems nextPageToken'),
|
|
('aiplatform', 'v1', AI + 'ListEvaluationMetricsResponse', 'evaluationMetrics nextPageToken'),
|
|
('aiplatform', 'v1', AI + 'ListOnlineEvaluatorsResponse', 'onlineEvaluators nextPageToken'),
|
|
('cloudtrace', 'v1', 'ListTracesResponse', 'traces nextPageToken'),
|
|
('cloudtrace', 'v1', 'Trace', 'traceId projectId spans'),
|
|
('cloudtrace', 'v1', 'TraceSpan', 'spanId parentSpanId name kind startTime endTime labels'),
|
|
('logging', 'v2', 'ListLogEntriesResponse', 'entries nextPageToken'),
|
|
('logging', 'v2', 'LogEntry',
|
|
'logName timestamp severity trace spanId textPayload jsonPayload labels'),
|
|
]
|
|
# (service, version, schema, property, enum values the code compares against)
|
|
ENUM_VALUES = [
|
|
('aiplatform', 'v1', AI + 'EvaluationRun', 'state', 'SUCCEEDED FAILED CANCELLED'),
|
|
('aiplatform', 'v1', AI + 'EvaluationItem', 'evaluationItemType', 'REQUEST'),
|
|
('aiplatform', 'v1', AI + 'PairwiseMetricResult', 'pairwiseChoice', 'BASELINE CANDIDATE TIE'),
|
|
]
|
|
|
|
|
|
# Coverage: every Discovery method with one of these id prefixes must be a
|
|
# table row or be listed in EXCLUDED with its reason.
|
|
COVERED = [
|
|
('aiplatform', 'v1', ['aiplatform.projects.locations.evaluat',
|
|
'aiplatform.projects.locations.generate',
|
|
'aiplatform.projects.locations.onlineEvaluators']),
|
|
('cloudtrace', 'v1', ['']),
|
|
('cloudtrace', 'v2', ['']),
|
|
('logging', 'v2', ['logging.entries']),
|
|
]
|
|
EXCLUDED = {
|
|
'logging.entries.tail': 'server-streaming RPC with no end; this module has no unbounded stream (hard caps rule)',
|
|
}
|
|
|
|
|
|
def fetch(url):
|
|
with urllib.request.urlopen(url, timeout=60) as r:
|
|
return json.load(r)
|
|
|
|
|
|
def methods(node):
|
|
for m in node.get('methods', {}).values():
|
|
yield m
|
|
for child in node.get('resources', {}).values():
|
|
yield from methods(child)
|
|
|
|
|
|
def parse_table():
|
|
text = open(TABLE).read()
|
|
rows = re.findall(r'\{"([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)",\s*"([^"]*)",\s*"([^"]*)"\}', text)
|
|
if not rows:
|
|
sys.exit('no rows parsed from calls_table.inc')
|
|
return rows
|
|
|
|
|
|
def main():
|
|
docs = {k: fetch(u) for k, u in DISCOVERY.items()}
|
|
errors = []
|
|
rows = parse_table()
|
|
for rpc_id, http, path, svc, ver, body_csv, query_csv in rows:
|
|
doc = docs.get((svc, ver))
|
|
if doc is None:
|
|
errors.append('%s: no discovery doc for %s %s' % (rpc_id, svc, ver))
|
|
continue
|
|
found = {m['id']: m for m in methods(doc)}
|
|
m = found.get(rpc_id)
|
|
if m is None:
|
|
errors.append('%s: absent from discovery' % rpc_id)
|
|
continue
|
|
if m['httpMethod'] != http:
|
|
errors.append('%s: verb %s != %s' % (rpc_id, http, m['httpMethod']))
|
|
if m['path'] != ver + path:
|
|
errors.append('%s: path %s != %s' % (rpc_id, ver + path, m['path']))
|
|
ref = (m.get('request') or {}).get('$ref')
|
|
want_body = sorted(doc['schemas'][ref]['properties']) if ref else []
|
|
if sorted(x for x in body_csv.split(',') if x) != want_body:
|
|
errors.append('%s: body fields differ from discovery' % rpc_id)
|
|
want_query = sorted(k for k, v in m.get('parameters', {}).items() if v['location'] == 'query')
|
|
if sorted(x for x in query_csv.split(',') if x) != want_query:
|
|
errors.append('%s: query params differ from discovery' % rpc_id)
|
|
declared = {(r[3], r[4], r[0]) for r in rows}
|
|
for svc, ver, prefixes in COVERED:
|
|
for m in methods(docs[(svc, ver)]):
|
|
if any(m['id'].startswith(p) for p in prefixes) and (svc, ver, m['id']) not in declared \
|
|
and m['id'] not in EXCLUDED:
|
|
errors.append('discovery method %s is not in the table' % m['id'])
|
|
for svc, ver, schema, fields in SCHEMA_FIELDS:
|
|
s = docs[(svc, ver)]['schemas'].get(schema)
|
|
if s is None:
|
|
errors.append('schema %s absent' % schema)
|
|
continue
|
|
for f in fields.split():
|
|
if f not in s.get('properties', {}):
|
|
errors.append('%s.%s absent' % (schema, f))
|
|
for svc, ver, schema, prop, values in ENUM_VALUES:
|
|
p = docs[(svc, ver)]['schemas'][schema]['properties'][prop]
|
|
for v in values.split():
|
|
if v not in p.get('enum', []):
|
|
errors.append('%s.%s lacks enum %s' % (schema, prop, v))
|
|
if errors:
|
|
print('FAIL: %d discovery mismatches' % len(errors))
|
|
for e in errors:
|
|
print(' ' + e)
|
|
sys.exit(1)
|
|
print('OK: %d calls and %d schemas match discovery' % (len(rows), len(SCHEMA_FIELDS)))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|