diff --git a/example/app/google/DiscoveryDemo.cpp b/example/app/google/DiscoveryDemo.cpp index 64a6bbc..a975ba4 100644 --- a/example/app/google/DiscoveryDemo.cpp +++ b/example/app/google/DiscoveryDemo.cpp @@ -1,13 +1,287 @@ #include "../GoogleDemos.hpp" #include "../../fw/Router.hpp" +#include "../../fw/Templates.hpp" +#include "google/discovery/caps.h" +#include "google/discovery/engine.h" namespace molo::example::app { -// every Google API generically (all 316 Discovery documents). Stub: the package agent fills this in. +// Every Google API generically: the Discovery engine over src/google/discovery. +// GET routes need no credential. Every route that reaches Google (documents +// and the API itself) is a POST: 422 on bad input first, then 503 +// google_not_configured without a credential. A call runs at once when the +// method is a GET; any other verb is a dry run (the validated request) unless +// the body says "confirm":true. + +namespace { + +namespace dc = google::discovery; +using Json = clients::body::Json; + +constexpr std::size_t kMaxBody = 512 * 1024; +constexpr std::size_t kMaxText = 200; + +const char kPage[] = R"DISCOVERY({% extends "base.html" %} +{% block content %} +

Google Discovery

+

Find any Google API (the whole Discovery directory) and call any +method by id. Requests are validated against the real Discovery document before +any traffic. Only GET methods run at once; other verbs are dry runs until you +tick confirm.

+

Checking Google credentials...

+
+ +
+ + +
+ + + +
+ + + + + + + +
+ +

+
+ + +{% endblock %} +)DISCOVERY"; + +fw::Response json_out(const Json& j, int status = 200) { + fw::Response res = fw::JsonResponse(j.dump(), status); + res.headers.set("Cache-Control", "no-store"); + return res; +} + +[[noreturn]] void bad(const std::string& why) { + throw fw::HttpError(422, "validation_error", why); +} + +[[noreturn]] void rethrow(const google::GoogleError& e) { + if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED") + throw fw::HttpError(503, "google_not_configured", e.message()); + if (e.service().empty() && e.status() == "INVALID_ARGUMENT") + throw fw::HttpError(422, "validation_error", e.message()); + if (e.service() == "discovery" && e.status() == "NOT_FOUND") + throw fw::HttpError(404, "not_found", e.message()); + throw fw::HttpError(502, "google_error", e.status() + ": " + e.message()); +} + +template +auto guarded(F&& f) -> decltype(f()) { + try { + return f(); + } catch (const google::GoogleError& e) { + rethrow(e); + } +} + +Json parse_body(const fw::Request& req) { + if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large"); + if (req.body.empty()) return Json::object(); + try { + Json j = Json::parse(req.body); + if (!j.is_object()) bad("body must be a JSON object"); + return j; + } catch (const clients::body::JsonError&) { + bad("body is not valid JSON"); + } +} + +std::string text(const Json& body, const char* key, bool required = false) { + const Json* v = body.find(key); + if (!v || v->is_null()) { + if (required) bad(std::string(key) + " is required"); + return ""; + } + if (!v->is_string()) bad(std::string(key) + " must be a string"); + if (v->as_string().size() > kMaxText) bad(std::string(key) + " is too long"); + return v->as_string(); +} + +bool flag(const Json& body, const char* key) { + const Json* v = body.find(key); + if (!v || v->is_null()) return false; + if (!v->is_bool()) bad(std::string(key) + " must be true or false"); + return v->as_bool(); +} + +std::size_t bounded(const Json& body, const char* key, std::size_t lo, std::size_t hi, + std::size_t fallback) { + const Json* v = body.find(key); + if (!v || v->is_null()) return fallback; + if (!v->is_number()) bad(std::string(key) + " must be a number"); + long n = v->as_int(); + if (n < static_cast(lo) || n > static_cast(hi)) + bad(std::string(key) + " must be " + std::to_string(lo) + ".." + std::to_string(hi)); + return static_cast(n); +} + +// "name" or "name:version", letters digits _ - . and one colon. +std::string api_ref(const Json& body) { + std::string api = text(body, "api", true); + std::size_t colons = 0; + for (char c : api) { + if (c == ':') ++colons; + else if (!(std::isalnum(static_cast(c)) || c == '_' || c == '-' || c == '.')) + bad("api must be name or name:version"); + } + if (colons > 1) bad("api must be name or name:version"); + return api; +} + +std::string method_id(const Json& body) { + std::string id = text(body, "method", true); + for (char c : id) + if (!(std::isalnum(static_cast(c)) || c == '_' || c == '-' || c == '.')) + bad("method must be a Discovery method id such as serviceusage.services.list"); + return id; +} + +dc::CallArgs args_from(const Json& body) { + dc::CallArgs a; + if (const Json* p = body.find("params"); p && !p->is_null()) { + if (!p->is_object()) bad("params must be a JSON object"); + a.params = *p; + } + if (const Json* b = body.find("body"); b && !b->is_null()) { + if (!b->is_object()) bad("body must be a JSON object"); + a.body = *b; + } + a.location = text(body, "location"); + try { + a.auth = dc::auth_mode_from(text(body, "auth")); + } catch (const google::GoogleError& e) { + bad(e.message()); + } + a.max_pages = bounded(body, "maxPages", 1, dc::DiscoveryCaps::kMaxPages, 1); + a.items_key = text(body, "itemsKey"); + a.wait = flag(body, "wait"); + return a; +} + +} // namespace + void register_google_discovery_demo(fw::Router& router, Services& services) { - (void)router; - (void)services; + if (!services.google) services.google = google::GoogleClient::from_env(); + auto client = services.google; + auto catalog = std::make_shared(std::make_shared()); + auto engine = std::make_shared(client, catalog); + fw::Environment* env = services.templates.get(); + services.templates->add_template("google_discovery.html", kPage); + services.add_feature("google-discovery", "/google/discovery"); + + // Shape checks come first (422); then the credential (503). + auto need_credential = [client]() { + if (!client->configured()) throw google::NotConfiguredError("discovery"); + }; + + router.get("/google/discovery", "google-discovery-page", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("Google Discovery"); + return fw::Response(200, env->render("google_discovery.html", ctx), "text/html; charset=utf-8"); + }); + + router.get("/google/discovery/status", "google-discovery-status", [client, catalog](fw::Request req) { + (void)req; + dc::CatalogStats s = catalog->stats(); + Json caps = Json::object(); + caps.set("max_pages", Json(static_cast(dc::DiscoveryCaps::kMaxPages))); + caps.set("max_items", Json(static_cast(dc::DiscoveryCaps::kMaxItems))); + caps.set("max_doc_bytes", Json(static_cast(dc::DiscoveryCaps::kMaxDocBytes))); + caps.set("max_body_bytes", Json(static_cast(dc::DiscoveryCaps::kMaxBodyBytes))); + caps.set("max_cached_apis", Json(static_cast(dc::DiscoveryCaps::kMaxCachedApis))); + caps.set("max_age_seconds", Json(static_cast(dc::DiscoveryCaps::kMaxAge.count()))); + caps.set("min_refetch_seconds", Json(static_cast(dc::DiscoveryCaps::kMinRefetch.count()))); + Json o = Json::object(); + o.set("configured", Json(client->configured())); + o.set("directory", Json(std::string(dc::kDirectoryUrl))); + o.set("cached_apis", Json(static_cast(s.cached))); + o.set("cached_bytes", Json(static_cast(s.cached_bytes))); + o.set("fetches", Json(static_cast(s.fetches))); + o.set("caps", caps); + return json_out(o); + }); + + router.post("/google/discovery/apis", "google-discovery-apis", [engine, need_credential](fw::Request req) { + Json b = parse_body(req); + std::string filter = text(b, "filter"); + std::size_t limit = bounded(b, "limit", 1, dc::DiscoveryCaps::kMaxListed, 50); + bool all = flag(b, "all"); + return guarded([&] { + need_credential(); + Json o = Json::object(); + o.set("apis", engine->list_apis(filter, limit, !all)); + return json_out(o); + }); + }); + + router.post("/google/discovery/methods", "google-discovery-methods", [engine, need_credential](fw::Request req) { + Json b = parse_body(req); + std::string api = api_ref(b); + std::string filter = text(b, "filter"); + std::size_t limit = bounded(b, "limit", 1, dc::DiscoveryCaps::kMaxListed, 100); + return guarded([&] { + need_credential(); + return json_out(engine->list_methods(api, filter, limit)); + }); + }); + + router.post("/google/discovery/describe", "google-discovery-describe", [engine, need_credential](fw::Request req) { + Json b = parse_body(req); + std::string api = api_ref(b); + std::string method = method_id(b); + return guarded([&] { + need_credential(); + return json_out(engine->describe(api, method)); + }); + }); + + router.post("/google/discovery/plan", "google-discovery-plan", [engine, need_credential](fw::Request req) { + Json b = parse_body(req); + std::string api = api_ref(b); + std::string method = method_id(b); + dc::CallArgs args = args_from(b); + return guarded([&] { + need_credential(); + return json_out(engine->plan(api, method, args).to_json()); + }); + }); + + router.post("/google/discovery/call", "google-discovery-call", [engine, need_credential](fw::Request req) { + Json b = parse_body(req); + std::string api = api_ref(b); + std::string method = method_id(b); + dc::CallArgs args = args_from(b); + bool confirm = flag(b, "confirm"); + return guarded([&] { + need_credential(); + dc::PreparedCall plan = engine->plan(api, method, args); // validates, no API traffic + if (plan.http != clients::Method::kGet && !confirm) { + Json o = plan.to_json(); + o.set("dry_run", Json(true)); + o.set("hint", Json("send confirm:true to run it")); + return json_out(o); + } + return json_out(engine->call(api, method, args).to_json()); + }); + }); } } // namespace molo::example::app diff --git a/example/assets/google_discovery.js b/example/assets/google_discovery.js new file mode 100644 index 0000000..97d46bd --- /dev/null +++ b/example/assets/google_discovery.js @@ -0,0 +1,68 @@ +// Google Discovery page: thin client over /google/discovery/* (all limits server side). +(function () { + "use strict"; + var $ = function (id) { return document.getElementById(id); }; + var out = $("dc-out"), err = $("dc-error"); + + function post(path, body) { + err.textContent = ""; + out.textContent = "..."; + return fetch(path, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body) + }).then(function (res) { + return res.text().then(function (t) { + var j; + try { j = JSON.parse(t); } catch (e) { j = { message: t }; } + if (!res.ok) { + err.textContent = j.message || ("HTTP " + res.status); + out.textContent = ""; + } else { + out.textContent = JSON.stringify(j, null, 2); + } + }); + }).catch(function (e) { err.textContent = String(e); out.textContent = ""; }); + } + + function json(id) { + var raw = $(id).value.trim(); + if (!raw) return undefined; + try { return JSON.parse(raw); } catch (e) { err.textContent = id + " is not valid JSON"; return null; } + } + + function callBody() { + var params = json("dc-params"), body = json("dc-body"); + if (params === null || body === null) return null; + var b = { + api: $("dc-api").value.trim(), + method: $("dc-method").value.trim(), + location: $("dc-location").value.trim(), + auth: $("dc-auth").value, + maxPages: parseInt($("dc-pages").value, 10) || 1, + wait: $("dc-wait").checked, + confirm: $("dc-confirm").checked + }; + if (params !== undefined) b.params = params; + if (body !== undefined) b.body = body; + return b; + } + + fetch("/google/discovery/status").then(function (r) { return r.json(); }).then(function (s) { + $("dc-config").textContent = s.configured + ? "Google credential found. Documents cached: " + s.cached_apis + "." + : "No Google credential: every action answers 503 until one is configured."; + }); + + $("dc-apis").onclick = function () { + post("/google/discovery/apis", { filter: $("dc-filter").value.trim(), limit: 50 }); + }; + $("dc-methods").onclick = function () { + post("/google/discovery/methods", { api: $("dc-api").value.trim(), filter: $("dc-method").value.trim(), limit: 100 }); + }; + $("dc-describe").onclick = function () { + post("/google/discovery/describe", { api: $("dc-api").value.trim(), method: $("dc-method").value.trim() }); + }; + $("dc-plan").onclick = function () { var b = callBody(); if (b) post("/google/discovery/plan", b); }; + $("dc-call").onclick = function () { var b = callBody(); if (b) post("/google/discovery/call", b); }; +})(); diff --git a/src/google/discovery/SPEC.md b/src/google/discovery/SPEC.md new file mode 100644 index 0000000..fa27da7 --- /dev/null +++ b/src/google/discovery/SPEC.md @@ -0,0 +1,213 @@ +# Google Discovery Engine - Specification + +Owned directory: `src/google/discovery/`. Namespace `molo::google::discovery`. +Depends on the `src/google` foundation (`GoogleClient`, `GoogleError`, +`ServiceEndpoint`, `Operation`) and `clients::Client` for document fetches. +No other google package depends on it and it depends on none. Plan: +`plans/18-breadth.md` section 1. + +## 1. Scope + +One generic engine that reaches every API in Google's Discovery directory +(`https://discovery.googleapis.com/discovery/v1/apis`: 538 entries, 316 +preferred, 2026-10-25) and calls any method by its Discovery id +(`serviceusage.services.list`). It fetches and caches a Discovery document, +lists APIs and methods, validates parameters and body against the document +before any traffic, builds the URL (path templates, query, repeated values, +regional host), chooses key or OAuth, walks `nextPageToken`, and polls +long-running operations. Typed classes in other packages are better for the +APIs they cover; this engine is the breadth. + +Not here (each is classified Limited or Rejected, see section 5): media +upload and download bodies, resumable upload, bidirectional or client +streaming, non-JSON replies, the global batch endpoint (`batchPath`, shut +down by Google in 2020), and object-typed query parameters. + +## 2. Module map (one concern per file) + +| File | Concern | +|---|---| +| `caps.h` | Static ceilings (`DiscoveryCaps`) | +| `model.h/.cpp` | `Api`, `Method`, `Param`: parse a document, classify every method (`Support` Full / Limited / Rejected with notes), pagination and operation detection, host allowlist | +| `validate.h/.cpp` | `ParamValidator` (type, format, enum, pattern, range, repeated, path safety) and `BodyValidator` (schema walk: `$ref`, objects, maps, arrays, enums, int64 strings, free-form) | +| `planner.h/.cpp` | `Planner::plan`: `(Api, Method, CallArgs)` to `PreparedCall` (endpoint, encoded path and query, body, credential choice). No traffic | +| `catalog.h/.cpp` | `Directory`, `DocumentSource` (`HttpDocumentSource`, `FileDocumentSource`), `Catalog` (bounded LRU cache with age and refetch floor) | +| `engine.h/.cpp` | `Engine`: `list_apis`, `list_methods`, `describe`, `plan`, `call` (paging, long-running wait) | +| `tools/check_discovery_directory.py` | Fetches the whole directory and every preferred document, runs the engine's own classification over them, reports what it cannot handle | +| `tools/audit_main.cc` | The C++ half of that checker (built by the script; not globbed by `google.mk`) | + +## 3. Static caps (see caps.h) + +| Knob | Value | +|---|---| +| Document size | 8 MiB (largest real: compute v1, 6.4 MB) | +| Directory size / items | 2 MiB / 1000 | +| Cached documents / cached bytes | 32 / 96 MiB, least recently used evicted | +| Document max age | 24 h | +| Refetch floor | 60 s per document, even after `forget` | +| Methods per document | 10000 (largest real: compute v1, 1018; aiplatform v1, 1137) | +| Resource nesting | 16 | +| Parameter value | 2048 chars, no control characters | +| Repeated values / query values | 100 / 64 | +| Request target | 6000 chars | +| Request body | 4 MiB, schema depth 32, 200000 visited nodes | +| Pages per call / items per call | 50 / 10000 (default 1 page) | +| Listing | 500 rows | +| Operation wait | foundation `OperationCaps`: 400 polls, 2 s interval, 600 s | + +## 4. Contracts + +- Documents are fetched without a credential, over HTTPS, from hosts that + are `googleapis.com` or a subdomain only (`is_google_host_url`). A document + whose `rootUrl` or regional `endpointUrl` fails that test is rejected + (`BAD_DOCUMENT`), as is a document that is not the one the directory + listed, a wrong `kind` or `protocol`, a duplicate method id, or anything + over a cap. +- API reference: `name` (preferred version) or `name:version`. The method id + prefix is not always the API name (`admin` serves `reports.activities.list`), + so the API is always given explicitly. +- Validation order, all before traffic: API and method exist (404), method + not Rejected, `params` is an object, regional `location` exists in the + document, every param is declared (global params: only `fields`, + `quotaUser`, `prettyPrint`; `key`, `access_token`, `oauth_token`, `alt`, + `callback`, `uploadType`, `upload_protocol`, `$.xgafv` are refused by name), + values fit type / format / enum / pattern / range, required params are + present, path values have no dot segment, the body fits the request schema. + Failures are `GoogleError` 400 `INVALID_ARGUMENT` naming the parameter or + JSON path. After validation, no credential is `NotConfiguredError` (503). +- Path templates are `{x}` (value fully percent-encoded, `/` included) or + `{+x}` (reserved expansion: each `/` segment encoded, slashes kept). + Query parameters are sorted by name, repeated parameters repeat the key. +- Body: methods with no request schema take none; POST / PUT / PATCH without + a request schema or without a body send `{}`. Unknown properties are + rejected when the schema lists properties; free-form objects and `any` + pass; `null` passes (proto JSON unset); int64 / uint64 are digit strings + (an integral JSON number is accepted and converted). +- Credential: `AuthMode::kAuto` sends the API key for methods that declare no + OAuth scope (205 of 13194) and OAuth for the rest; `kKey` / `kOAuth` force + it through `ServiceEndpoint::prefer_key`. `GoogleClient` falls back to the + other credential only when the forced one is not configured. +- Regional hosts: `location` selects the `endpoints[]` row of the document + (69 documents have rows, e.g. `workflows.us-central1.rep.googleapis.com`). + An unknown location is rejected with examples; a document without rows + rejects any location. Nothing is guessed from the location string. +- Pagination: `max_pages > 1` needs a `pageToken` query param and a reply + schema with `nextPageToken`. The array to collect is the single array + property of the reply (`items_key` otherwise). The merged reply has the + collected array; `truncated` is true when a page or item cap stopped the + walk while a token remained (the token stays in the body). +- Long-running: a reply whose schema has `name` and boolean `done` is an + operation. With `wait`, the engine finds the API's `operations.get` method + whose `name` pattern matches, polls it with the foundation caps, throws + `OPERATION_FAILED` on an error result and `DEADLINE_EXCEEDED` at the cap. + APIs without an `operations.get` (111 methods) reject `wait` loudly. + Operations that use another shape (Compute `status` / `selfLink`) are + plain resources to this engine. +- Destructive or sending methods are ordinary methods here; the caller (the + demo) decides. The demo runs GET at once and dry-runs every other verb + unless `confirm:true`. + +## 5. Shapes seen + +Read from the 310 reachable preferred documents (13194 methods, 27285 +schemas), 2026-10-25, and how each is handled: + +| Shape | Seen | Handling | +|---|---|---| +| `path` templates `{x}` / `{+x}` only; every variable declared as a path param | 4712 / 10563 variables, no exception | Substituted and encoded; anything else Rejected | +| `flatPath` | 13034 methods | Ignored: `path` is authoritative and equivalent | +| `rootUrl` + `servicePath`; 12 documents have a non-empty `servicePath` | 310 | `base_url` = root (or regional root) + servicePath; `path` is relative to it | +| Param `location` | `path`, `query` only | Anything else Rejected | +| Param types | string, integer, boolean, number; `object` once (`discoveryengine ...dataConnector.mcp` `extensions`) | Object params Rejected (1 method) | +| Param `format` | int32, uint32, int64, uint64, double, float, byte, date-time, google-datetime, google-duration, google-fieldmask | Checked for integers and the 64-bit strings, date and duration shape; others are strings | +| `repeated` | 509 params | Array or scalar; cap 100 | +| `enum` | 710 params | Checked | +| `pattern` | 12636 params | `std::regex` ECMAScript; a pattern it cannot parse is left to Google | +| `minimum` / `maximum` (strings) | 306 / 61 | Checked | +| `$ref` request/response, nested `$ref`, `items`, `additionalProperties` (map types), `any`, readOnly | all | Body validator follows `$ref` (chain cap 16), maps, arrays; readOnly is not enforced | +| `mediaUpload` (simple, resumable) | 53 methods | Limited: the metadata call is made, no media bytes | +| `supportsMediaDownload` | 20 methods | Limited: JSON reply only, `alt=media` is refused | +| Reply `HttpBody` | 81 methods | Limited: the reply may not be JSON; a parse failure is `BAD_RESPONSE` | +| `streamingType` | `NONE` 4, `SERVER_SIDE` 2 (admob) | NONE ignored; SERVER_SIDE Limited (read whole); anything else Rejected | +| `batchPath` | 310 documents | Not used (Google removed the global batch endpoint) | +| `endpoints[]` | 69 documents | Regional hosts, see section 4 | +| `mtlsRootUrl`, `canonicalName`, `version_module`, `icons`, `auth` | many | Ignored | +| Operation-shaped replies (`name` + `done`) | 2359 methods | `wait`; 111 are in documents with no `operations.get` | +| `pageToken` + `nextPageToken` | 2386 methods | Paging | +| Other paging schemes (`start-index`, `nextLink`) | not counted | Not paged; one call per request | + +## Required APIs + +Enable these on the project (placeholder project `example-214512124`; full +list in `src/google/APIS.md`). The engine itself needs none: Discovery +documents are public. Calling an API needs that API enabled; a disabled one +answers 403 `SERVICE_DISABLED`. The live checks use: + +- `serviceusage.googleapis.com`: https://console.developers.google.com/apis/api/serviceusage.googleapis.com/overview?project=example-214512124 + +Scopes: `cloud-platform` covers the Cloud APIs. Workspace and other APIs +need the scopes named in each method's `scopes` (`describe` shows them); the +service-account credential in this repository supplies `cloud-platform` only. + +## Sources + +- `https://discovery.googleapis.com/discovery/v1/apis` (directory, 538 + entries, 316 preferred) and every preferred `discoveryRestUrl`, fetched + 2026-10-25. +- Discovery format reference: `https://developers.google.com/discovery/v1/reference/apis` + and `.../v1/using` (parameter and method fields). + +## Verified against discovery + +`python3 src/google/discovery/tools/check_discovery_directory.py`, +2026-10-25: 316 preferred APIs listed, 310 documents served and loaded by the +engine, 0 rejected documents. Six listed APIs are not served by Google +(`area120tables:v1alpha1`, `datalabeling:v1beta1`, `mybusinessqanda:v1` 404; +`contentwarehouse:v1`, `poly:v1`, `rapidmigrationassessment:v1` 502): not an +engine fault, listed by the script as UNREACHABLE. Of 13194 methods: 13037 +Full, 156 Limited (81 HttpBody replies, 53 media upload, 20 media download, +2 server streaming), 1 Rejected (object query parameter). Dry plan of every +callable method with synthesized required parameters: 13185 planned, 8 +skipped (synthetic value does not fit an exotic pattern), 0 failed. + +## Verified live + +`make google-live` as the service account on project `lisa-448004`, +2026-10-25 (read-only): + +- Real directory over HTTPS: 538 items, 316 preferred; documents of + serviceusage, cloudresourcemanager, tasks, workflows parsed. +- Largest documents fit the caps: compute v1 (1018 methods, 6419010 bytes), + aiplatform v1 (1137 methods, 4027902 bytes). +- `serviceusage.services.get` for `serviceusage.googleapis.com` answered 200. +- `serviceusage.services.list` with `pageSize=3`, 3 pages: 9 items, + `truncated` true (the page cap, the token left in the body). +- `serviceusage.operations.get` with a missing name: 400 INVALID_ARGUMENT + (Google validates the name shape); the reply is recognised as an operation. +- Cloud Resource Manager answers `PERMISSION_DENIED` (API not enabled on the + project): reported by Google, not exercised. + +## Unverified + +- Long-running `wait` polling to completion: no safe read-only operation + exists on this project, so only detection and the `operations.get` lookup + are exercised live. The loop is the foundation's `wait_operation` logic + re-implemented over the planner. +- Key credential on a no-scope method live: no API key with a keyless-capable + API was used; the choice is verified at the request level (header). +- Regional endpoints are verified for URL construction only, not by a call. +- Methods Limited or Rejected are never called by design. +- Behaviour for the 8 methods whose required-parameter patterns the audit + could not synthesize is only the validation path. + +## Gaps + +- Media upload (simple, resumable), media download and non-JSON replies: + `Method::support` says Limited; the foundation `GoogleClient` is JSON only + (`src/google/client.h`). +- Batch (`batchPath`) is not offered. +- Paging schemes other than `pageToken` / `nextPageToken`. +- `AuthMode::kOAuth` falls back to the key when no OAuth credential exists + (foundation `GoogleClient::pick_auth`). +- Documents are cached in memory only; there is no on-disk cache. +- Test and audit coverage of `wait` against a real running operation. diff --git a/src/google/discovery/caps.h b/src/google/discovery/caps.h new file mode 100644 index 0000000..d59e8d6 --- /dev/null +++ b/src/google/discovery/caps.h @@ -0,0 +1,44 @@ +#pragma once + +// Static ceilings for the Discovery engine. Never autotuned. Measured +// against the 310 reachable preferred documents on 2026-10-25: largest +// document 6.4 MB (compute v1), most methods in one document is below +// kMaxMethods, directory 538 entries. + +#include +#include + +namespace molo::google::discovery { + +struct DiscoveryCaps { + // Documents and cache. + static constexpr std::size_t kMaxDocBytes = 8u * 1024u * 1024u; + static constexpr std::size_t kMaxDirectoryBytes = 2u * 1024u * 1024u; + static constexpr std::size_t kMaxDirectoryItems = 1000; + static constexpr std::size_t kMaxCachedApis = 32; + static constexpr std::size_t kMaxCachedBytes = 96u * 1024u * 1024u; + static constexpr std::chrono::seconds kMaxAge{24 * 3600}; + // Floor: a document is never refetched more often than this, even when + // the caller forces a refresh. + static constexpr std::chrono::seconds kMinRefetch{60}; + // Document structure. + static constexpr std::size_t kMaxMethods = 10000; + static constexpr std::size_t kMaxResourceDepth = 16; + static constexpr std::size_t kMaxSchemas = 20000; + // Listing. + static constexpr std::size_t kMaxListed = 500; + // Request validation. + static constexpr std::size_t kMaxParamChars = 2048; + static constexpr std::size_t kMaxRepeated = 100; + static constexpr std::size_t kMaxQueryParams = 64; + static constexpr std::size_t kMaxPathChars = 6000; + static constexpr std::size_t kMaxBodyBytes = 4u * 1024u * 1024u; + static constexpr std::size_t kMaxSchemaDepth = 32; + static constexpr std::size_t kMaxSchemaVisits = 200000; + // Pagination. + static constexpr std::size_t kDefaultPages = 1; + static constexpr std::size_t kMaxPages = 50; + static constexpr std::size_t kMaxItems = 10000; +}; + +} // namespace molo::google::discovery diff --git a/src/google/discovery/catalog.cpp b/src/google/discovery/catalog.cpp new file mode 100644 index 0000000..855ff9b --- /dev/null +++ b/src/google/discovery/catalog.cpp @@ -0,0 +1,285 @@ +#include "google/discovery/catalog.h" + +#include +#include +#include +#include + +#include "clients/transports/http_transport.h" +#include "google/discovery/caps.h" +#include "google/errors.h" + +namespace molo::google::discovery { + +using clients::body::Json; + +const char* const kDirectoryUrl = "https://discovery.googleapis.com/discovery/v1/apis"; + +namespace { + +std::string str(const Json& o, const char* key) { + const Json* v = o.find(key); + return v && v->is_string() ? v->as_string() : ""; +} + +std::string lower(std::string s) { + for (char& c : s) c = static_cast(std::tolower(static_cast(c))); + return s; +} + +bool valid_name(const std::string& s) { + if (s.empty() || s.size() > 64) return false; + for (char c : s) + if (!(std::isalnum(static_cast(c)) || c == '_' || c == '-' || c == '.')) return false; + return true; +} + +} // namespace + +Json DirectoryItem::to_json() const { + Json o = Json::object(); + o.set("id", Json(id)); + o.set("name", Json(name)); + o.set("version", Json(version)); + o.set("title", Json(title)); + o.set("description", Json(description)); + o.set("discoveryRestUrl", Json(rest_url)); + o.set("documentationLink", Json(documentation)); + o.set("preferred", Json(preferred)); + return o; +} + +Directory Directory::parse(const std::string& text) { + if (text.size() > DiscoveryCaps::kMaxDirectoryBytes) + throw GoogleError(502, "BAD_DOCUMENT", "discovery directory is larger than the cap"); + Json doc; + try { + doc = Json::parse(text); + } catch (const std::exception& e) { + throw GoogleError(502, "BAD_DOCUMENT", std::string("discovery directory: ") + e.what()); + } + const Json* items = doc.is_object() ? doc.find("items") : nullptr; + if (!items || !items->is_array() || str(doc, "kind") != "discovery#directoryList") + throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: not a directoryList"); + if (items->as_array().size() > DiscoveryCaps::kMaxDirectoryItems) + throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: more items than the cap"); + Directory dir; + for (const Json& it : items->as_array()) { + DirectoryItem row; + row.id = str(it, "id"); + row.name = str(it, "name"); + row.version = str(it, "version"); + row.title = str(it, "title"); + row.description = str(it, "description"); + row.rest_url = str(it, "discoveryRestUrl"); + row.documentation = str(it, "documentationLink"); + const Json* p = it.find("preferred"); + row.preferred = p && p->is_bool() && p->as_bool(); + if (row.id.empty() || row.name.empty() || row.version.empty() || + !is_google_host_url(row.rest_url)) + throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: bad item " + row.id); + dir.items_.push_back(std::move(row)); + } + return dir; +} + +std::size_t Directory::preferred_count() const { + return static_cast(std::count_if(items_.begin(), items_.end(), + [](const DirectoryItem& i) { return i.preferred; })); +} + +const DirectoryItem* Directory::find(const std::string& name, const std::string& version) const { + const DirectoryItem* fallback = nullptr; + for (const DirectoryItem& i : items_) { + if (i.name != name) continue; + if (!version.empty()) { + if (i.version == version) return &i; + } else if (i.preferred) { + return &i; + } else if (!fallback) { + fallback = &i; + } + } + return version.empty() ? fallback : nullptr; +} + +std::vector Directory::search(const std::string& needle, std::size_t limit, + bool preferred_only) const { + std::vector out; + std::string n = lower(needle); + limit = std::min(limit, DiscoveryCaps::kMaxListed); + for (const DirectoryItem& i : items_) { + if (out.size() >= limit) break; + if (preferred_only && !i.preferred) continue; + if (n.empty() || lower(i.name).find(n) != std::string::npos || + lower(i.title).find(n) != std::string::npos) + out.push_back(&i); + } + return out; +} + +namespace { +clients::ClientConfig source_config() { + clients::ClientConfig c; + c.auth = nullptr; + c.persist_cookies = false; + return c; +} +} // namespace + +HttpDocumentSource::HttpDocumentSource() + : client_(std::make_unique(), source_config()) {} + +HttpDocumentSource::HttpDocumentSource(std::unique_ptr transport) + : client_(std::move(transport), source_config()) {} + +std::string HttpDocumentSource::fetch(const std::string& url, std::size_t max_bytes) { + if (!is_google_host_url(url)) + throw GoogleError::invalid("discovery url is not an https googleapis.com url: " + url); + try { + clients::ClientResponse res = client_.get_blocking(url); + if (!res.is_success()) + throw GoogleError(res.status(), res.status() == 404 ? "NOT_FOUND" : "UNAVAILABLE", + "discovery fetch failed with HTTP " + std::to_string(res.status()), "discovery"); + if (res.body().size() > max_bytes) + throw GoogleError(502, "BAD_DOCUMENT", "discovery document is larger than the cap", "discovery"); + return res.body(); + } catch (const GoogleError&) { + throw; + } catch (const std::exception& e) { + throw GoogleError(502, "UNAVAILABLE", e.what(), "discovery"); + } +} + +std::string FileDocumentSource::fetch(const std::string& url, std::size_t max_bytes) { + std::string file; + if (url.rfind(kDirectoryUrl, 0) == 0) { + file = "directory.json"; + } else { + if (index_.empty()) { + std::ifstream din(dir_ + "/directory.json", std::ios::binary); + if (!din) throw GoogleError(404, "NOT_FOUND", "saved directory.json missing", "discovery"); + std::ostringstream dss; + dss << din.rdbuf(); + for (const DirectoryItem& i : Directory::parse(dss.str()).items()) + index_[i.rest_url] = i.name + "_" + i.version + ".json"; + } + auto hit = index_.find(url); + if (hit == index_.end()) throw GoogleError::invalid("no saved document for url: " + url); + file = hit->second; + } + std::ifstream in(dir_ + "/" + file, std::ios::binary); + if (!in) throw GoogleError(404, "NOT_FOUND", "saved discovery document missing: " + file, "discovery"); + std::ostringstream ss; + ss << in.rdbuf(); + std::string text = ss.str(); + if (text.size() > max_bytes) + throw GoogleError(502, "BAD_DOCUMENT", "discovery document is larger than the cap", "discovery"); + return text; +} + +Catalog::Catalog(std::shared_ptr source, Clock clock) + : source_(std::move(source)), clock_(std::move(clock)) {} + +std::chrono::steady_clock::time_point Catalog::now() const { + return clock_ ? clock_() : std::chrono::steady_clock::now(); +} + +std::shared_ptr Catalog::directory() { + std::lock_guard lock(mu_); + auto t = now(); + if (directory_ && t - directory_at_ < DiscoveryCaps::kMaxAge) { + ++stats_.hits; + return directory_; + } + ++stats_.fetches; + std::string text = source_->fetch(kDirectoryUrl, DiscoveryCaps::kMaxDirectoryBytes); + directory_ = std::make_shared(Directory::parse(text)); + directory_at_ = t; + return directory_; +} + +std::shared_ptr Catalog::api(const std::string& ref) { + std::string name = ref, version; + if (std::size_t c = ref.find(':'); c != std::string::npos) { + name = ref.substr(0, c); + version = ref.substr(c + 1); + } + if (!valid_name(name) || (!version.empty() && !valid_name(version))) + throw GoogleError::invalid("api must be name or name:version, got '" + ref + "'"); + std::shared_ptr dir = directory(); + const DirectoryItem* item = dir->find(name, version); + if (!item) throw GoogleError(404, "NOT_FOUND", "no such API in the Discovery directory: " + ref, "discovery"); + std::lock_guard lock(mu_); + auto t = now(); + auto it = entries_.find(item->id); + if (it != entries_.end()) { + bool stale = t - it->second.fetched >= DiscoveryCaps::kMaxAge; + auto lf = last_fetch_.find(item->id); + bool floor = lf != last_fetch_.end() && t - lf->second < DiscoveryCaps::kMinRefetch; + if (!stale || floor) { + lru_.remove(item->id); + lru_.push_front(item->id); + ++stats_.hits; + return it->second.api; + } + } + ++stats_.fetches; + std::string text = source_->fetch(item->rest_url, DiscoveryCaps::kMaxDocBytes); + Json doc; + try { + doc = Json::parse(text); + } catch (const std::exception& e) { + throw GoogleError(502, "BAD_DOCUMENT", std::string("discovery document ") + item->id + ": " + e.what()); + } + Api parsed = Api::parse(doc); + parsed.raw_bytes = text.size(); + if (parsed.name() != item->name || parsed.version() != item->version) + throw GoogleError(502, "BAD_DOCUMENT", "document " + parsed.id() + " is not the one requested: " + item->id); + Entry e; + e.api = std::make_shared(std::move(parsed)); + e.fetched = t; + e.bytes = text.size(); + entries_[item->id] = e; + last_fetch_[item->id] = t; + lru_.remove(item->id); + lru_.push_front(item->id); + evict_locked(); + return e.api; +} + +void Catalog::evict_locked() { + auto total = [&] { + std::size_t b = 0; + for (const auto& [k, v] : entries_) b += v.bytes; + return b; + }; + while (entries_.size() > 1 && + (entries_.size() > DiscoveryCaps::kMaxCachedApis || total() > DiscoveryCaps::kMaxCachedBytes)) { + entries_.erase(lru_.back()); + lru_.pop_back(); + ++stats_.evictions; + } +} + +void Catalog::forget(const std::string& ref) { + std::lock_guard lock(mu_); + std::string name = ref, version; + if (std::size_t c = ref.find(':'); c != std::string::npos) { + name = ref.substr(0, c); + version = ref.substr(c + 1); + } + for (auto& [id, e] : entries_) + if (e.api->name() == name && (version.empty() || e.api->version() == version)) + e.fetched = std::chrono::steady_clock::time_point{}; +} + +CatalogStats Catalog::stats() const { + std::lock_guard lock(mu_); + CatalogStats s = stats_; + s.cached = entries_.size(); + for (const auto& [k, v] : entries_) s.cached_bytes += v.bytes; + return s; +} + +} // namespace molo::google::discovery diff --git a/src/google/discovery/catalog.h b/src/google/discovery/catalog.h new file mode 100644 index 0000000..5c3e23d --- /dev/null +++ b/src/google/discovery/catalog.h @@ -0,0 +1,125 @@ +#pragma once + +// Directory of Google APIs plus a bounded cache of parsed Discovery +// documents. Documents come from a DocumentSource: HTTPS in production +// (no credential, googleapis.com hosts only), a directory of saved real +// documents for offline audits. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clients/client.h" +#include "google/discovery/model.h" + +namespace molo::google::discovery { + +struct DirectoryItem { + std::string id; // bigquery:v2 + std::string name; + std::string version; + std::string title; + std::string description; + std::string rest_url; // discoveryRestUrl + std::string documentation; + bool preferred = false; + + clients::body::Json to_json() const; +}; + +class Directory { + public: + // Parses the body of https://discovery.googleapis.com/discovery/v1/apis. + static Directory parse(const std::string& text); + const std::vector& items() const { return items_; } + std::size_t preferred_count() const; + // Version empty: the preferred version. nullptr when absent. + const DirectoryItem* find(const std::string& name, const std::string& version = "") const; + // Case-insensitive substring over name and title, at most `limit` rows. + std::vector search(const std::string& needle, std::size_t limit, + bool preferred_only = true) const; + + private: + std::vector items_; +}; + +class DocumentSource { + public: + virtual ~DocumentSource() = default; + // Returns the body. Throws GoogleError on failure; never more than max_bytes. + virtual std::string fetch(const std::string& url, std::size_t max_bytes) = 0; +}; + +// HTTPS GET without credentials; refuses any host outside googleapis.com. +class HttpDocumentSource : public DocumentSource { + public: + HttpDocumentSource(); + explicit HttpDocumentSource(std::unique_ptr transport); + std::string fetch(const std::string& url, std::size_t max_bytes) override; + + private: + clients::Client client_; +}; + +// Serves saved real documents: "/_.json" for a document +// URL, "/directory.json" for the directory. +class FileDocumentSource : public DocumentSource { + public: + explicit FileDocumentSource(std::string dir) : dir_(std::move(dir)) {} + std::string fetch(const std::string& url, std::size_t max_bytes) override; + + private: + std::string dir_; + std::map index_; // rest url -> file +}; + +extern const char* const kDirectoryUrl; + +struct CatalogStats { + std::size_t cached = 0; + std::size_t cached_bytes = 0; + std::size_t fetches = 0; + std::size_t hits = 0; + std::size_t evictions = 0; +}; + +class Catalog { + public: + using Clock = std::function; + explicit Catalog(std::shared_ptr source, Clock clock = nullptr); + + // Cached up to DiscoveryCaps::kMaxAge; refetch is never more often than + // kMinRefetch. + std::shared_ptr directory(); + // `ref` is "name" (preferred version) or "name:version". + std::shared_ptr api(const std::string& ref); + // Drops one cached document (subject to the kMinRefetch floor on refetch). + void forget(const std::string& ref); + CatalogStats stats() const; + + private: + struct Entry { + std::shared_ptr api; + std::chrono::steady_clock::time_point fetched; + std::size_t bytes = 0; + }; + std::chrono::steady_clock::time_point now() const; + void evict_locked(); + + std::shared_ptr source_; + Clock clock_; + mutable std::mutex mu_; + std::shared_ptr directory_; + std::chrono::steady_clock::time_point directory_at_{}; + std::list lru_; // most recent first, keys are item ids + std::map entries_; + std::map last_fetch_; + CatalogStats stats_; +}; + +} // namespace molo::google::discovery diff --git a/src/google/discovery/engine.cpp b/src/google/discovery/engine.cpp new file mode 100644 index 0000000..482770e --- /dev/null +++ b/src/google/discovery/engine.cpp @@ -0,0 +1,228 @@ +#include "google/discovery/engine.h" + +#include +#include + +#include "google/discovery/caps.h" +#include "google/errors.h" + +namespace molo::google::discovery { + +using clients::body::Json; + +namespace { +[[noreturn]] void bad(const std::string& why) { throw GoogleError::invalid(why); } + +std::string str(const Json& o, const char* key) { + const Json* v = o.find(key); + return v && v->is_string() ? v->as_string() : ""; +} +} // namespace + +Json CallResult::to_json() const { + Json o = Json::object(); + o.set("body", body); + o.set("pages", Json(static_cast(pages))); + o.set("truncated", Json(truncated)); + o.set("operationWaited", Json(operation_waited)); + o.set("itemsKey", Json(items_key)); + o.set("support", Json(support_name(support))); + Json n = Json::array(); + for (const std::string& s : notes) n.push(Json(s)); + o.set("notes", n); + return o; +} + +Engine::Engine(std::shared_ptr client, std::shared_ptr catalog, + Sleeper sleeper) + : client_(std::move(client)), catalog_(std::move(catalog)), sleeper_(std::move(sleeper)) { + if (!sleeper_) + sleeper_ = [](std::chrono::milliseconds d) { std::this_thread::sleep_for(d); }; +} + +Json Engine::list_apis(const std::string& filter, std::size_t limit, bool preferred_only) { + auto dir = catalog_->directory(); + Json out = Json::array(); + for (const DirectoryItem* i : dir->search(filter, limit, preferred_only)) out.push(i->to_json()); + return out; +} + +Json Engine::list_methods(const std::string& api_ref, const std::string& filter, + std::size_t limit) { + auto api = catalog_->api(api_ref); + limit = std::min(limit, DiscoveryCaps::kMaxListed); + Json out = Json::array(); + std::size_t matched = 0; + for (const Method& m : api->methods()) { + if (!filter.empty() && m.id.find(filter) == std::string::npos) continue; + ++matched; + if (out.as_array().size() < limit) out.push(m.summary()); + } + Json o = Json::object(); + o.set("api", Json(api->id())); + o.set("total", Json(static_cast(api->methods().size()))); + o.set("matched", Json(static_cast(matched))); + o.set("truncated", Json(matched > out.as_array().size())); + o.set("methods", out); + return o; +} + +const Method& Engine::method_of(const Api& api, const std::string& method_id) const { + const Method* m = api.find(method_id); + if (!m) throw GoogleError(404, "NOT_FOUND", "api " + api.id() + " has no method " + method_id, "discovery"); + return *m; +} + +Json Engine::describe(const std::string& api_ref, const std::string& method_id) { + auto api = catalog_->api(api_ref); + const Method& m = method_of(*api, method_id); + Json o = m.summary(); + Json params = Json::array(); + for (const auto& [n, p] : m.params) { + Json pj = Json::object(); + pj.set("name", Json(n)); + pj.set("in", Json(p.location)); + pj.set("type", Json(p.type)); + if (!p.format.empty()) pj.set("format", Json(p.format)); + pj.set("required", Json(p.required)); + pj.set("repeated", Json(p.repeated)); + if (!p.pattern.empty()) pj.set("pattern", Json(p.pattern)); + if (!p.enum_values.empty()) { + Json e = Json::array(); + for (const std::string& v : p.enum_values) e.push(Json(v)); + pj.set("enum", e); + } + params.push(pj); + } + o.set("parameters", params); + Json scopes = Json::array(); + for (const std::string& s : m.scopes) scopes.push(Json(s)); + o.set("scopes", scopes); + o.set("request", Json(m.request_ref)); + o.set("response", Json(m.response_ref)); + o.set("pageable", Json(api->is_pageable(m))); + o.set("itemsKey", Json(api->is_pageable(m) ? api->items_key(m) : "")); + o.set("operation", Json(!m.response_ref.empty() && api->is_operation_schema(m.response_ref))); + Json locs = Json::array(); + for (const EndpointRow& e : api->endpoints()) locs.push(Json(e.location)); + o.set("locations", locs); + return o; +} + +PreparedCall Engine::plan(const std::string& api_ref, const std::string& method_id, + const CallArgs& args) { + auto api = catalog_->api(api_ref); + return Planner::plan(*api, method_of(*api, method_id), args); +} + +Json Engine::send(const PreparedCall& p) { + return client_->request(p.http, p.endpoint, p.path, p.has_body ? &p.body : nullptr); +} + +CallResult Engine::call(const std::string& api_ref, const std::string& method_id, + const CallArgs& args) { + if (args.max_pages == 0 || args.max_pages > DiscoveryCaps::kMaxPages) + bad("max_pages must be 1 to " + std::to_string(DiscoveryCaps::kMaxPages)); + auto api = catalog_->api(api_ref); + const Method& m = method_of(*api, method_id); + PreparedCall first = Planner::plan(*api, m, args); // all validation, no traffic + if (args.max_pages > 1) { + if (!first.pageable) bad("method " + m.id + " is not pageable (needs a pageToken parameter and a nextPageToken reply)"); + if (args.items_key.empty() && api->items_key(m).empty()) + bad("method " + m.id + " has several list fields: name one with items_key"); + } + if (!client_->configured()) throw NotConfiguredError(api->name()); + + CallResult out; + out.support = first.support; + out.notes = first.notes; + if (args.max_pages > 1) { + walk_pages(*api, m, args, first, out); + return out; + } + out.body = send(first); + if (args.wait && first.operation) { + out.body = wait_for(*api, first, args, out.body); + out.operation_waited = true; + } + return out; +} + +void Engine::walk_pages(const Api& api, const Method& m, const CallArgs& args, + const PreparedCall& first, CallResult& out) { + std::string key = !args.items_key.empty() ? args.items_key : api.items_key(m); + out.items_key = key; + Json items = Json::array(); + Json last = Json::object(); + PreparedCall call = first; + std::size_t count = 0; + for (std::size_t page = 0; page < args.max_pages; ++page) { + last = send(call); + out.pages = page + 1; + if (const Json* arr = last.find(key); arr && arr->is_array()) + for (const Json& item : arr->as_array()) { + if (count >= DiscoveryCaps::kMaxItems) { + out.truncated = true; + break; + } + items.push(item); + ++count; + } + const Json* next = last.find("nextPageToken"); + std::string token = next && next->is_string() ? next->as_string() : ""; + if (token.empty()) break; + if (out.truncated || page + 1 == args.max_pages) { + out.truncated = true; + break; + } + CallArgs again = args; + again.params.set("pageToken", Json(token)); + call = Planner::plan(api, m, again); + } + if (last.is_object()) { + last.set(key, items); + if (!out.truncated) last.as_object().erase("nextPageToken"); + } + out.body = last; +} + +Json Engine::wait_for(const Api& api, const PreparedCall& origin, const CallArgs& args, + const Json& op_in) { + Json op = op_in; + std::string name = str(op, "name"); + if (name.empty()) bad("operation reply has no name"); + const Method* getter = nullptr; + for (const Method* g : api.operation_getters()) { + const Param& p = g->params.at("name"); + if (p.pattern.empty() ? api.operation_getters().size() == 1 + : std::regex_search(name, std::regex(p.pattern))) { + getter = g; + break; + } + } + if (!getter) + bad("no operations.get method of " + api.id() + " accepts the operation name " + name); + CallArgs poll; + poll.params.set("name", Json(name)); + poll.location = args.location; + poll.auth = args.auth; + PreparedCall get = Planner::plan(api, *getter, poll); + (void)origin; + std::chrono::milliseconds waited{0}; + for (std::size_t i = 0; i < OperationCaps::kMaxPolls; ++i) { + Operation parsed = Operation::from_json(op); + if (parsed.done) { + if (parsed.failed()) + throw GoogleError(502, "OPERATION_FAILED", parsed.error_message, api.name()); + return op; + } + if (waited >= OperationCaps::kMaxWait) break; + sleeper_(OperationCaps::kInterval); + waited += OperationCaps::kInterval; + op = send(get); + } + throw GoogleError(504, "DEADLINE_EXCEEDED", + "operation still running at the wait cap: " + name, api.name()); +} + +} // namespace molo::google::discovery diff --git a/src/google/discovery/engine.h b/src/google/discovery/engine.h new file mode 100644 index 0000000..a26eab8 --- /dev/null +++ b/src/google/discovery/engine.h @@ -0,0 +1,68 @@ +#pragma once + +// The Discovery engine: reach any method of any Google Discovery API by id. +// Engine e(client, catalog); +// e.call("bigquery", "bigquery.datasets.list", args); +// Documents are fetched without credentials and cached (Catalog); the call +// itself goes through GoogleClient with the credential the planner chose. + +#include +#include +#include + +#include "google/client.h" +#include "google/discovery/catalog.h" +#include "google/discovery/planner.h" +#include "google/operation.h" + +namespace molo::google::discovery { + +struct CallResult { + clients::body::Json body; + std::size_t pages = 1; + bool truncated = false; // page or item cap hit with more pages left + bool operation_waited = false; + std::string items_key; // set when pages were merged + Support support = Support::kFull; + std::vector notes; + + clients::body::Json to_json() const; +}; + +class Engine { + public: + Engine(std::shared_ptr client, std::shared_ptr catalog, + Sleeper sleeper = nullptr); + + bool configured() const { return client_->configured(); } + Catalog& catalog() { return *catalog_; } + + // Directory rows as Json; limit is capped at DiscoveryCaps::kMaxListed. + clients::body::Json list_apis(const std::string& filter, std::size_t limit, + bool preferred_only = true); + // Methods of one API (optionally those whose id contains `filter`). + clients::body::Json list_methods(const std::string& api_ref, const std::string& filter, + std::size_t limit); + clients::body::Json describe(const std::string& api_ref, const std::string& method_id); + + // Validation only: no call to Google (the document fetch is not a call to + // the API). Throws GoogleError::invalid on any problem. + PreparedCall plan(const std::string& api_ref, const std::string& method_id, + const CallArgs& args); + CallResult call(const std::string& api_ref, const std::string& method_id, + const CallArgs& args); + + private: + const Method& method_of(const Api& api, const std::string& method_id) const; + clients::body::Json send(const PreparedCall& p); + clients::body::Json wait_for(const Api& api, const PreparedCall& origin, + const CallArgs& args, const clients::body::Json& op); + void walk_pages(const Api& api, const Method& m, const CallArgs& args, + const PreparedCall& first, CallResult& out); + + std::shared_ptr client_; + std::shared_ptr catalog_; + Sleeper sleeper_; +}; + +} // namespace molo::google::discovery diff --git a/src/google/discovery/model.cpp b/src/google/discovery/model.cpp new file mode 100644 index 0000000..b5fc51e --- /dev/null +++ b/src/google/discovery/model.cpp @@ -0,0 +1,338 @@ +#include "google/discovery/model.h" + +#include +#include +#include + +#include "google/discovery/caps.h" +#include "google/errors.h" + +namespace molo::google::discovery { + +using clients::body::Json; + +namespace { + +[[noreturn]] void bad_doc(const std::string& why) { + throw GoogleError(502, "BAD_DOCUMENT", "discovery document: " + why); +} + +std::string str(const Json& o, const char* key) { + const Json* v = o.find(key); + return v && v->is_string() ? v->as_string() : ""; +} + +bool flag(const Json& o, const char* key) { + const Json* v = o.find(key); + return v && v->is_bool() && v->as_bool(); +} + +std::optional num_or_string(const Json& o, const char* key) { + const Json* v = o.find(key); + if (!v) return std::nullopt; + if (v->is_number()) return v->as_number(); + if (v->is_string()) { + try { + return std::stod(v->as_string()); + } catch (...) { + } + } + return std::nullopt; +} + +std::string ref_of(const Json& o, const char* key) { + const Json* v = o.find(key); + return v && v->is_object() ? str(*v, "$ref") : ""; +} + +Param parse_param(const std::string& name, const Json& p) { + Param out; + out.name = name; + out.type = str(p, "type"); + out.format = str(p, "format"); + out.location = str(p, "location"); + out.pattern = str(p, "pattern"); + out.description = str(p, "description"); + out.required = flag(p, "required"); + out.repeated = flag(p, "repeated"); + out.deprecated = flag(p, "deprecated"); + if (const Json* e = p.find("enum"); e && e->is_array()) + for (const Json& v : e->as_array()) + if (v.is_string()) out.enum_values.push_back(v.as_string()); + out.minimum = num_or_string(p, "minimum"); + out.maximum = num_or_string(p, "maximum"); + return out; +} + +// Names of {var} / {+var} in a path template, in order. +std::vector template_vars(const std::string& path, bool* malformed) { + std::vector out; + for (std::size_t i = 0; i < path.size(); ++i) { + if (path[i] == '}') { + *malformed = true; + return out; + } + if (path[i] != '{') continue; + std::size_t close = path.find('}', i); + if (close == std::string::npos) { + *malformed = true; + return out; + } + std::string v = path.substr(i + 1, close - i - 1); + if (!v.empty() && v[0] == '+') v.erase(0, 1); + if (v.empty() || v.find('{') != std::string::npos) { + *malformed = true; + return out; + } + out.push_back(v); + i = close; + } + return out; +} + +void classify(Method& m) { + auto reject = [&](const std::string& why) { + m.support = Support::kRejected; + m.notes.push_back(why); + }; + auto limit = [&](const std::string& why) { + if (m.support == Support::kFull) m.support = Support::kLimited; + m.notes.push_back(why); + }; + static const std::set verbs = {"GET", "POST", "PUT", "PATCH", "DELETE"}; + if (!verbs.count(m.http_method)) reject("http method " + m.http_method + " is not supported"); + if (m.streaming_type == "SERVER_SIDE") + limit("server-streaming method: the reply is read whole, no incremental delivery"); + else if (!m.streaming_type.empty() && m.streaming_type != "NONE") + reject("streaming method (" + m.streaming_type + ") needs a socket, not a request"); + bool malformed = false; + std::set in_path; + for (const std::string& v : template_vars(m.path, &malformed)) in_path.insert(v); + if (malformed) reject("path template is malformed"); + for (const std::string& v : in_path) { + auto it = m.params.find(v); + if (it == m.params.end()) reject("path variable " + v + " has no declared parameter"); + else if (it->second.location != "path") + reject("path variable " + v + " is declared as " + it->second.location); + } + for (const auto& [name, p] : m.params) { + if (p.location != "path" && p.location != "query") + reject("parameter " + name + " has location " + p.location); + else if (p.location == "path" && !in_path.count(name)) + reject("path parameter " + name + " is not in the path template"); + if (p.type == "object") reject("parameter " + name + " is an object (map) parameter"); + else if (p.type != "string" && p.type != "integer" && p.type != "number" && + p.type != "boolean") + reject("parameter " + name + " has unknown type " + p.type); + } + if (m.media.supported) + limit(std::string("media upload (") + (m.media.simple ? "simple" : "") + + (m.media.simple && m.media.resumable ? ", " : "") + + (m.media.resumable ? "resumable" : "") + + ") not implemented: only the metadata call is made"); + if (m.supports_download) limit("media download (alt=media) not implemented: JSON reply only"); + if (m.response_ref == "HttpBody") limit("reply is google.api.HttpBody: may not be JSON"); +} + +void collect(const Json& node, std::vector& out, std::size_t depth) { + if (depth > DiscoveryCaps::kMaxResourceDepth) bad_doc("resources nested too deep"); + if (const Json* ms = node.find("methods"); ms && ms->is_object()) { + for (const auto& [mname, mj] : ms->as_object()) { + (void)mname; + if (!mj.is_object()) bad_doc("method is not an object"); + if (out.size() >= DiscoveryCaps::kMaxMethods) bad_doc("more methods than the cap"); + Method m; + m.id = str(mj, "id"); + m.http_method = str(mj, "httpMethod"); + m.path = str(mj, "path"); + m.flat_path = str(mj, "flatPath"); + if (m.id.empty() || m.path.empty()) bad_doc("method without id or path"); + m.request_ref = ref_of(mj, "request"); + m.response_ref = ref_of(mj, "response"); + m.supports_download = flag(mj, "supportsMediaDownload"); + m.deprecated = flag(mj, "deprecated"); + m.streaming_type = str(mj, "streamingType"); + if (const Json* sc = mj.find("scopes"); sc && sc->is_array()) + for (const Json& s : sc->as_array()) + if (s.is_string()) m.scopes.push_back(s.as_string()); + if (const Json* ps = mj.find("parameters"); ps && ps->is_object()) + for (const auto& [pn, pj] : ps->as_object()) + if (pj.is_object()) m.params[pn] = parse_param(pn, pj); + if (const Json* po = mj.find("parameterOrder"); po && po->is_array()) + for (const Json& s : po->as_array()) + if (s.is_string()) m.param_order.push_back(s.as_string()); + if (const Json* mu = mj.find("mediaUpload"); mu && mu->is_object()) { + m.media.supported = true; + if (const Json* pr = mu->find("protocols"); pr && pr->is_object()) { + m.media.simple = pr->has("simple"); + m.media.resumable = pr->has("resumable"); + } + if (auto sz = num_or_string(*mu, "maxSize")) m.media.max_size = *sz; + } + classify(m); + out.push_back(std::move(m)); + } + } + if (const Json* rs = node.find("resources"); rs && rs->is_object()) + for (const auto& [rn, rj] : rs->as_object()) { + (void)rn; + if (rj.is_object()) collect(rj, out, depth + 1); + } +} + +} // namespace + +const char* support_name(Support s) { + switch (s) { + case Support::kFull: return "full"; + case Support::kLimited: return "limited"; + case Support::kRejected: return "rejected"; + } + return "rejected"; +} + +bool is_google_host_url(const std::string& url) { + static const std::string kScheme = "https://"; + if (url.rfind(kScheme, 0) != 0) return false; + std::size_t end = url.find_first_of("/?#", kScheme.size()); + std::string host = url.substr(kScheme.size(), end == std::string::npos + ? std::string::npos + : end - kScheme.size()); + if (host.empty() || host.find('@') != std::string::npos) return false; + if (std::size_t colon = host.find(':'); colon != std::string::npos) { + if (host.substr(colon) != ":443") return false; + host.erase(colon); + } + static const std::string kSuffix = ".googleapis.com"; + if (host.size() <= kSuffix.size() || + host.compare(host.size() - kSuffix.size(), kSuffix.size(), kSuffix) != 0) + return false; + for (char c : host) + if (!(std::isalnum(static_cast(c)) || c == '.' || c == '-')) return false; + return true; +} + +Json Method::summary() const { + Json o = Json::object(); + o.set("id", Json(id)); + o.set("httpMethod", Json(http_method)); + o.set("path", Json(path)); + o.set("support", Json(support_name(support))); + o.set("hasBody", Json(!request_ref.empty())); + o.set("oauth", Json(needs_oauth())); + o.set("deprecated", Json(deprecated)); + Json req = Json::array(); + for (const auto& [n, p] : params) + if (p.required) req.push(Json(n)); + o.set("required", req); + Json notes_j = Json::array(); + for (const std::string& n : notes) notes_j.push(Json(n)); + o.set("notes", notes_j); + return o; +} + +Api Api::parse(const Json& doc) { + if (!doc.is_object()) bad_doc("not an object"); + if (str(doc, "kind") != "discovery#restDescription") bad_doc("kind is not discovery#restDescription"); + if (str(doc, "protocol") != "rest") bad_doc("protocol is not rest"); + Api api; + api.id_ = str(doc, "id"); + api.name_ = str(doc, "name"); + api.version_ = str(doc, "version"); + api.title_ = str(doc, "title"); + api.root_url_ = str(doc, "rootUrl"); + api.service_path_ = str(doc, "servicePath"); + api.batch_path_ = str(doc, "batchPath"); + if (api.id_.empty() || api.name_.empty() || api.version_.empty()) bad_doc("missing id, name or version"); + if (!is_google_host_url(api.root_url_)) bad_doc("rootUrl is not an https googleapis.com host: " + api.root_url_); + if (api.root_url_.back() != '/') bad_doc("rootUrl has no trailing slash"); + if (api.service_path_.find_first_of("?#{}") != std::string::npos) bad_doc("servicePath is not a plain path"); + if (const Json* eps = doc.find("endpoints"); eps && eps->is_array()) + for (const Json& e : eps->as_array()) { + EndpointRow row{str(e, "location"), str(e, "endpointUrl"), str(e, "description")}; + if (row.location.empty() || !is_google_host_url(row.url) || row.url.back() != '/') + bad_doc("endpoint row is not a googleapis.com root URL: " + row.url); + api.endpoints_.push_back(std::move(row)); + } + if (const Json* ps = doc.find("parameters"); ps && ps->is_object()) + for (const auto& [n, p] : ps->as_object()) { + (void)p; + api.global_params_.push_back(n); + } + if (const Json* s = doc.find("schemas"); s && s->is_object()) { + if (s->as_object().size() > DiscoveryCaps::kMaxSchemas) bad_doc("more schemas than the cap"); + api.schemas_ = *s; + } + collect(doc, api.methods_, 0); + for (std::size_t i = 0; i < api.methods_.size(); ++i) + if (!api.index_.emplace(api.methods_[i].id, i).second) + bad_doc("duplicate method id " + api.methods_[i].id); + return api; +} + +std::string Api::base_url() const { + std::string b = root_url_ + service_path_; + while (!b.empty() && b.back() == '/') b.pop_back(); + return b; +} + +const Method* Api::find(const std::string& method_id) const { + auto it = index_.find(method_id); + return it == index_.end() ? nullptr : &methods_[it->second]; +} + +const Json* Api::schema(const std::string& name) const { return schemas_.find(name); } + +std::size_t Api::schema_count() const { return schemas_.as_object().size(); } + +bool Api::is_operation_schema(const std::string& ref) const { + const Json* s = schema(ref); + if (!s) return false; + const Json* props = s->find("properties"); + if (!props || !props->is_object()) return false; + const Json* done = props->find("done"); + return props->has("name") && done && done->is_object() && str(*done, "type") == "boolean"; +} + +bool Api::is_pageable(const Method& m) const { + auto it = m.params.find("pageToken"); + if (it == m.params.end() || it->second.location != "query") return false; + const Json* s = schema(m.response_ref); + if (!s) return false; + const Json* props = s->find("properties"); + return props && props->is_object() && props->has("nextPageToken"); +} + +std::string Api::items_key(const Method& m) const { + const Json* s = schema(m.response_ref); + const Json* props = s ? s->find("properties") : nullptr; + if (!props || !props->is_object()) return ""; + std::string found; + for (const auto& [n, p] : props->as_object()) { + if (n == "nextPageToken" || n == "unreachable") continue; + if (p.is_object() && str(p, "type") == "array") { + if (!found.empty()) return ""; // ambiguous: caller names it + found = n; + } + } + return found; +} + +std::vector Api::operation_getters() const { + std::vector out; + static const std::string kSuffix = "operations.get"; + for (const Method& m : methods_) { + if (m.id.size() < kSuffix.size() || + m.id.compare(m.id.size() - kSuffix.size(), kSuffix.size(), kSuffix) != 0) + continue; + if (m.http_method != "GET" || m.support == Support::kRejected) continue; + if (!is_operation_schema(m.response_ref)) continue; + std::size_t required = 0; + bool name_path = m.params.count("name") && m.params.at("name").location == "path"; + for (const auto& [n, p] : m.params) required += p.required ? 1 : 0; + if (name_path && required == 1) out.push_back(&m); + } + return out; +} + +} // namespace molo::google::discovery diff --git a/src/google/discovery/model.h b/src/google/discovery/model.h new file mode 100644 index 0000000..59d00da --- /dev/null +++ b/src/google/discovery/model.h @@ -0,0 +1,118 @@ +#pragma once + +// Parsed Discovery document: parameters, methods, schemas. Pure data plus +// classification of what the engine can and cannot do with each method. +// Every shape here was read from the real documents (see SPEC.md section +// "Shapes seen"); anything else is rejected loudly by Api::parse. + +#include +#include +#include +#include + +#include "clients/body/json.h" + +namespace molo::google::discovery { + +// Full: called as documented. Limited: callable, but part of the method's +// surface is not reachable (media bytes, non-JSON reply); notes say which. +// Rejected: never called; the notes say why. +enum class Support { kFull, kLimited, kRejected }; +const char* support_name(Support s); + +struct Param { + std::string name; + std::string type; // string integer number boolean (object: rejected) + std::string format; + std::string location; // path or query + std::string pattern; + std::string description; + bool required = false; + bool repeated = false; + bool deprecated = false; + std::vector enum_values; + std::optional minimum; + std::optional maximum; +}; + +struct MediaUpload { + bool supported = false; + bool simple = false; + bool resumable = false; + double max_size = 0; // bytes, 0: not stated +}; + +struct Method { + std::string id; // bigquery.jobs.query + std::string http_method; // GET POST PUT PATCH DELETE + std::string path; // v1/{+name}:cancel (relative to base) + std::string flat_path; + std::string request_ref; // schema name, empty: no body + std::string response_ref; // schema name, empty: no body + std::map params; + std::vector param_order; + std::vector scopes; + MediaUpload media; + bool supports_download = false; + bool deprecated = false; + std::string streaming_type; + Support support = Support::kFull; + std::vector notes; // why Limited or Rejected, one per cause + + bool needs_oauth() const { return !scopes.empty(); } + clients::body::Json summary() const; +}; + +struct EndpointRow { + std::string location; + std::string url; // root URL, trailing slash + std::string description; +}; + +class Api { + public: + // Throws GoogleError (502 BAD_DOCUMENT) when the document is not one the + // engine can trust: wrong kind/protocol, host outside googleapis.com, + // over the method cap, duplicate method ids. + static Api parse(const clients::body::Json& doc); + + const std::string& id() const { return id_; } + const std::string& name() const { return name_; } + const std::string& version() const { return version_; } + const std::string& title() const { return title_; } + const std::string& root_url() const { return root_url_; } + const std::string& service_path() const { return service_path_; } + // root_url + service_path without trailing slash. + std::string base_url() const; + const std::vector& endpoints() const { return endpoints_; } + const std::vector& methods() const { return methods_; } + const Method* find(const std::string& method_id) const; + const clients::body::Json* schema(const std::string& name) const; + std::size_t schema_count() const; + // Query parameters every method accepts, as declared by the document. + const std::vector& global_params() const { return global_params_; } + bool batch_path_declared() const { return !batch_path_.empty(); } + + bool is_operation_schema(const std::string& ref) const; + bool is_pageable(const Method& m) const; + // The array property to collect when paging m, or "" when none/ambiguous. + std::string items_key(const Method& m) const; + // Methods ending in "operations.get" usable to poll an operation. + std::vector operation_getters() const; + + // Raw bytes of the parsed text, for cache accounting. + std::size_t raw_bytes = 0; + + private: + std::string id_, name_, version_, title_, root_url_, service_path_, batch_path_; + std::vector endpoints_; + std::vector methods_; + std::map index_; + clients::body::Json schemas_ = clients::body::Json::object(); + std::vector global_params_; +}; + +// True for an https URL whose host is googleapis.com or a subdomain of it. +bool is_google_host_url(const std::string& url); + +} // namespace molo::google::discovery diff --git a/src/google/discovery/planner.cpp b/src/google/discovery/planner.cpp new file mode 100644 index 0000000..1177157 --- /dev/null +++ b/src/google/discovery/planner.cpp @@ -0,0 +1,207 @@ +#include "google/discovery/planner.h" + +#include +#include + +#include "clients/url.h" +#include "google/discovery/caps.h" +#include "google/discovery/validate.h" +#include "google/errors.h" + +namespace molo::google::discovery { + +using clients::body::Json; + +namespace { + +[[noreturn]] void bad(const std::string& why) { throw GoogleError::invalid(why); } + +// Document-level query parameters a caller may set. Everything else the +// document declares globally (alt, key, access_token, oauth_token, callback, +// uploadType, upload_protocol, $.xgafv) either changes the reply format, +// carries a credential, or switches protocol: rejected by name. +bool allowed_global(const std::string& name) { + return name == "fields" || name == "quotaUser" || name == "prettyPrint"; +} + +} // namespace + +const char* auth_mode_name(AuthMode m) { + switch (m) { + case AuthMode::kAuto: return "auto"; + case AuthMode::kKey: return "key"; + case AuthMode::kOAuth: return "oauth"; + } + return "auto"; +} + +AuthMode auth_mode_from(const std::string& name) { + if (name.empty() || name == "auto") return AuthMode::kAuto; + if (name == "key") return AuthMode::kKey; + if (name == "oauth") return AuthMode::kOAuth; + bad("auth must be auto, key or oauth"); +} + +std::string Planner::encode_path_value(const std::string& value, bool reserved) { + if (!reserved) return clients::Url::encode_component(value); + std::string out; + std::size_t start = 0; + while (true) { + std::size_t end = value.find('/', start); + out += clients::Url::encode_component( + value.substr(start, end == std::string::npos ? std::string::npos : end - start)); + if (end == std::string::npos) break; + out += '/'; + start = end + 1; + } + return out; +} + +Json PreparedCall::to_json() const { + Json o = Json::object(); + o.set("api", Json(api_id)); + o.set("method", Json(method_id)); + o.set("httpMethod", Json(clients::method_name(http))); + o.set("url", Json(endpoint.base_url + path)); + o.set("credential", Json(endpoint.prefer_key ? "key" : "oauth")); + o.set("hasBody", Json(has_body)); + o.set("support", Json(support_name(support))); + o.set("pageable", Json(pageable)); + o.set("operation", Json(operation)); + Json n = Json::array(); + for (const std::string& s : notes) n.push(Json(s)); + o.set("notes", n); + return o; +} + +PreparedCall Planner::plan(const Api& api, const Method& m, const CallArgs& args) { + if (m.support == Support::kRejected) { + std::string why; + for (const std::string& n : m.notes) why += (why.empty() ? "" : "; ") + n; + bad("method " + m.id + " cannot be called by this engine: " + why); + } + if (!args.params.is_object()) bad("params must be an object"); + + // Endpoint: the global host, or a documented regional one. + std::string root = api.root_url(); + if (!args.location.empty()) { + auto it = std::find_if(api.endpoints().begin(), api.endpoints().end(), + [&](const EndpointRow& e) { return e.location == args.location; }); + if (it == api.endpoints().end()) { + std::string known; + for (std::size_t i = 0; i < api.endpoints().size() && i < 8; ++i) + known += (i ? ", " : "") + api.endpoints()[i].location; + bad("api " + api.id() + " has no endpoint for location '" + args.location + "'" + + (known.empty() ? " (it documents no regional endpoints)" : " (e.g. " + known + ")")); + } + root = it->url; + } + std::string base = root + api.service_path(); + while (!base.empty() && base.back() == '/') base.pop_back(); + + // Parameters. + std::set seen; + std::vector> query; + std::map path_values; + for (const auto& [name, value] : args.params.as_object()) { + auto it = m.params.find(name); + if (it == m.params.end()) { + const auto& g = api.global_params(); + if (std::find(g.begin(), g.end(), name) != g.end()) { + if (!allowed_global(name)) + bad("parameter " + name + " is reserved: it is set by the engine or changes the reply format"); + Param p; + p.name = name; + p.location = "query"; + p.type = name == "prettyPrint" ? "boolean" : "string"; + for (const std::string& v : ParamValidator::values(p, value)) query.emplace_back(name, v); + seen.insert(name); + continue; + } + std::string known; + for (const auto& [n, p] : m.params) known += (known.empty() ? "" : ", ") + n; + bad("method " + m.id + " has no parameter '" + name + "' (accepts: " + known + ")"); + } + const Param& p = it->second; + std::vector vals = ParamValidator::values(p, value); + seen.insert(name); + if (p.location == "path") { + if (vals.size() != 1) bad("path parameter " + name + " takes one value"); + path_values[name] = vals[0]; + } else { + for (const std::string& v : vals) query.emplace_back(name, v); + } + } + for (const auto& [name, p] : m.params) + if (p.required && !seen.count(name)) bad("required parameter " + name + " is missing"); + if (query.size() > DiscoveryCaps::kMaxQueryParams) + bad("more than " + std::to_string(DiscoveryCaps::kMaxQueryParams) + " query values"); + + // Path: substitute {x} and {+x}. + std::string path; + for (std::size_t i = 0; i < m.path.size(); ++i) { + if (m.path[i] != '{') { + path += m.path[i]; + continue; + } + std::size_t close = m.path.find('}', i); + std::string var = m.path.substr(i + 1, close - i - 1); + bool reserved = !var.empty() && var[0] == '+'; + if (reserved) var.erase(0, 1); + auto v = path_values.find(var); + if (v == path_values.end()) bad("required path parameter " + var + " is missing"); + path += encode_path_value(v->second, reserved); + i = close; + } + std::string qs; + for (const auto& [k, v] : query) + qs += (qs.empty() ? "?" : "&") + clients::Url::encode_component(k) + "=" + + clients::Url::encode_component(v); + path = "/" + path + qs; + if (path.size() > DiscoveryCaps::kMaxPathChars) + bad("request target is longer than " + std::to_string(DiscoveryCaps::kMaxPathChars)); + + PreparedCall out; + out.api_id = api.id(); + out.method_id = m.id; + out.support = m.support; + out.notes = m.notes; + out.path = path; + if (m.http_method == "GET") out.http = clients::Method::kGet; + else if (m.http_method == "POST") out.http = clients::Method::kPost; + else if (m.http_method == "PUT") out.http = clients::Method::kPut; + else if (m.http_method == "PATCH") out.http = clients::Method::kPatch; + else out.http = clients::Method::kDelete; + + out.endpoint.name = api.name(); + out.endpoint.base_url = base; + switch (args.auth) { + case AuthMode::kKey: out.endpoint.prefer_key = true; break; + case AuthMode::kOAuth: out.endpoint.prefer_key = false; break; + case AuthMode::kAuto: out.endpoint.prefer_key = !m.needs_oauth(); break; + } + out.oauth_scoped = m.needs_oauth(); + + // Body. + bool verb_has_body = out.http == clients::Method::kPost || out.http == clients::Method::kPut || + out.http == clients::Method::kPatch; + if (m.request_ref.empty()) { + if (!args.body.is_null()) bad("method " + m.id + " takes no request body"); + if (verb_has_body) { + out.body = Json::object(); + out.has_body = true; + } + } else if (!verb_has_body) { + bad("method " + m.id + " declares a body on a " + m.http_method); + } else { + out.body = args.body.is_null() ? Json::object() : args.body; + BodyValidator(api).validate(m.request_ref, out.body); + out.has_body = true; + } + + out.pageable = api.is_pageable(m); + out.operation = !m.response_ref.empty() && api.is_operation_schema(m.response_ref); + return out; +} + +} // namespace molo::google::discovery diff --git a/src/google/discovery/planner.h b/src/google/discovery/planner.h new file mode 100644 index 0000000..289b15b --- /dev/null +++ b/src/google/discovery/planner.h @@ -0,0 +1,59 @@ +#pragma once + +// Turns (Api, Method, arguments) into one validated HTTP request. No +// network traffic happens here: every rule that can fail does so before a +// byte is sent. + +#include +#include + +#include "clients/body/json.h" +#include "clients/request.h" +#include "google/discovery/model.h" +#include "google/endpoints.h" + +namespace molo::google::discovery { + +// Which credential reaches Google. Auto: methods that declare no OAuth scope +// take the API key, every other method takes OAuth. Key / OAuth force the +// choice (the client falls back to the other credential only when the forced +// one is not configured). +enum class AuthMode { kAuto, kKey, kOAuth }; +const char* auth_mode_name(AuthMode mode); +AuthMode auth_mode_from(const std::string& name); // throws invalid + +struct CallArgs { + clients::body::Json params = clients::body::Json::object(); // name -> value + clients::body::Json body; // null: none + std::string location; // regional endpoint (Api::endpoints) or empty + AuthMode auth = AuthMode::kAuto; + std::size_t max_pages = 1; // > 1 walks nextPageToken (list methods only) + std::string items_key; // array to collect when paging, else derived + bool wait = false; // poll a long-running operation to completion +}; + +struct PreparedCall { + std::string api_id; + std::string method_id; + clients::Method http = clients::Method::kGet; + ServiceEndpoint endpoint; + std::string path; // "/v1/..." with query string + clients::body::Json body; + bool has_body = false; + Support support = Support::kFull; + std::vector notes; + bool pageable = false; + bool operation = false; // the reply is a google.longrunning.Operation + bool oauth_scoped = false; + + clients::body::Json to_json() const; +}; + +class Planner { + public: + static PreparedCall plan(const Api& api, const Method& method, const CallArgs& args); + // Percent-encodes one value for a path template ({+x} keeps '/'). + static std::string encode_path_value(const std::string& value, bool reserved); +}; + +} // namespace molo::google::discovery diff --git a/src/google/discovery/tools/audit_main.cc b/src/google/discovery/tools/audit_main.cc new file mode 100644 index 0000000..6b9efe1 --- /dev/null +++ b/src/google/discovery/tools/audit_main.cc @@ -0,0 +1,189 @@ +// Audit tool: reads saved Discovery documents (the same files the engine +// would fetch) and prints, as JSON, which APIs and methods the engine +// cannot handle or only handles in part, and why. Driven by +// check_discovery_directory.py; one classification, the engine's own. +// discovery-audit _.json> +#include +#include +#include +#include +#include +#include + +#include "google/discovery/catalog.h" +#include "google/discovery/model.h" +#include "google/discovery/planner.h" +#include "google/errors.h" + +using namespace molo::google; +using namespace molo::google::discovery; +using molo::clients::body::Json; + +namespace { + +// A value that satisfies a parameter's declaration, for a dry plan. +Json synth(const Param& p) { + if (!p.enum_values.empty()) return Json(p.enum_values[0]); + if (p.type == "boolean") return Json(true); + if (p.type == "integer") { + long v = p.minimum ? static_cast(*p.minimum) : 1; + return p.format == "int64" ? Json(std::to_string(v)) : Json(v); + } + if (p.type == "number") return Json(p.minimum ? *p.minimum : 1.0); + if (p.format == "int64" || p.format == "uint64") return Json("1"); + if (p.format == "google-datetime" || p.format == "date-time") return Json("2026-10-25T10:00:00Z"); + if (p.format == "google-duration") return Json("1s"); + std::string s = p.pattern; + auto rep = [&](const std::string& a, const std::string& b) { + for (std::size_t i = s.find(a); i != std::string::npos; i = s.find(a, i + b.size())) + s.replace(i, a.size(), b); + }; + if (s.empty()) return Json("x1"); + rep("[^/]+", "x1"); + rep("[^/]*", "x1"); + rep("[^/]", "x"); + if (!s.empty() && s.front() == '^') s.erase(0, 1); + if (!s.empty() && s.back() == '$') s.pop_back(); + return Json(s); +} + +Json synth_args(const Method& m) { + Json params = Json::object(); + for (const auto& [n, p] : m.params) { + if (!p.required) continue; + Json v = synth(p); + if (p.repeated) { + Json a = Json::array(); + a.push(v); + v = a; + } + params.set(n, v); + } + return params; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 2) { + std::fprintf(stderr, "usage: discovery-audit \n"); + return 2; + } + auto source = std::make_shared(argv[1]); + Catalog catalog(source); + Json report = Json::object(); + Json apis = Json::array(); + Json failed = Json::array(); + std::map reasons; + long methods = 0, full = 0, limited = 0, rejected = 0, pageable = 0, lro = 0, lro_unpollable = 0, + regional = 0, batch = 0, planned = 0, plan_skipped = 0, plan_failed = 0; + Json plan_failures = Json::array(); + std::shared_ptr dir; + try { + dir = catalog.directory(); + } catch (const std::exception& e) { + std::fprintf(stderr, "directory: %s\n", e.what()); + return 1; + } + long preferred = 0; + for (const DirectoryItem& item : dir->items()) { + if (!item.preferred) continue; + ++preferred; + std::shared_ptr api; + try { + api = catalog.api(item.name + ":" + item.version); + } catch (const std::exception& e) { + Json f = Json::object(); + f.set("api", Json(item.id)); + f.set("why", Json(std::string(e.what()))); + failed.push(f); + continue; + } + bool has_getter = !api->operation_getters().empty(); + Json a = Json::object(); + a.set("api", Json(api->id())); + long a_full = 0, a_limited = 0, a_rejected = 0; + Json problems = Json::array(); + for (const Method& m : api->methods()) { + ++methods; + if (m.support == Support::kFull) ++full, ++a_full; + if (m.support == Support::kLimited) ++limited, ++a_limited; + if (m.support == Support::kRejected) ++rejected, ++a_rejected; + if (api->is_pageable(m)) ++pageable; + bool op = !m.response_ref.empty() && api->is_operation_schema(m.response_ref); + if (op) { + ++lro; + if (!has_getter) { + ++lro_unpollable; + reasons["long-running reply but the API has no operations.get (wait unavailable)"]++; + } + } + if (m.support != Support::kRejected) { + CallArgs args; + args.params = synth_args(m); + try { + (void)Planner::plan(*api, m, args); + ++planned; + } catch (const GoogleError& e) { + std::string why = e.message(); + if (why.find("does not match") != std::string::npos) { + ++plan_skipped; // synthetic value did not fit an exotic pattern + } else { + ++plan_failed; + Json f = Json::object(); + f.set("method", Json(m.id)); + f.set("why", Json(why)); + plan_failures.push(f); + } + } + } + for (const std::string& n : m.notes) { + // Reason class: strip parameter and path names so counts group. + std::string cls = n.substr(0, n.find(" (") == std::string::npos ? n.size() : n.find(" (")); + if (cls.rfind("parameter ", 0) == 0 || cls.rfind("path ", 0) == 0) cls = n; + reasons[std::string(support_name(m.support)) + ": " + cls]++; + } + if (m.support != Support::kFull) { + Json p = Json::object(); + p.set("method", Json(m.id)); + p.set("support", Json(support_name(m.support))); + Json notes = Json::array(); + for (const std::string& n : m.notes) notes.push(Json(n)); + p.set("notes", notes); + problems.push(p); + } + } + if (!api->endpoints().empty()) ++regional; + if (api->batch_path_declared()) ++batch; + a.set("methods", Json(static_cast(api->methods().size()))); + a.set("full", Json(a_full)); + a.set("limited", Json(a_limited)); + a.set("rejected", Json(a_rejected)); + a.set("operationGetter", Json(has_getter)); + a.set("regionalEndpoints", Json(static_cast(api->endpoints().size()))); + a.set("problems", problems); + apis.push(a); + } + report.set("preferred", Json(preferred)); + report.set("loaded", Json(static_cast(preferred) - static_cast(failed.as_array().size()))); + report.set("failed", failed); + report.set("methods", Json(methods)); + report.set("full", Json(full)); + report.set("limited", Json(limited)); + report.set("rejected", Json(rejected)); + report.set("pageable", Json(pageable)); + report.set("longRunning", Json(lro)); + report.set("longRunningWithoutGetter", Json(lro_unpollable)); + report.set("apisWithRegionalEndpoints", Json(regional)); + report.set("apisDeclaringBatchPath", Json(batch)); + Json rs = Json::object(); + for (const auto& [k, v] : reasons) rs.set(k, Json(v)); + report.set("reasons", rs); + report.set("planned", Json(planned)); + report.set("planSkippedPattern", Json(plan_skipped)); + report.set("planFailed", Json(plan_failed)); + report.set("planFailures", plan_failures); + report.set("apis", apis); + std::cout << report.dump() << "\n"; + return 0; +} diff --git a/src/google/discovery/tools/check_discovery_directory.py b/src/google/discovery/tools/check_discovery_directory.py new file mode 100644 index 0000000..d393299 --- /dev/null +++ b/src/google/discovery/tools/check_discovery_directory.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Walk Google's whole Discovery directory and report what the engine cannot do. + + python3 src/google/discovery/tools/check_discovery_directory.py [--cache DIR] + [--no-fetch] [--json FILE] [--max-rejected N] [--workers N] + +1. Fetches https://discovery.googleapis.com/discovery/v1/apis and the + document of every preferred API into --cache (default + build/google/discovery-cache). Fetch failures are listed as UNREACHABLE: + Google lists them but does not serve them (not an engine fault). +2. Builds build/google/discovery-audit (src/google/discovery/tools/audit_main.cc) + against libgoogle.a and runs it on the saved files, so the verdict comes + from the engine's own classification, never from a second implementation. +3. Prints totals, a reason table and the APIs with rejected or limited + methods. Exit 1 when a fetched document does not parse, or when rejected + methods exceed --max-rejected (default: no ceiling). + +python3 standard library only. +""" +import argparse +import concurrent.futures as cf +import json +import os +import subprocess +import sys +import urllib.request + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) +DIRECTORY = "https://discovery.googleapis.com/discovery/v1/apis" +AUDIT_SRC = os.path.join(ROOT, "src/google/discovery/tools/audit_main.cc") +AUDIT_BIN = os.path.join(ROOT, "build/google/discovery-audit") +LIBS = ["build/google/libgoogle.a", "build/clients/libclients.a", "build/liboperability.a"] + + +def get(url, tries=3): + last = None + for _ in range(tries): + try: + with urllib.request.urlopen(url, timeout=90) as r: + return r.read() + except Exception as e: # noqa: BLE001 - report whatever the network said + last = e + raise last + + +def fetch_all(cache, workers): + os.makedirs(cache, exist_ok=True) + raw = get(DIRECTORY) + with open(os.path.join(cache, "directory.json"), "wb") as f: + f.write(raw) + items = [i for i in json.loads(raw)["items"] if i.get("preferred")] + unreachable = [] + + def one(item): + path = os.path.join(cache, "%s_%s.json" % (item["name"], item["version"])) + try: + data = get(item["discoveryRestUrl"]) + with open(path, "wb") as f: + f.write(data) + return None + except Exception as e: # noqa: BLE001 + if os.path.exists(path): + os.remove(path) + return (item["id"], str(e)) + + with cf.ThreadPoolExecutor(workers) as ex: + for r in ex.map(one, items): + if r: + unreachable.append(r) + return len(items), unreachable + + +def build_audit(): + subprocess.run(["make", "-s", "-j8", "google", "build/clients/libclients.a", + "build/liboperability.a"], cwd=ROOT, check=True) + cmd = ["g++", "-std=c++23", "-Wall", "-Wextra", "-Werror", "-DMOLO_HAS_WOLFSSL", + "-I" + os.path.join(ROOT, "src"), AUDIT_SRC] + [os.path.join(ROOT, l) for l in LIBS] + \ + ["-lwolfssl", "-o", AUDIT_BIN] + subprocess.run(cmd, cwd=ROOT, check=True) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--cache", default=os.path.join(ROOT, "build/google/discovery-cache")) + ap.add_argument("--no-fetch", action="store_true", help="reuse the files already in --cache") + ap.add_argument("--json", help="write the full audit report here") + ap.add_argument("--max-rejected", type=int, default=None) + ap.add_argument("--workers", type=int, default=16) + ap.add_argument("--list", type=int, default=40, help="APIs to list with problems") + args = ap.parse_args() + + unreachable = [] + listed = None + if not args.no_fetch: + listed, unreachable = fetch_all(args.cache, args.workers) + build_audit() + out = subprocess.run([AUDIT_BIN, args.cache], capture_output=True, text=True) + if out.returncode != 0: + print(out.stderr, file=sys.stderr) + return 1 + rep = json.loads(out.stdout) + if args.json: + with open(args.json, "w") as f: + json.dump(rep, f, indent=1) + + unreach_ids = {u[0] for u in unreachable} + parse_failures = [f for f in rep["failed"] if f["api"] not in unreach_ids] + gone = [f for f in rep["failed"] if f["api"] in unreach_ids or "NOT_FOUND" in f["why"]] + print("preferred APIs in the directory : %d" % rep["preferred"]) + print("documents loaded by the engine : %d" % rep["loaded"]) + print("UNREACHABLE at the source : %d" % len(gone)) + for f in gone: + print(" %-40s %s" % (f["api"], f["why"][:90])) + parse_failures = [f for f in parse_failures if f not in gone] + print("documents the engine REJECTS : %d" % len(parse_failures)) + for f in parse_failures: + print(" %-40s %s" % (f["api"], f["why"][:160])) + print("methods : %d" % rep["methods"]) + print(" full / limited / rejected : %d / %d / %d" % (rep["full"], rep["limited"], rep["rejected"])) + print(" pageable (pageToken) : %d" % rep["pageable"]) + print(" long-running replies : %d (%d in APIs with no operations.get)" % + (rep["longRunning"], rep["longRunningWithoutGetter"])) + print("dry plans (required params only): %d planned, %d skipped (synthetic value vs exotic pattern), %d FAILED" % + (rep["planned"], rep["planSkippedPattern"], rep["planFailed"])) + for f in rep["planFailures"][:20]: + print(" %s: %s" % (f["method"], f["why"][:140])) + print("APIs with regional endpoints : %d" % rep["apisWithRegionalEndpoints"]) + print("APIs declaring a batchPath : %d (batch endpoints are not used)" % rep["apisDeclaringBatchPath"]) + print("\nreasons (methods):") + for k, v in sorted(rep["reasons"].items(), key=lambda kv: -kv[1]): + print(" %6d %s" % (v, k)) + bad = [a for a in rep["apis"] if a["rejected"] or a["limited"]] + bad.sort(key=lambda a: -(a["rejected"] * 1000 + a["limited"])) + print("\nAPIs with rejected or limited methods: %d of %d" % (len(bad), len(rep["apis"]))) + for a in bad[:args.list]: + sample = a["problems"][0] + print(" %-34s rejected %-3d limited %-3d e.g. %s: %s" % + (a["api"], a["rejected"], a["limited"], sample["method"], sample["notes"][0][:70])) + if len(bad) > args.list: + print(" ... %d more (use --json)" % (len(bad) - args.list)) + + code = 0 + if parse_failures or rep["planFailed"]: + code = 1 + if args.max_rejected is not None and rep["rejected"] > args.max_rejected: + print("FAIL: %d rejected methods exceed --max-rejected %d" % (rep["rejected"], args.max_rejected)) + code = 1 + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/google/discovery/validate.cpp b/src/google/discovery/validate.cpp new file mode 100644 index 0000000..31170af --- /dev/null +++ b/src/google/discovery/validate.cpp @@ -0,0 +1,220 @@ +#include "google/discovery/validate.h" + +#include +#include +#include + +#include "google/discovery/caps.h" +#include "google/errors.h" + +namespace molo::google::discovery { + +using clients::body::Json; + +namespace { + +[[noreturn]] void bad(const std::string& why) { throw GoogleError::invalid(why); } + +std::string sval(const Json& o, const char* key) { + const Json* v = o.find(key); + return v && v->is_string() ? v->as_string() : ""; +} + +bool all_digits(const std::string& s, bool allow_sign) { + std::size_t i = allow_sign && !s.empty() && s[0] == '-' ? 1 : 0; + if (i >= s.size() || s.size() - i > 20) return false; + for (; i < s.size(); ++i) + if (s[i] < '0' || s[i] > '9') return false; + return true; +} + +bool integral(double d) { return std::isfinite(d) && d == std::floor(d) && std::fabs(d) < 9.0e15; } + +std::string int_text(double d) { return std::to_string(static_cast(d)); } + +bool shape_ok(const std::string& format, const std::string& v) { + auto digit = [](char c) { return c >= '0' && c <= '9'; }; + if (format == "date-time" || format == "google-datetime") { + // 2026-10-25T10:00:00Z shape: YYYY-MM-DDT + return v.size() >= 11 && digit(v[0]) && digit(v[1]) && digit(v[2]) && digit(v[3]) && + v[4] == '-' && v[7] == '-' && (v[10] == 'T' || v[10] == 't'); + } + if (format == "google-duration") { + if (v.size() < 2 || v.back() != 's') return false; + bool dot = false; + for (std::size_t i = v[0] == '-' ? 1 : 0; i + 1 < v.size(); ++i) { + if (v[i] == '.' && !dot) dot = true; + else if (!digit(v[i])) return false; + } + return v.size() > 1; + } + return true; +} + +void control_free(const std::string& what, const std::string& v) { + for (unsigned char c : v) + if (c < 0x20 || c == 0x7f) bad(what + " contains a control character"); +} + +bool pattern_ok(const std::string& pattern, const std::string& v) { + try { + std::regex re(pattern, std::regex::ECMAScript | std::regex::optimize); + return std::regex_search(v, re); + } catch (const std::regex_error&) { + return true; // pattern syntax std::regex cannot read: Google checks it + } +} + +std::string one(const Param& p, const Json& v) { + const std::string who = "parameter " + p.name; + std::string out; + if (p.type == "string") { + if (v.is_string()) out = v.as_string(); + else if (v.is_number() && (p.format == "int64" || p.format == "uint64") && integral(v.as_number())) + out = int_text(v.as_number()); + else bad(who + " must be a string"); + if (p.format == "int64" && !all_digits(out, true)) bad(who + " must be a 64-bit integer"); + if (p.format == "uint64" && !all_digits(out, false)) bad(who + " must be an unsigned 64-bit integer"); + if (p.format == "uint64" && out.size() > 20) bad(who + " is too large"); + if (!shape_ok(p.format, out)) bad(who + " is not a valid " + p.format); + } else if (p.type == "integer") { + double d; + if (v.is_number()) d = v.as_number(); + else if (v.is_string() && all_digits(v.as_string(), true)) d = std::stod(v.as_string()); + else bad(who + " must be an integer"); + if (!integral(d)) bad(who + " must be an integer"); + if (p.format == "int32" && (d < -2147483648.0 || d > 2147483647.0)) bad(who + " is outside int32"); + if (p.format == "uint32" && (d < 0 || d > 4294967295.0)) bad(who + " is outside uint32"); + if (p.minimum && d < *p.minimum) bad(who + " is below the minimum " + int_text(*p.minimum)); + if (p.maximum && d > *p.maximum) bad(who + " is above the maximum " + int_text(*p.maximum)); + out = int_text(d); + } else if (p.type == "number") { + if (!v.is_number() || !std::isfinite(v.as_number())) bad(who + " must be a number"); + if (p.minimum && v.as_number() < *p.minimum) bad(who + " is below the minimum"); + if (p.maximum && v.as_number() > *p.maximum) bad(who + " is above the maximum"); + Json tmp(v.as_number()); + out = tmp.dump(); + } else if (p.type == "boolean") { + if (v.is_bool()) out = v.as_bool() ? "true" : "false"; + else if (v.is_string() && (v.as_string() == "true" || v.as_string() == "false")) out = v.as_string(); + else bad(who + " must be true or false"); + } else { + bad(who + " has an unsupported type " + p.type); + } + if (out.size() > DiscoveryCaps::kMaxParamChars) bad(who + " is longer than the cap"); + control_free(who, out); + if (!p.enum_values.empty() && + std::find(p.enum_values.begin(), p.enum_values.end(), out) == p.enum_values.end()) + bad(who + " is not one of its enum values"); + if (!p.pattern.empty() && !pattern_ok(p.pattern, out)) bad(who + " does not match " + p.pattern); + if (p.location == "path") { + if (out.empty()) bad(who + " is empty"); + // No dot segments: a path value must not climb out of its template. + std::size_t start = 0; + while (start <= out.size()) { + std::size_t end = out.find('/', start); + std::string seg = out.substr(start, end == std::string::npos ? std::string::npos : end - start); + if (seg == "." || seg == "..") bad(who + " contains a dot segment"); + if (end == std::string::npos) break; + start = end + 1; + } + } + return out; +} + +} // namespace + +std::vector ParamValidator::values(const Param& p, const Json& value) { + if (value.is_null()) bad("parameter " + p.name + " is null"); + std::vector out; + if (value.is_array()) { + if (!p.repeated) bad("parameter " + p.name + " is not repeated, got an array"); + if (value.as_array().size() > DiscoveryCaps::kMaxRepeated) + bad("parameter " + p.name + " has more than " + + std::to_string(DiscoveryCaps::kMaxRepeated) + " values"); + for (const Json& v : value.as_array()) out.push_back(one(p, v)); + return out; + } + if (value.is_object()) bad("parameter " + p.name + " must be a scalar"); + out.push_back(one(p, value)); + return out; +} + +struct BodyValidator::Walk { + std::size_t visits = 0; +}; + +void BodyValidator::validate(const std::string& ref, const Json& body) const { + if (body.dump().size() > DiscoveryCaps::kMaxBodyBytes) bad("request body is larger than the cap"); + const Json* schema = api_.schema(ref); + if (!schema) bad("unknown request schema " + ref); + Walk walk; + node(*schema, body, ref, walk, 0); +} + +void BodyValidator::node(const Json& schema_in, const Json& value, const std::string& at, + Walk& walk, std::size_t depth) const { + if (++walk.visits > DiscoveryCaps::kMaxSchemaVisits) bad("request body is too large to validate"); + if (depth > DiscoveryCaps::kMaxSchemaDepth) + bad("request body is nested deeper than " + std::to_string(DiscoveryCaps::kMaxSchemaDepth)); + const Json* schema = &schema_in; + for (int hops = 0; schema->find("$ref"); ++hops) { + if (hops > 16) bad("schema $ref chain too long at " + at); + const Json* next = api_.schema(sval(*schema, "$ref")); + if (!next) bad("unknown schema " + sval(*schema, "$ref") + " at " + at); + schema = next; + } + if (value.is_null()) return; // proto JSON: null is the unset value + std::string type = sval(*schema, "type"); + if (type.empty() && schema->find("properties")) type = "object"; + std::string format = sval(*schema, "format"); + if (type == "any" || type.empty()) return; + if (type == "string") { + std::string text; + if (value.is_string()) text = value.as_string(); + else if (value.is_number() && (format == "int64" || format == "uint64") && integral(value.as_number())) + text = int_text(value.as_number()); + else bad(at + " must be a string"); + if ((format == "int64" && !all_digits(text, true)) || (format == "uint64" && !all_digits(text, false))) + bad(at + " must be a " + format + " number"); + if (const Json* e = schema->find("enum"); e && e->is_array()) { + bool hit = false; + for (const Json& v : e->as_array()) hit = hit || (v.is_string() && v.as_string() == text); + if (!hit) bad(at + " is not one of its enum values: " + text); + } + } else if (type == "integer") { + if (!value.is_number() || !integral(value.as_number())) bad(at + " must be an integer"); + double d = value.as_number(); + if (format == "int32" && (d < -2147483648.0 || d > 2147483647.0)) bad(at + " is outside int32"); + if (format == "uint32" && (d < 0 || d > 4294967295.0)) bad(at + " is outside uint32"); + } else if (type == "number") { + if (!value.is_number()) bad(at + " must be a number"); + } else if (type == "boolean") { + if (!value.is_bool()) bad(at + " must be true or false"); + } else if (type == "array") { + if (!value.is_array()) bad(at + " must be an array"); + const Json* items = schema->find("items"); + std::size_t i = 0; + for (const Json& v : value.as_array()) { + if (items && items->is_object()) node(*items, v, at + "/" + std::to_string(i), walk, depth + 1); + ++i; + } + } else if (type == "object") { + if (!value.is_object()) bad(at + " must be an object"); + const Json* props = schema->find("properties"); + const Json* extra = schema->find("additionalProperties"); + bool has_props = props && props->is_object() && !props->as_object().empty(); + bool has_extra = extra && extra->is_object(); + for (const auto& [k, v] : value.as_object()) { + const Json* ps = has_props ? props->find(k) : nullptr; + if (ps) node(*ps, v, at + "/" + k, walk, depth + 1); + else if (has_extra) node(*extra, v, at + "/" + k, walk, depth + 1); + else if (has_props) bad(at + " has no property " + k); + // neither: free-form object, anything goes + } + } else { + bad(at + " has unsupported schema type " + type); + } +} + +} // namespace molo::google::discovery diff --git a/src/google/discovery/validate.h b/src/google/discovery/validate.h new file mode 100644 index 0000000..be9d0f9 --- /dev/null +++ b/src/google/discovery/validate.h @@ -0,0 +1,38 @@ +#pragma once + +// Validation before traffic: parameter values against their Discovery +// declaration, request bodies against the document's schemas. Every failure +// throws GoogleError::invalid naming the parameter or JSON path. + +#include +#include + +#include "clients/body/json.h" +#include "google/discovery/model.h" + +namespace molo::google::discovery { + +class ParamValidator { + public: + // Canonical string form(s) of a value: one per element for repeated + // parameters. Checks type, format shape, enum, pattern, minimum, maximum, + // length and the repeated cap. + static std::vector values(const Param& param, + const clients::body::Json& value); +}; + +class BodyValidator { + public: + explicit BodyValidator(const Api& api) : api_(api) {} + // Validates `body` against the schema named `ref`. Unknown properties, + // wrong types, values outside an enum, depth and size overruns throw. + void validate(const std::string& ref, const clients::body::Json& body) const; + + private: + struct Walk; + void node(const clients::body::Json& schema, const clients::body::Json& value, + const std::string& at, Walk& walk, std::size_t depth) const; + const Api& api_; +}; + +} // namespace molo::google::discovery diff --git a/tests/example/google/discovery/page_test.cpp b/tests/example/google/discovery/page_test.cpp new file mode 100644 index 0000000..c5a4e51 --- /dev/null +++ b/tests/example/google/discovery/page_test.cpp @@ -0,0 +1,36 @@ +// Routes: GET /google/discovery (page) and GET /google/discovery/status. +// Both answer without a credential and never touch the network. +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +int main() { + { + GoogleRig rig(false); + Fixture f(rig.client); + auto page = f.client->get("/google/discovery"); + check(page.status == 200, "page 200"); + check(contains(page.headers.get("Content-Type"), "text/html"), "html"); + check(contains(page.body, "Google Discovery") && contains(page.body, "viewport"), "page and mobile viewport"); + check(contains(page.body, "/embedded/google_discovery.js"), "script wired"); + check(f.client->post("/google/discovery", "{}", "application/json").status == 405, "post to the page is 405"); + + auto st = f.client->get("/google/discovery/status"); + check(st.status == 200, "status 200"); + check(contains(st.headers.get("Content-Type"), "application/json"), "json"); + check(contains(st.body, "\"configured\":false"), "not configured"); + check(contains(st.body, "\"max_pages\":50") && contains(st.body, "\"max_items\":10000"), "caps shown"); + check(contains(st.body, "https://discovery.googleapis.com/discovery/v1/apis"), "directory url shown"); + check(f.client->post("/google/discovery/status", "{}", "application/json").status == 405, "post to status is 405"); + check(rig.mock->send_count() == 0, "no upstream traffic"); + } + { + GoogleRig rig(true); + Fixture f(rig.client); + auto st = f.client->get("/google/discovery/status"); + check(contains(st.body, "\"configured\":true"), "configured with a key"); + check(!contains(st.body, kTestKey), "credential never shown"); + check(rig.mock->send_count() == 0, "status makes no upstream call"); + } + return finish("google/discovery/page"); +} diff --git a/tests/example/google/discovery/routes_test.cpp b/tests/example/google/discovery/routes_test.cpp new file mode 100644 index 0000000..78b21dd --- /dev/null +++ b/tests/example/google/discovery/routes_test.cpp @@ -0,0 +1,69 @@ +// Routes: POST /google/discovery/{apis,methods,describe,plan,call} +// Bad input is 422 (with or without a credential) before anything else; a +// well formed request without a credential is 503 google_not_configured; +// GET is 405; nothing reaches Google in any of these cases. +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +namespace { +struct Route { + const char* path; + const char* valid; + const char* bad; +}; + +const Route kRoutes[] = { + {"/google/discovery/apis", "{\"filter\":\"bigquery\",\"limit\":10}", "{\"limit\":0}"}, + {"/google/discovery/methods", "{\"api\":\"bigquery:v2\",\"filter\":\"jobs\"}", "{}"}, + {"/google/discovery/describe", "{\"api\":\"bigquery\",\"method\":\"bigquery.jobs.query\"}", + "{\"api\":\"bigquery\"}"}, + {"/google/discovery/plan", + "{\"api\":\"serviceusage\",\"method\":\"serviceusage.services.list\",\"params\":{\"parent\":\"projects/p\"}}", + "{\"api\":\"serviceusage\",\"method\":\"serviceusage.services.list\",\"params\":[1]}"}, + {"/google/discovery/call", + "{\"api\":\"serviceusage\",\"method\":\"serviceusage.services.list\",\"params\":{\"parent\":\"projects/p\"},\"maxPages\":2}", + "{\"api\":\"serviceusage\",\"method\":\"serviceusage.services.list\",\"maxPages\":51}"}, +}; +} // namespace + +int main() { + GoogleRig rig(false); + Fixture f(rig.client); + for (const Route& r : kRoutes) { + std::string p = r.path; + auto ok = f.client->post(r.path, r.valid, "application/json"); + check(ok.status == 503 && contains(ok.body, "google_not_configured"), p + ": no credential is 503"); + check(f.client->post(r.path, r.bad, "application/json").status == 422, p + ": bad input is 422"); + check(f.client->post(r.path, "not json", "application/json").status == 422, p + ": invalid json is 422"); + check(f.client->post(r.path, "[]", "application/json").status == 422, p + ": non-object body is 422"); + check(f.client->get(r.path).status == 405, p + ": get is 405"); + } + // Shape rules shared by the method routes. + const char* methods[] = {"/google/discovery/describe", "/google/discovery/plan", "/google/discovery/call"}; + for (const char* path : methods) { + std::string p = path; + check(f.client->post(path, "{\"api\":\"../x\",\"method\":\"a.b\"}", "application/json").status == 422, p + ": path-like api is 422"); + check(f.client->post(path, "{\"api\":\"a:b:c\",\"method\":\"a.b\"}", "application/json").status == 422, p + ": two colons is 422"); + check(f.client->post(path, "{\"api\":\"a\",\"method\":\"a b\"}", "application/json").status == 422, p + ": spaced method is 422"); + check(f.client->post(path, "{\"api\":5,\"method\":\"a.b\"}", "application/json").status == 422, p + ": non-string api is 422"); + } + const char* callers[] = {"/google/discovery/plan", "/google/discovery/call"}; + for (const char* path : callers) { + std::string p = path; + std::string head = "{\"api\":\"tasks\",\"method\":\"tasks.tasks.list\","; + check(f.client->post(path, head + "\"auth\":\"magic\"}", "application/json").status == 422, p + ": unknown auth is 422"); + check(f.client->post(path, head + "\"body\":[1]}", "application/json").status == 422, p + ": array body is 422"); + check(f.client->post(path, head + "\"wait\":\"yes\"}", "application/json").status == 422, p + ": non-bool wait is 422"); + check(f.client->post(path, head + "\"maxPages\":\"3\"}", "application/json").status == 422, p + ": text maxPages is 422"); + } + check(f.client->post("/google/discovery/call", + "{\"api\":\"tasks\",\"method\":\"tasks.tasks.list\",\"confirm\":\"yes\"}", + "application/json").status == 422, + "non-bool confirm is 422"); + std::string big(600 * 1024, 'a'); + check(f.client->post("/google/discovery/plan", "{\"x\":\"" + big + "\"}", "application/json").status == 413, + "oversized body is 413"); + check(rig.mock->send_count() == 0, "nothing reached Google"); + return finish("google/discovery/routes"); +} diff --git a/tests/src/google/discovery/fixtures/abusiveexperiencereport_v1.json b/tests/src/google/discovery/fixtures/abusiveexperiencereport_v1.json new file mode 100644 index 0000000..1362f7a --- /dev/null +++ b/tests/src/google/discovery/fixtures/abusiveexperiencereport_v1.json @@ -0,0 +1,225 @@ +{ + "rootUrl": "https://abusiveexperiencereport.googleapis.com/", + "baseUrl": "https://abusiveexperiencereport.googleapis.com/", + "servicePath": "", + "resources": { + "sites": { + "methods": { + "get": { + "httpMethod": "GET", + "parameters": { + "name": { + "pattern": "^sites/[^/]+$", + "required": true, + "location": "path", + "description": "Required. The name of the site whose summary to get, e.g. `sites/http%3A%2F%2Fwww.google.com%2F`. Format: `sites/{site}`", + "type": "string" + } + }, + "id": "abusiveexperiencereport.sites.get", + "path": "v1/{+name}", + "flatPath": "v1/sites/{sitesId}", + "description": "Gets a site's Abusive Experience Report summary.", + "parameterOrder": [ + "name" + ], + "response": { + "$ref": "SiteSummaryResponse" + } + } + } + }, + "violatingSites": { + "methods": { + "list": { + "httpMethod": "GET", + "parameters": {}, + "parameterOrder": [], + "flatPath": "v1/violatingSites", + "path": "v1/violatingSites", + "description": "Lists sites that are failing in the Abusive Experience Report.", + "response": { + "$ref": "ViolatingSitesResponse" + }, + "id": "abusiveexperiencereport.violatingSites.list" + } + } + } + }, + "kind": "discovery#restDescription", + "ownerDomain": "google.com", + "batchPath": "batch", + "mtlsRootUrl": "https://abusiveexperiencereport.mtls.googleapis.com/", + "parameters": { + "quotaUser": { + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.", + "location": "query", + "type": "string" + }, + "$.xgafv": { + "enum": [ + "1", + "2" + ], + "type": "string", + "location": "query", + "description": "V1 error format.", + "enumDescriptions": [ + "v1 error format", + "v2 error format" + ] + }, + "callback": { + "location": "query", + "description": "JSONP", + "type": "string" + }, + "alt": { + "enum": [ + "json", + "media", + "proto" + ], + "enumDescriptions": [ + "Responses with Content-Type of application/json", + "Media download with context-dependent Content-Type", + "Responses with Content-Type of application/x-protobuf" + ], + "type": "string", + "default": "json", + "location": "query", + "description": "Data format for response." + }, + "key": { + "location": "query", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "type": "string" + }, + "prettyPrint": { + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query", + "type": "boolean" + }, + "oauth_token": { + "location": "query", + "description": "OAuth 2.0 token for the current user.", + "type": "string" + }, + "fields": { + "type": "string", + "location": "query", + "description": "Selector specifying which fields to include in a partial response." + }, + "upload_protocol": { + "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").", + "type": "string", + "location": "query" + }, + "uploadType": { + "location": "query", + "type": "string", + "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\")." + }, + "access_token": { + "description": "OAuth access token.", + "type": "string", + "location": "query" + } + }, + "description": "Views Abusive Experience Report data, and gets a list of sites that have a significant number of abusive experiences.", + "version": "v1", + "icons": { + "x32": "http://www.google.com/images/icons/product/search-32.gif", + "x16": "http://www.google.com/images/icons/product/search-16.gif" + }, + "title": "Abusive Experience Report API", + "version_module": true, + "canonicalName": "Abusive Experience Report", + "id": "abusiveexperiencereport:v1", + "schemas": { + "SiteSummaryResponse": { + "type": "object", + "description": "Response message for GetSiteSummary.", + "properties": { + "reviewedSite": { + "type": "string", + "description": "The name of the reviewed site, e.g. `google.com`." + }, + "lastChangeTime": { + "description": "The time at which the site's status last changed.", + "format": "google-datetime", + "type": "string" + }, + "underReview": { + "description": "Whether the site is currently under review.", + "type": "boolean" + }, + "filterStatus": { + "type": "string", + "description": "The site's [enforcement status](https://support.google.com/webtools/answer/7538608).", + "enumDescriptions": [ + "N/A.", + "Enforcement is on.", + "Enforcement is off.", + "Enforcement is paused.", + "Enforcement is pending." + ], + "enum": [ + "UNKNOWN", + "ON", + "OFF", + "PAUSED", + "PENDING" + ] + }, + "reportUrl": { + "type": "string", + "description": "A link to the full Abusive Experience Report for the site. Not set in ViolatingSitesResponse. Note that you must complete the [Search Console verification process](https://support.google.com/webmasters/answer/9008080) for the site before you can access the full report." + }, + "enforcementTime": { + "description": "The time at which [enforcement](https://support.google.com/webtools/answer/7538608) against the site began or will begin. Not set when the filter_status is OFF.", + "type": "string", + "format": "google-datetime" + }, + "abusiveStatus": { + "description": "The site's Abusive Experience Report status.", + "enum": [ + "UNKNOWN", + "PASSING", + "FAILING" + ], + "type": "string", + "enumDescriptions": [ + "Not reviewed.", + "Passing.", + "Failing." + ] + } + }, + "id": "SiteSummaryResponse" + }, + "ViolatingSitesResponse": { + "properties": { + "violatingSites": { + "description": "The list of violating sites.", + "items": { + "$ref": "SiteSummaryResponse" + }, + "type": "array" + } + }, + "type": "object", + "description": "Response message for ListViolatingSites.", + "id": "ViolatingSitesResponse" + } + }, + "fullyEncodeReservedExpansion": true, + "basePath": "", + "ownerName": "Google", + "protocol": "rest", + "discoveryVersion": "v1", + "revision": "20261021", + "documentationLink": "https://developers.google.com/abusive-experience-report/", + "name": "abusiveexperiencereport" +} diff --git a/tests/src/google/discovery/fixtures/admob_v1.json b/tests/src/google/discovery/fixtures/admob_v1.json new file mode 100644 index 0000000..4178899 --- /dev/null +++ b/tests/src/google/discovery/fixtures/admob_v1.json @@ -0,0 +1,1257 @@ +{ + "description": "The AdMob API allows publishers to programmatically get information about their AdMob account. ", + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/admob.report": { + "description": "See your AdMob data" + }, + "https://www.googleapis.com/auth/admob.readonly": { + "description": "See your AdMob data" + } + } + } + }, + "fullyEncodeReservedExpansion": true, + "revision": "20261022", + "kind": "discovery#restDescription", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "ownerDomain": "google.com", + "version": "v1", + "batchPath": "batch", + "schemas": { + "MediationReportSpecDimensionFilter": { + "type": "object", + "properties": { + "dimension": { + "description": "Applies the filter criterion to the specified dimension.", + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "A date in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "A month in the YYYYMM format (for example, \"202107\"). Requests can specify at most one time dimension.", + "The date of the first day of a week in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "The [unique ID of the ad source](/admob/api/v1/ad_sources) (for example, \"5450213213286189855\" and \"AdMob Network\" as label value).", + "The unique ID of the ad source instance (for example, \"ca-app-pub-1234:asi:5678\" and \"AdMob (default)\" as label value).", + "The unique ID of the ad unit (for example, \"ca-app-pub-1234/8790\"). If AD_UNIT dimension is specified, then APP is included automatically.", + "The unique ID of the mobile application (for example, \"ca-app-pub-1234~1234\").", + "The unique ID of the mediation group (for example, \"ca-app-pub-1234:mg:1234\" and \"AdMob (default)\" as label value).", + "CLDR country code of the place where the ad views/clicks occur (for example, \"US\" or \"FR\"). This is a geography dimension.", + "Format of the ad unit (for example, \"banner\", \"native\"), an ad delivery dimension.", + "Mobile OS platform of the app (for example, \"Android\" or \"iOS\").", + "Mobile operating system version, e.g. \"iOS 13.5.1\".", + "GMA SDK version, e.g. \"iOS 7.62.0\".", + "For Android, the app version name can be found in versionName in PackageInfo. For iOS, the app version name can be found in CFBundleShortVersionString.", + "Restriction mode for ads serving (e.g. \"Non-personalized ads\")." + ], + "type": "string", + "enum": [ + "DIMENSION_UNSPECIFIED", + "DATE", + "MONTH", + "WEEK", + "AD_SOURCE", + "AD_SOURCE_INSTANCE", + "AD_UNIT", + "APP", + "MEDIATION_GROUP", + "COUNTRY", + "FORMAT", + "PLATFORM", + "MOBILE_OS_VERSION", + "GMA_SDK_VERSION", + "APP_VERSION_NAME", + "SERVING_RESTRICTION" + ] + }, + "matchesAny": { + "description": "Matches a row if its value for the specified dimension is in one of the values specified in this condition.", + "$ref": "StringList" + } + }, + "id": "MediationReportSpecDimensionFilter", + "description": "Describes which report rows to match based on their dimension values." + }, + "DateRange": { + "properties": { + "endDate": { + "description": "End date of the date range, inclusive. Must be greater than or equal to the start date.", + "$ref": "Date" + }, + "startDate": { + "$ref": "Date", + "description": "Start date of the date range, inclusive. Must be less than or equal to the end date." + } + }, + "description": "Specification of a single date range. Both dates are inclusive.", + "id": "DateRange", + "type": "object" + }, + "GenerateNetworkReportRequest": { + "id": "GenerateNetworkReportRequest", + "properties": { + "reportSpec": { + "$ref": "NetworkReportSpec", + "description": "Network report specification." + } + }, + "description": "Request to generate an AdMob Network report.", + "type": "object" + }, + "ListPublisherAccountsResponse": { + "description": "Response for the publisher account list request.", + "type": "object", + "id": "ListPublisherAccountsResponse", + "properties": { + "account": { + "type": "array", + "items": { + "$ref": "PublisherAccount" + }, + "description": "Publisher that the client credentials can access." + }, + "nextPageToken": { + "description": "If not empty, indicates that there might be more accounts for the request; you must pass this value in a new `ListPublisherAccountsRequest`.", + "type": "string" + } + } + }, + "ReportHeader": { + "properties": { + "dateRange": { + "$ref": "DateRange", + "description": "The date range for which the report is generated. This is identical to the range specified in the report request." + }, + "reportingTimeZone": { + "description": "The report time zone. The value is a time-zone ID as specified by the CLDR project, for example, \"America/Los_Angeles\".", + "type": "string" + }, + "localizationSettings": { + "description": "Localization settings of the report. This is identical to the settings in the report request.", + "$ref": "LocalizationSettings" + } + }, + "id": "ReportHeader", + "description": "Groups data helps to treat the generated report. Always sent as a first message in the stream response.", + "type": "object" + }, + "GenerateMediationReportRequest": { + "properties": { + "reportSpec": { + "$ref": "MediationReportSpec", + "description": "Network report specification." + } + }, + "description": "Request to generate an AdMob Mediation report.", + "id": "GenerateMediationReportRequest", + "type": "object" + }, + "NetworkReportSpec": { + "description": "The specification for generating an AdMob Network report. For example, the specification to get clicks and estimated earnings for only the 'US' and 'CN' countries can look like the following example: { 'date_range': { 'start_date': {'year': 2021, 'month': 9, 'day': 1}, 'end_date': {'year': 2021, 'month': 9, 'day': 30} }, 'dimensions': ['DATE', 'APP', 'COUNTRY'], 'metrics': ['CLICKS', 'ESTIMATED_EARNINGS'], 'dimension_filters': [ { 'dimension': 'COUNTRY', 'matches_any': {'values': [{'value': 'US', 'value': 'CN'}]} } ], 'sort_conditions': [ {'dimension':'APP', order: 'ASCENDING'}, {'metric':'CLICKS', order: 'DESCENDING'} ], 'localization_settings': { 'currency_code': 'USD', 'language_code': 'en-US' } } For a better understanding, you can treat the preceding specification like the following pseudo SQL: SELECT DATE, APP, COUNTRY, CLICKS, ESTIMATED_EARNINGS FROM NETWORK_REPORT WHERE DATE \u003e= '2021-09-01' AND DATE \u003c= '2021-09-30' AND COUNTRY IN ('US', 'CN') GROUP BY DATE, APP, COUNTRY ORDER BY APP ASC, CLICKS DESC;", + "type": "object", + "id": "NetworkReportSpec", + "properties": { + "timeZone": { + "type": "string", + "description": "A report time zone. Accepts an IANA TZ name values, such as \"America/Los_Angeles.\" If no time zone is defined, the account default takes effect. Check default value by the get account action. **Warning:** The \"America/Los_Angeles\" is the only supported value at the moment." + }, + "maxReportRows": { + "description": "Maximum number of report data rows to return. If the value is not set, the API returns as many rows as possible, up to 100000. Acceptable values are 1-100000, inclusive. Values larger than 100000 return an error.", + "type": "integer", + "format": "int32" + }, + "dateRange": { + "$ref": "DateRange", + "description": "The date range for which the report is generated." + }, + "localizationSettings": { + "$ref": "LocalizationSettings", + "description": "Localization settings of the report." + }, + "metrics": { + "items": { + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "The number of ad requests. The value is an integer. **Warning:** The metric is incompatible with [AD_TYPE](#Dimension.ENUM_VALUES.AD_TYPE) dimension.", + "The number of times a user clicks an ad. The value is an integer.", + "The estimated earnings of the AdMob publisher. The currency unit (USD, EUR, or other) of the earning metrics are determined by the localization setting for currency. The amount is in micros. For example, $6.50 would be represented as 6500000.", + "The total number of ads shown to users. The value is an integer.", + "The ratio of clicks over impressions. The value is a double precision (approximate) decimal value.", + "The estimated earnings per thousand ad impressions. The value is in micros. For example, $1.03 would be represented as 1030000. Equivalent to eCPM in the AdMob UI. **Warning:** The metric is incompatible with [AD_TYPE](#Dimension.ENUM_VALUES.AD_TYPE) dimension.", + "The number of times ads are returned in response to a request. The value is an integer.", + "The ratio of matched ad requests over the total ad requests. The value is a double precision (approximate) decimal value. **Warning:** The metric is incompatible with [AD_TYPE](#Dimension.ENUM_VALUES.AD_TYPE) dimension.", + "The ratio of ads that are displayed over ads that are returned, defined as impressions / matched requests. The value is a double precision (approximate) decimal value." + ], + "type": "string", + "enum": [ + "METRIC_UNSPECIFIED", + "AD_REQUESTS", + "CLICKS", + "ESTIMATED_EARNINGS", + "IMPRESSIONS", + "IMPRESSION_CTR", + "IMPRESSION_RPM", + "MATCHED_REQUESTS", + "MATCH_RATE", + "SHOW_RATE" + ] + }, + "type": "array", + "description": "List of metrics of the report. A report must specify at least one metric." + }, + "dimensionFilters": { + "description": "Describes which report rows to match based on their dimension values.", + "items": { + "$ref": "NetworkReportSpecDimensionFilter" + }, + "type": "array" + }, + "dimensions": { + "items": { + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "A date in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "A month in the YYYYMM format (for example, \"202107\"). Requests can specify at most one time dimension.", + "The date of the first day of a week in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "The unique ID of the ad unit (for example, \"ca-app-pub-1234/1234\"). If AD_UNIT dimension is specified, then APP is included automatically.", + "The unique ID of the mobile application (for example, \"ca-app-pub-1234~1234\").", + "Type of the ad (for example, \"text\" or \"image\"), an ad delivery dimension. **Warning:** The dimension is incompatible with [AD_REQUESTS](#Metric.ENUM_VALUES.AD_REQUESTS), [MATCH_RATE](#Metric.ENUM_VALUES.MATCH_RATE) and [IMPRESSION_RPM](#Metric.ENUM_VALUES.IMPRESSION_RPM) metrics.", + "CLDR country code of the place where the ad views/clicks occur (for example, \"US\" or \"FR\"). This is a geography dimension.", + "Format of the ad unit (for example, \"banner\", \"native\"), an ad delivery dimension.", + "Mobile OS platform of the app (for example, \"Android\" or \"iOS\").", + "Mobile operating system version, e.g. \"iOS 13.5.1\".", + "GMA SDK version, e.g. \"iOS 7.62.0\".", + "For Android, the app version name can be found in versionName in PackageInfo. For iOS, the app version name can be found in CFBundleShortVersionString.", + "Restriction mode for ads serving (e.g. \"Non-personalized ads\")." + ], + "type": "string", + "enum": [ + "DIMENSION_UNSPECIFIED", + "DATE", + "MONTH", + "WEEK", + "AD_UNIT", + "APP", + "AD_TYPE", + "COUNTRY", + "FORMAT", + "PLATFORM", + "MOBILE_OS_VERSION", + "GMA_SDK_VERSION", + "APP_VERSION_NAME", + "SERVING_RESTRICTION" + ] + }, + "type": "array", + "description": "List of dimensions of the report. The value combination of these dimensions determines the row of the report. If no dimensions are specified, the report returns a single row of requested metrics for the entire account." + }, + "sortConditions": { + "type": "array", + "items": { + "$ref": "NetworkReportSpecSortCondition" + }, + "description": "Describes the sorting of report rows. The order of the condition in the list defines its precedence; the earlier the condition, the higher its precedence. If no sort conditions are specified, the row ordering is undefined." + } + } + }, + "ReportRowMetricValue": { + "properties": { + "microsValue": { + "description": "Amount in micros. One million is equivalent to one unit. Currency value is in the unit (USD, EUR or other) specified by the request. For example, $6.50 whould be represented as 6500000 micros.", + "type": "string", + "format": "int64" + }, + "doubleValue": { + "type": "number", + "description": "Double precision (approximate) decimal values. Rates are from 0 to 1.", + "format": "double" + }, + "integerValue": { + "format": "int64", + "description": "Metric integer value.", + "type": "string" + } + }, + "type": "object", + "id": "ReportRowMetricValue", + "description": "Representation of a metric value." + }, + "AppLinkedAppInfo": { + "properties": { + "appStoreId": { + "type": "string", + "description": "The app store ID of the app; present if and only if the app is linked to an app store. If the app is added to the Google Play store, it will be the application ID of the app. For example: \"com.example.myapp\". See https://developer.android.com/studio/build/application-id. If the app is added to the Apple App Store, it will be app store ID. For example \"105169111\". Note that setting the app store id is considered an irreversible action. Once an app is linked, it cannot be unlinked." + }, + "displayName": { + "type": "string", + "description": "Output only. Display name of the app as it appears in the app store. This is an output-only field, and may be empty if the app cannot be found in the store.", + "readOnly": true + } + }, + "id": "AppLinkedAppInfo", + "type": "object", + "description": "Information from the app store if the app is linked to an app store." + }, + "MediationReportSpecSortCondition": { + "type": "object", + "description": "Sorting direction to be applied on a dimension or a metric.", + "id": "MediationReportSpecSortCondition", + "properties": { + "metric": { + "description": "Sort by the specified metric.", + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "The number of requests. The value is an integer.", + "The number of times a user clicks an ad. The value is an integer.", + "The estimated earnings of the AdMob publisher. The currency unit (USD, EUR, or other) of the earning metrics are determined by the localization setting for currency. The amount is in micros. For example, $6.50 would be represented as 6500000. Estimated earnings per mediation group and per ad source instance level is supported dating back to October 20, 2019. Third-party estimated earnings will show 0 for dates prior to October 20, 2019.", + "The total number of ads shown to users. The value is an integer.", + "The ratio of clicks over impressions. The value is a double precision (approximate) decimal value.", + "The number of times ads are returned in response to a request. The value is an integer.", + "The ratio of matched ad requests over the total ad requests. The value is a double precision (approximate) decimal value.", + "The third-party ad network's estimated average eCPM. The currency unit (USD, EUR, or other) of the earning metrics are determined by the localization setting for currency. The amount is in micros. For example, $2.30 would be represented as 2300000. The estimated average eCPM per mediation group and per ad source instance level is supported dating back to October 20, 2019. Third-party estimated average eCPM will show 0 for dates prior to October 20, 2019.", + "The ad load latency is the total time it takes for an ad to load measured in seconds. It starts when the ad is requested and ends when a response is received. See https://support.google.com/admob/answer/12571120 for additional information." + ], + "type": "string", + "enum": [ + "METRIC_UNSPECIFIED", + "AD_REQUESTS", + "CLICKS", + "ESTIMATED_EARNINGS", + "IMPRESSIONS", + "IMPRESSION_CTR", + "MATCHED_REQUESTS", + "MATCH_RATE", + "OBSERVED_ECPM", + "AD_LOAD_LATENCY" + ] + }, + "dimension": { + "type": "string", + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "A date in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "A month in the YYYYMM format (for example, \"202107\"). Requests can specify at most one time dimension.", + "The date of the first day of a week in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "The [unique ID of the ad source](/admob/api/v1/ad_sources) (for example, \"5450213213286189855\" and \"AdMob Network\" as label value).", + "The unique ID of the ad source instance (for example, \"ca-app-pub-1234:asi:5678\" and \"AdMob (default)\" as label value).", + "The unique ID of the ad unit (for example, \"ca-app-pub-1234/8790\"). If AD_UNIT dimension is specified, then APP is included automatically.", + "The unique ID of the mobile application (for example, \"ca-app-pub-1234~1234\").", + "The unique ID of the mediation group (for example, \"ca-app-pub-1234:mg:1234\" and \"AdMob (default)\" as label value).", + "CLDR country code of the place where the ad views/clicks occur (for example, \"US\" or \"FR\"). This is a geography dimension.", + "Format of the ad unit (for example, \"banner\", \"native\"), an ad delivery dimension.", + "Mobile OS platform of the app (for example, \"Android\" or \"iOS\").", + "Mobile operating system version, e.g. \"iOS 13.5.1\".", + "GMA SDK version, e.g. \"iOS 7.62.0\".", + "For Android, the app version name can be found in versionName in PackageInfo. For iOS, the app version name can be found in CFBundleShortVersionString.", + "Restriction mode for ads serving (e.g. \"Non-personalized ads\")." + ], + "enum": [ + "DIMENSION_UNSPECIFIED", + "DATE", + "MONTH", + "WEEK", + "AD_SOURCE", + "AD_SOURCE_INSTANCE", + "AD_UNIT", + "APP", + "MEDIATION_GROUP", + "COUNTRY", + "FORMAT", + "PLATFORM", + "MOBILE_OS_VERSION", + "GMA_SDK_VERSION", + "APP_VERSION_NAME", + "SERVING_RESTRICTION" + ], + "description": "Sort by the specified dimension." + }, + "order": { + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "Sort dimension value or metric value in ascending order.", + "Sort dimension value or metric value in descending order." + ], + "description": "Sorting order of the dimension or metric.", + "type": "string", + "enum": [ + "SORT_ORDER_UNSPECIFIED", + "ASCENDING", + "DESCENDING" + ] + } + } + }, + "ReportRowDimensionValue": { + "description": "Representation of a dimension value.", + "type": "object", + "id": "ReportRowDimensionValue", + "properties": { + "value": { + "type": "string", + "description": "Dimension value in the format specified in the report's spec Dimension enum." + }, + "displayLabel": { + "description": "The localized string representation of the value. If unspecified, the display label should be derived from the value.", + "type": "string" + } + } + }, + "AppManualAppInfo": { + "properties": { + "displayName": { + "description": "The display name of the app as shown in the AdMob UI, which is provided by the user. The maximum length allowed is 80 characters.", + "type": "string" + } + }, + "type": "object", + "id": "AppManualAppInfo", + "description": "Information provided for manual apps which are not linked to an application store (Example: Google Play, App Store)." + }, + "StringList": { + "id": "StringList", + "properties": { + "values": { + "items": { + "type": "string" + }, + "description": "The string values.", + "type": "array" + } + }, + "description": "List of string values.", + "type": "object" + }, + "Date": { + "description": "Represents a whole or partial calendar date, such as a birthday. The time of day and time zone are either specified elsewhere or are insignificant. The date is relative to the Gregorian Calendar. This can represent one of the following: * A full date, with non-zero year, month, and day values. * A month and day, with a zero year (for example, an anniversary). * A year on its own, with a zero month and a zero day. * A year and month, with a zero day (for example, a credit card expiration date). Related types: * google.type.TimeOfDay * google.type.DateTime * google.protobuf.Timestamp", + "id": "Date", + "type": "object", + "properties": { + "year": { + "format": "int32", + "type": "integer", + "description": "Year of the date. Must be from 1 to 9999, or 0 to specify a date without a year." + }, + "day": { + "type": "integer", + "description": "Day of a month. Must be from 1 to 31 and valid for the year and month, or 0 to specify a year by itself or a year and month where the day isn't significant.", + "format": "int32" + }, + "month": { + "format": "int32", + "type": "integer", + "description": "Month of a year. Must be from 1 to 12, or 0 to specify a year without a month and day." + } + } + }, + "ListAdUnitsResponse": { + "type": "object", + "description": "Response for the ad units list request.", + "id": "ListAdUnitsResponse", + "properties": { + "adUnits": { + "description": "The resulting ad units for the requested account.", + "type": "array", + "items": { + "$ref": "AdUnit" + } + }, + "nextPageToken": { + "type": "string", + "description": "If not empty, indicates that there may be more ad units for the request; this value should be passed in a new `ListAdUnitsRequest`." + } + } + }, + "ListAppsResponse": { + "id": "ListAppsResponse", + "properties": { + "apps": { + "type": "array", + "items": { + "$ref": "App" + }, + "description": "The resulting apps for the requested account." + }, + "nextPageToken": { + "type": "string", + "description": "If not empty, indicates that there may be more apps for the request; this value should be passed in a new `ListAppsRequest`." + } + }, + "type": "object", + "description": "Response for the apps list request." + }, + "MediationReportSpec": { + "description": "The specification for generating an AdMob Mediation report. For example, the specification to get observed ECPM sliced by ad source and app for the 'US' and 'CN' countries can look like the following example: { \"date_range\": { \"start_date\": {\"year\": 2021, \"month\": 9, \"day\": 1}, \"end_date\": {\"year\": 2021, \"month\": 9, \"day\": 30} }, \"dimensions\": [\"AD_SOURCE\", \"APP\", \"COUNTRY\"], \"metrics\": [\"OBSERVED_ECPM\"], \"dimension_filters\": [ { \"dimension\": \"COUNTRY\", \"matches_any\": {\"values\": [{\"value\": \"US\", \"value\": \"CN\"}]} } ], \"sort_conditions\": [ {\"dimension\":\"APP\", order: \"ASCENDING\"} ], \"localization_settings\": { \"currency_code\": \"USD\", \"language_code\": \"en-US\" } } For a better understanding, you can treat the preceding specification like the following pseudo SQL: SELECT AD_SOURCE, APP, COUNTRY, OBSERVED_ECPM FROM MEDIATION_REPORT WHERE DATE \u003e= '2021-09-01' AND DATE \u003c= '2021-09-30' AND COUNTRY IN ('US', 'CN') GROUP BY AD_SOURCE, APP, COUNTRY ORDER BY APP ASC;", + "id": "MediationReportSpec", + "type": "object", + "properties": { + "maxReportRows": { + "description": "Maximum number of report data rows to return. If the value is not set, the API returns as many rows as possible, up to 100000. Acceptable values are 1-100000, inclusive. Values larger than 100000 return an error.", + "type": "integer", + "format": "int32" + }, + "timeZone": { + "description": "A report time zone. Accepts an IANA TZ name values, such as \"America/Los_Angeles.\" If no time zone is defined, the account default takes effect. Check default value by the get account action. **Warning:** The \"America/Los_Angeles\" is the only supported value at the moment.", + "type": "string" + }, + "metrics": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "METRIC_UNSPECIFIED", + "AD_REQUESTS", + "CLICKS", + "ESTIMATED_EARNINGS", + "IMPRESSIONS", + "IMPRESSION_CTR", + "MATCHED_REQUESTS", + "MATCH_RATE", + "OBSERVED_ECPM", + "AD_LOAD_LATENCY" + ], + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "The number of requests. The value is an integer.", + "The number of times a user clicks an ad. The value is an integer.", + "The estimated earnings of the AdMob publisher. The currency unit (USD, EUR, or other) of the earning metrics are determined by the localization setting for currency. The amount is in micros. For example, $6.50 would be represented as 6500000. Estimated earnings per mediation group and per ad source instance level is supported dating back to October 20, 2019. Third-party estimated earnings will show 0 for dates prior to October 20, 2019.", + "The total number of ads shown to users. The value is an integer.", + "The ratio of clicks over impressions. The value is a double precision (approximate) decimal value.", + "The number of times ads are returned in response to a request. The value is an integer.", + "The ratio of matched ad requests over the total ad requests. The value is a double precision (approximate) decimal value.", + "The third-party ad network's estimated average eCPM. The currency unit (USD, EUR, or other) of the earning metrics are determined by the localization setting for currency. The amount is in micros. For example, $2.30 would be represented as 2300000. The estimated average eCPM per mediation group and per ad source instance level is supported dating back to October 20, 2019. Third-party estimated average eCPM will show 0 for dates prior to October 20, 2019.", + "The ad load latency is the total time it takes for an ad to load measured in seconds. It starts when the ad is requested and ends when a response is received. See https://support.google.com/admob/answer/12571120 for additional information." + ] + }, + "description": "List of metrics of the report. A report must specify at least one metric." + }, + "dimensions": { + "items": { + "type": "string", + "enum": [ + "DIMENSION_UNSPECIFIED", + "DATE", + "MONTH", + "WEEK", + "AD_SOURCE", + "AD_SOURCE_INSTANCE", + "AD_UNIT", + "APP", + "MEDIATION_GROUP", + "COUNTRY", + "FORMAT", + "PLATFORM", + "MOBILE_OS_VERSION", + "GMA_SDK_VERSION", + "APP_VERSION_NAME", + "SERVING_RESTRICTION" + ], + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "A date in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "A month in the YYYYMM format (for example, \"202107\"). Requests can specify at most one time dimension.", + "The date of the first day of a week in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "The [unique ID of the ad source](/admob/api/v1/ad_sources) (for example, \"5450213213286189855\" and \"AdMob Network\" as label value).", + "The unique ID of the ad source instance (for example, \"ca-app-pub-1234:asi:5678\" and \"AdMob (default)\" as label value).", + "The unique ID of the ad unit (for example, \"ca-app-pub-1234/8790\"). If AD_UNIT dimension is specified, then APP is included automatically.", + "The unique ID of the mobile application (for example, \"ca-app-pub-1234~1234\").", + "The unique ID of the mediation group (for example, \"ca-app-pub-1234:mg:1234\" and \"AdMob (default)\" as label value).", + "CLDR country code of the place where the ad views/clicks occur (for example, \"US\" or \"FR\"). This is a geography dimension.", + "Format of the ad unit (for example, \"banner\", \"native\"), an ad delivery dimension.", + "Mobile OS platform of the app (for example, \"Android\" or \"iOS\").", + "Mobile operating system version, e.g. \"iOS 13.5.1\".", + "GMA SDK version, e.g. \"iOS 7.62.0\".", + "For Android, the app version name can be found in versionName in PackageInfo. For iOS, the app version name can be found in CFBundleShortVersionString.", + "Restriction mode for ads serving (e.g. \"Non-personalized ads\")." + ] + }, + "type": "array", + "description": "List of dimensions of the report. The value combination of these dimensions determines the row of the report. If no dimensions are specified, the report returns a single row of requested metrics for the entire account." + }, + "dateRange": { + "description": "The date range for which the report is generated.", + "$ref": "DateRange" + }, + "localizationSettings": { + "$ref": "LocalizationSettings", + "description": "Localization settings of the report." + }, + "dimensionFilters": { + "items": { + "$ref": "MediationReportSpecDimensionFilter" + }, + "description": "Describes which report rows to match based on their dimension values.", + "type": "array" + }, + "sortConditions": { + "items": { + "$ref": "MediationReportSpecSortCondition" + }, + "type": "array", + "description": "Describes the sorting of report rows. The order of the condition in the list defines its precedence; the earlier the condition, the higher its precedence. If no sort conditions are specified, the row ordering is undefined." + } + } + }, + "NetworkReportSpecDimensionFilter": { + "properties": { + "matchesAny": { + "description": "Matches a row if its value for the specified dimension is in one of the values specified in this condition.", + "$ref": "StringList" + }, + "dimension": { + "type": "string", + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "A date in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "A month in the YYYYMM format (for example, \"202107\"). Requests can specify at most one time dimension.", + "The date of the first day of a week in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "The unique ID of the ad unit (for example, \"ca-app-pub-1234/1234\"). If AD_UNIT dimension is specified, then APP is included automatically.", + "The unique ID of the mobile application (for example, \"ca-app-pub-1234~1234\").", + "Type of the ad (for example, \"text\" or \"image\"), an ad delivery dimension. **Warning:** The dimension is incompatible with [AD_REQUESTS](#Metric.ENUM_VALUES.AD_REQUESTS), [MATCH_RATE](#Metric.ENUM_VALUES.MATCH_RATE) and [IMPRESSION_RPM](#Metric.ENUM_VALUES.IMPRESSION_RPM) metrics.", + "CLDR country code of the place where the ad views/clicks occur (for example, \"US\" or \"FR\"). This is a geography dimension.", + "Format of the ad unit (for example, \"banner\", \"native\"), an ad delivery dimension.", + "Mobile OS platform of the app (for example, \"Android\" or \"iOS\").", + "Mobile operating system version, e.g. \"iOS 13.5.1\".", + "GMA SDK version, e.g. \"iOS 7.62.0\".", + "For Android, the app version name can be found in versionName in PackageInfo. For iOS, the app version name can be found in CFBundleShortVersionString.", + "Restriction mode for ads serving (e.g. \"Non-personalized ads\")." + ], + "enum": [ + "DIMENSION_UNSPECIFIED", + "DATE", + "MONTH", + "WEEK", + "AD_UNIT", + "APP", + "AD_TYPE", + "COUNTRY", + "FORMAT", + "PLATFORM", + "MOBILE_OS_VERSION", + "GMA_SDK_VERSION", + "APP_VERSION_NAME", + "SERVING_RESTRICTION" + ], + "description": "Applies the filter criterion to the specified dimension." + } + }, + "description": "Describes which report rows to match based on their dimension values.", + "id": "NetworkReportSpecDimensionFilter", + "type": "object" + }, + "NetworkReportSpecSortCondition": { + "id": "NetworkReportSpecSortCondition", + "type": "object", + "properties": { + "metric": { + "type": "string", + "enum": [ + "METRIC_UNSPECIFIED", + "AD_REQUESTS", + "CLICKS", + "ESTIMATED_EARNINGS", + "IMPRESSIONS", + "IMPRESSION_CTR", + "IMPRESSION_RPM", + "MATCHED_REQUESTS", + "MATCH_RATE", + "SHOW_RATE" + ], + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "The number of ad requests. The value is an integer. **Warning:** The metric is incompatible with [AD_TYPE](#Dimension.ENUM_VALUES.AD_TYPE) dimension.", + "The number of times a user clicks an ad. The value is an integer.", + "The estimated earnings of the AdMob publisher. The currency unit (USD, EUR, or other) of the earning metrics are determined by the localization setting for currency. The amount is in micros. For example, $6.50 would be represented as 6500000.", + "The total number of ads shown to users. The value is an integer.", + "The ratio of clicks over impressions. The value is a double precision (approximate) decimal value.", + "The estimated earnings per thousand ad impressions. The value is in micros. For example, $1.03 would be represented as 1030000. Equivalent to eCPM in the AdMob UI. **Warning:** The metric is incompatible with [AD_TYPE](#Dimension.ENUM_VALUES.AD_TYPE) dimension.", + "The number of times ads are returned in response to a request. The value is an integer.", + "The ratio of matched ad requests over the total ad requests. The value is a double precision (approximate) decimal value. **Warning:** The metric is incompatible with [AD_TYPE](#Dimension.ENUM_VALUES.AD_TYPE) dimension.", + "The ratio of ads that are displayed over ads that are returned, defined as impressions / matched requests. The value is a double precision (approximate) decimal value." + ], + "description": "Sort by the specified metric." + }, + "order": { + "type": "string", + "enum": [ + "SORT_ORDER_UNSPECIFIED", + "ASCENDING", + "DESCENDING" + ], + "description": "Sorting order of the dimension or metric.", + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "Sort dimension value or metric value in ascending order.", + "Sort dimension value or metric value in descending order." + ] + }, + "dimension": { + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "A date in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "A month in the YYYYMM format (for example, \"202107\"). Requests can specify at most one time dimension.", + "The date of the first day of a week in the YYYYMMDD format (for example, \"20210701\"). Requests can specify at most one time dimension.", + "The unique ID of the ad unit (for example, \"ca-app-pub-1234/1234\"). If AD_UNIT dimension is specified, then APP is included automatically.", + "The unique ID of the mobile application (for example, \"ca-app-pub-1234~1234\").", + "Type of the ad (for example, \"text\" or \"image\"), an ad delivery dimension. **Warning:** The dimension is incompatible with [AD_REQUESTS](#Metric.ENUM_VALUES.AD_REQUESTS), [MATCH_RATE](#Metric.ENUM_VALUES.MATCH_RATE) and [IMPRESSION_RPM](#Metric.ENUM_VALUES.IMPRESSION_RPM) metrics.", + "CLDR country code of the place where the ad views/clicks occur (for example, \"US\" or \"FR\"). This is a geography dimension.", + "Format of the ad unit (for example, \"banner\", \"native\"), an ad delivery dimension.", + "Mobile OS platform of the app (for example, \"Android\" or \"iOS\").", + "Mobile operating system version, e.g. \"iOS 13.5.1\".", + "GMA SDK version, e.g. \"iOS 7.62.0\".", + "For Android, the app version name can be found in versionName in PackageInfo. For iOS, the app version name can be found in CFBundleShortVersionString.", + "Restriction mode for ads serving (e.g. \"Non-personalized ads\")." + ], + "description": "Sort by the specified dimension.", + "type": "string", + "enum": [ + "DIMENSION_UNSPECIFIED", + "DATE", + "MONTH", + "WEEK", + "AD_UNIT", + "APP", + "AD_TYPE", + "COUNTRY", + "FORMAT", + "PLATFORM", + "MOBILE_OS_VERSION", + "GMA_SDK_VERSION", + "APP_VERSION_NAME", + "SERVING_RESTRICTION" + ] + } + }, + "description": "Sorting direction to be applied on a dimension or a metric." + }, + "GenerateNetworkReportResponse": { + "id": "GenerateNetworkReportResponse", + "description": "The streaming response for the AdMob Network report where the first response contains the report header, then a stream of row responses, and finally a footer as the last response message. For example: [{ \"header\": { \"dateRange\": { \"startDate\": {\"year\": 2018, \"month\": 9, \"day\": 1}, \"endDate\": {\"year\": 2018, \"month\": 9, \"day\": 1} }, \"localizationSettings\": { \"currencyCode\": \"USD\", \"languageCode\": \"en-US\" } } }, { \"row\": { \"dimensionValues\": { \"DATE\": {\"value\": \"20180918\"}, \"APP\": { \"value\": \"ca-app-pub-8123415297019784~1001342552\", displayLabel: \"My app name!\" } }, \"metricValues\": { \"ESTIMATED_EARNINGS\": {\"microsValue\": 6500000} } } }, { \"footer\": {\"matchingRowCount\": 1} }]", + "type": "object", + "properties": { + "header": { + "description": "Report generation settings that describes the report contents, such as the report date range and localization settings.", + "$ref": "ReportHeader" + }, + "row": { + "$ref": "ReportRow", + "description": "Actual report data." + }, + "footer": { + "description": "Additional information about the generated report, such as warnings about the data.", + "$ref": "ReportFooter" + } + } + }, + "PublisherAccount": { + "description": "A publisher account contains information relevant to the use of this API, such as the time zone used for the reports.", + "id": "PublisherAccount", + "properties": { + "name": { + "type": "string", + "description": "Resource name of this account. Format is accounts/{publisher_id}." + }, + "reportingTimeZone": { + "description": "The time zone that is used in reports that are generated for this account. The value is a time-zone ID as specified by the CLDR project, for example, \"America/Los_Angeles\".", + "type": "string" + }, + "publisherId": { + "description": "The unique ID by which this publisher account can be identified in the API requests (for example, pub-1234567890).", + "type": "string" + }, + "currencyCode": { + "description": "Currency code of the earning-related metrics, which is the 3-letter code defined in ISO 4217. The daily average rate is used for the currency conversion.", + "type": "string" + } + }, + "type": "object" + }, + "GenerateMediationReportResponse": { + "type": "object", + "properties": { + "row": { + "$ref": "ReportRow", + "description": "Actual report data." + }, + "header": { + "description": "Report generation settings that describes the report contents, such as the report date range and localization settings.", + "$ref": "ReportHeader" + }, + "footer": { + "description": "Additional information about the generated report, such as warnings about the data.", + "$ref": "ReportFooter" + } + }, + "description": "The streaming response for the AdMob Mediation report where the first response contains the report header, then a stream of row responses, and finally a footer as the last response message. For example: [{ \"header\": { \"date_range\": { \"start_date\": {\"year\": 2018, \"month\": 9, \"day\": 1}, \"end_date\": {\"year\": 2018, \"month\": 9, \"day\": 1} }, \"localization_settings\": { \"currency_code\": \"USD\", \"language_code\": \"en-US\" } } }, { \"row\": { \"dimension_values\": { \"DATE\": {\"value\": \"20180918\"}, \"APP\": { \"value\": \"ca-app-pub-8123415297019784~1001342552\", \"display_label\": \"My app name!\" } }, \"metric_values\": { \"ESTIMATED_EARNINGS\": {\"decimal_value\": \"1324746\"} } } }, { \"footer\": {\"matching_row_count\": 1} }]", + "id": "GenerateMediationReportResponse" + }, + "AdUnit": { + "type": "object", + "id": "AdUnit", + "description": "Describes an AdMob ad unit.", + "properties": { + "adTypes": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Ad media type supported by this ad unit. Possible values as follows: \"RICH_MEDIA\" - Text, image, and other non-video media. \"VIDEO\" - Video media." + }, + "displayName": { + "type": "string", + "description": "The display name of the ad unit as shown in the AdMob UI, which is provided by the user. The maximum length allowed is 80 characters." + }, + "name": { + "description": "Resource name for this ad unit. Format is accounts/{publisher_id}/adUnits/{ad_unit_id_fragment} Example: accounts/pub-9876543210987654/adUnits/0123456789", + "type": "string" + }, + "adUnitId": { + "description": "The externally visible ID of the ad unit which can be used to integrate with AdMob. This is a read only property. Example: ca-app-pub-9876543210987654/0123456789", + "type": "string" + }, + "adFormat": { + "type": "string", + "description": "AdFormat of the ad unit. Possible values are as follows: \"APP_OPEN\" - App Open ad format. \"BANNER\" - Banner ad format. \"BANNER_INTERSTITIAL\" - Legacy format that can be used as either banner or interstitial. This format can no longer be created but can be targeted by mediation groups. \"INTERSTITIAL\" - A full screen ad. Supported ad types are \"RICH_MEDIA\" and \"VIDEO\". \"NATIVE\" - Native ad format. \"REWARDED\" - An ad that, once viewed, gets a callback verifying the view so that a reward can be given to the user. Supported ad types are \"RICH_MEDIA\" (interactive) and video where video can not be excluded. \"REWARDED_INTERSTITIAL\" - Rewarded Interstitial ad format. Only supports video ad type. See https://support.google.com/admob/answer/9884467." + }, + "appId": { + "description": "The externally visible ID of the app this ad unit is associated with. Example: ca-app-pub-9876543210987654~0123456789", + "type": "string" + } + } + }, + "ReportRow": { + "id": "ReportRow", + "properties": { + "metricValues": { + "additionalProperties": { + "$ref": "ReportRowMetricValue" + }, + "description": "Map of metric values in a row, with keys as enum name of the metrics. If a metric being requested has no value returned, the map will not include it.", + "type": "object" + }, + "dimensionValues": { + "additionalProperties": { + "$ref": "ReportRowDimensionValue" + }, + "type": "object", + "description": "Map of dimension values in a row, with keys as enum name of the dimensions." + } + }, + "description": "A row of the returning report.", + "type": "object" + }, + "ReportFooter": { + "properties": { + "warnings": { + "type": "array", + "description": "Warnings associated with generation of the report.", + "items": { + "$ref": "ReportWarning" + } + }, + "matchingRowCount": { + "description": "Total number of rows that matched the request. Warning: This count does NOT always match the number of rows in the response. Do not make that assumption when processing the response.", + "format": "int64", + "type": "string" + } + }, + "type": "object", + "description": "Groups data available after report generation, for example, warnings and row counts. Always sent as the last message in the stream response.", + "id": "ReportFooter" + }, + "LocalizationSettings": { + "properties": { + "currencyCode": { + "type": "string", + "description": "Currency code of the earning related metrics, which is the 3-letter code defined in ISO 4217. The daily average rate is used for the currency conversion. Defaults to the account currency code if unspecified." + }, + "languageCode": { + "type": "string", + "description": "Language used for any localized text, such as some dimension value display labels. The language tag defined in the IETF BCP47. Defaults to 'en-US' if unspecified." + } + }, + "type": "object", + "id": "LocalizationSettings", + "description": "Localization settings for reports, such as currency and language. It affects how metrics are calculated." + }, + "App": { + "type": "object", + "properties": { + "appApprovalState": { + "type": "string", + "enum": [ + "APP_APPROVAL_STATE_UNSPECIFIED", + "ACTION_REQUIRED", + "IN_REVIEW", + "APPROVED" + ], + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "The app requires additional user action to be approved. Please refer to https://support.google.com/admob/answer/10564477 for details and next steps.", + "The app is pending review.", + "The app is approved and can serve ads." + ], + "readOnly": true, + "description": "Output only. The approval state for the app. The field is read-only." + }, + "name": { + "description": "Resource name for this app. Format is accounts/{publisher_id}/apps/{app_id_fragment} Example: accounts/pub-9876543210987654/apps/0123456789", + "type": "string" + }, + "manualAppInfo": { + "description": "The information for an app that is not linked to any app store. After an app is linked, this information is still retrivable. If no name is provided for the app upon creation, a placeholder name will be used.", + "$ref": "AppManualAppInfo" + }, + "linkedAppInfo": { + "$ref": "AppLinkedAppInfo", + "description": "Immutable. The information for an app that is linked to an app store. This field is present if and only if the app is linked to an app store." + }, + "platform": { + "description": "Describes the platform of the app. Limited to \"IOS\" and \"ANDROID\".", + "type": "string" + }, + "appId": { + "description": "The externally visible ID of the app which can be used to integrate with AdMob. This is a read only property. Example: ca-app-pub-9876543210987654~0123456789", + "type": "string" + } + }, + "id": "App", + "description": "Describes an AdMob app for a specific platform (For example: Android or iOS)." + }, + "ReportWarning": { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "Describes the details of the warning message, in English." + }, + "type": { + "type": "string", + "enum": [ + "TYPE_UNSPECIFIED", + "DATA_BEFORE_ACCOUNT_TIMEZONE_CHANGE", + "DATA_DELAYED", + "OTHER", + "REPORT_CURRENCY_NOT_ACCOUNT_CURRENCY" + ], + "description": "Type of the warning.", + "enumDescriptions": [ + "Default value for an unset field. Do not use.", + "Some data in this report is aggregated based on a time zone different from the requested time zone. This could happen if a local time-zone report has the start time before the last time this time zone changed. The description field will contain the date of the last time zone change.", + "There is an unusual delay in processing the source data for the requested date range. The report results might be less up to date than usual. AdMob is aware of the issue and is actively working to resolve it.", + "Warnings that are exposed without a specific type. Useful when new warning types are added but the API is not changed yet.", + "The currency being requested is not the account currency. The earning metrics will be based on the requested currency, and thus not a good estimation of the final payment anymore, due to the currency rate fluctuation." + ] + } + }, + "description": "Warnings associated with generation of the report.", + "id": "ReportWarning" + } + }, + "protocol": "rest", + "parameters": { + "fields": { + "location": "query", + "type": "string", + "description": "Selector specifying which fields to include in a partial response." + }, + "alt": { + "location": "query", + "type": "string", + "enum": [ + "json", + "media", + "proto" + ], + "default": "json", + "enumDescriptions": [ + "Responses with Content-Type of application/json", + "Media download with context-dependent Content-Type", + "Responses with Content-Type of application/x-protobuf" + ], + "description": "Data format for response." + }, + "callback": { + "location": "query", + "description": "JSONP", + "type": "string" + }, + "quotaUser": { + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.", + "location": "query" + }, + "oauth_token": { + "location": "query", + "type": "string", + "description": "OAuth 2.0 token for the current user." + }, + "prettyPrint": { + "type": "boolean", + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query" + }, + "access_token": { + "type": "string", + "location": "query", + "description": "OAuth access token." + }, + "upload_protocol": { + "location": "query", + "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").", + "type": "string" + }, + "uploadType": { + "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").", + "location": "query", + "type": "string" + }, + "$.xgafv": { + "type": "string", + "enum": [ + "1", + "2" + ], + "description": "V1 error format.", + "enumDescriptions": [ + "v1 error format", + "v2 error format" + ], + "location": "query" + }, + "key": { + "type": "string", + "location": "query", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token." + } + }, + "ownerName": "Google", + "rootUrl": "https://admob.googleapis.com/", + "title": "AdMob API", + "canonicalName": "AdMob", + "name": "admob", + "baseUrl": "https://admob.googleapis.com/", + "version_module": true, + "servicePath": "", + "mtlsRootUrl": "https://admob.mtls.googleapis.com/", + "resources": { + "accounts": { + "methods": { + "list": { + "description": "Lists the AdMob publisher account that was most recently signed in to from the AdMob UI. For more information, see https://support.google.com/admob/answer/10243672.", + "path": "v1/accounts", + "parameterOrder": [], + "id": "admob.accounts.list", + "streamingType": "NONE", + "response": { + "$ref": "ListPublisherAccountsResponse" + }, + "httpMethod": "GET", + "scopes": [ + "https://www.googleapis.com/auth/admob.readonly", + "https://www.googleapis.com/auth/admob.report" + ], + "parameters": { + "pageSize": { + "format": "int32", + "description": "Maximum number of accounts to return.", + "location": "query", + "type": "integer" + }, + "pageToken": { + "location": "query", + "description": "The value returned by the last `ListPublisherAccountsResponse`; indicates that this is a continuation of a prior `ListPublisherAccounts` call, and that the system should return the next page of data.", + "type": "string" + } + }, + "flatPath": "v1/accounts" + }, + "get": { + "path": "v1/{+name}", + "id": "admob.accounts.get", + "httpMethod": "GET", + "scopes": [ + "https://www.googleapis.com/auth/admob.readonly", + "https://www.googleapis.com/auth/admob.report" + ], + "parameters": { + "name": { + "type": "string", + "pattern": "^accounts/[^/]+$", + "location": "path", + "description": "Resource name of the publisher account to retrieve. Example: accounts/pub-9876543210987654", + "required": true + } + }, + "description": "Gets information about the specified AdMob publisher account.", + "flatPath": "v1/accounts/{accountsId}", + "streamingType": "NONE", + "response": { + "$ref": "PublisherAccount" + }, + "parameterOrder": [ + "name" + ] + } + }, + "resources": { + "apps": { + "methods": { + "list": { + "streamingType": "NONE", + "id": "admob.accounts.apps.list", + "path": "v1/{+parent}/apps", + "scopes": [ + "https://www.googleapis.com/auth/admob.readonly" + ], + "httpMethod": "GET", + "response": { + "$ref": "ListAppsResponse" + }, + "description": "List the apps under the specified AdMob account.", + "parameters": { + "pageSize": { + "location": "query", + "format": "int32", + "type": "integer", + "description": "The maximum number of apps to return. If unspecified or 0, at most 10,000 apps will be returned. The maximum value is 20,000; values above 20,000 will be coerced to 20,000." + }, + "parent": { + "pattern": "^accounts/[^/]+$", + "type": "string", + "description": "Required. Resource name of the account to list apps for. Example: accounts/pub-9876543210987654", + "required": true, + "location": "path" + }, + "pageToken": { + "description": "The value returned by the last `ListAppsResponse`; indicates that this is a continuation of a prior `ListApps` call, and that the system should return the next page of data.", + "location": "query", + "type": "string" + } + }, + "parameterOrder": [ + "parent" + ], + "flatPath": "v1/accounts/{accountsId}/apps" + } + } + }, + "adUnits": { + "methods": { + "list": { + "id": "admob.accounts.adUnits.list", + "flatPath": "v1/accounts/{accountsId}/adUnits", + "path": "v1/{+parent}/adUnits", + "parameterOrder": [ + "parent" + ], + "response": { + "$ref": "ListAdUnitsResponse" + }, + "description": "List the ad units under the specified AdMob account.", + "parameters": { + "pageSize": { + "format": "int32", + "type": "integer", + "description": "The maximum number of ad units to return. If unspecified or 0, at most 10,000 ad units will be returned. The maximum value is 20,000; values above 20,000 will be coerced to 20,000.", + "location": "query" + }, + "pageToken": { + "location": "query", + "type": "string", + "description": "The value returned by the last `ListAdUnitsResponse`; indicates that this is a continuation of a prior `ListAdUnits` call, and that the system should return the next page of data." + }, + "parent": { + "type": "string", + "location": "path", + "description": "Required. Resource name of the account to list ad units for. Example: accounts/pub-9876543210987654", + "pattern": "^accounts/[^/]+$", + "required": true + } + }, + "streamingType": "NONE", + "httpMethod": "GET", + "scopes": [ + "https://www.googleapis.com/auth/admob.readonly" + ] + } + } + }, + "mediationReport": { + "methods": { + "generate": { + "id": "admob.accounts.mediationReport.generate", + "response": { + "$ref": "GenerateMediationReportResponse" + }, + "parameterOrder": [ + "parent" + ], + "scopes": [ + "https://www.googleapis.com/auth/admob.readonly", + "https://www.googleapis.com/auth/admob.report" + ], + "flatPath": "v1/accounts/{accountsId}/mediationReport:generate", + "request": { + "$ref": "GenerateMediationReportRequest" + }, + "description": "Generates an AdMob Mediation report based on the provided report specification. Returns result of a server-side streaming RPC. The result is returned in a sequence of responses.", + "streamingType": "SERVER_SIDE", + "parameters": { + "parent": { + "description": "Resource name of the account to generate the report for. Example: accounts/pub-9876543210987654", + "required": true, + "pattern": "^accounts/[^/]+$", + "type": "string", + "location": "path" + } + }, + "httpMethod": "POST", + "path": "v1/{+parent}/mediationReport:generate" + } + } + }, + "networkReport": { + "methods": { + "generate": { + "httpMethod": "POST", + "response": { + "$ref": "GenerateNetworkReportResponse" + }, + "parameters": { + "parent": { + "pattern": "^accounts/[^/]+$", + "required": true, + "location": "path", + "type": "string", + "description": "Resource name of the account to generate the report for. Example: accounts/pub-9876543210987654" + } + }, + "path": "v1/{+parent}/networkReport:generate", + "flatPath": "v1/accounts/{accountsId}/networkReport:generate", + "description": "Generates an AdMob Network report based on the provided report specification. Returns result of a server-side streaming RPC. The result is returned in a sequence of responses.", + "streamingType": "SERVER_SIDE", + "id": "admob.accounts.networkReport.generate", + "scopes": [ + "https://www.googleapis.com/auth/admob.readonly", + "https://www.googleapis.com/auth/admob.report" + ], + "request": { + "$ref": "GenerateNetworkReportRequest" + }, + "parameterOrder": [ + "parent" + ] + } + } + } + } + } + }, + "discoveryVersion": "v1", + "id": "admob:v1", + "documentationLink": "https://developers.google.com/admob/api/", + "basePath": "" +} diff --git a/tests/src/google/discovery/fixtures/directory.json b/tests/src/google/discovery/fixtures/directory.json new file mode 100644 index 0000000..60a6802 --- /dev/null +++ b/tests/src/google/discovery/fixtures/directory.json @@ -0,0 +1,96 @@ +{ + "kind": "discovery#directoryList", + "discoveryVersion": "v1", + "items": [ + { + "kind": "discovery#directoryItem", + "id": "abusiveexperiencereport:v1", + "name": "abusiveexperiencereport", + "version": "v1", + "title": "Abusive Experience Report API", + "description": "Views Abusive Experience Report data, and gets a list of sites that have a significant number of abusive experiences.", + "discoveryRestUrl": "https://abusiveexperiencereport.googleapis.com/$discovery/rest?version=v1", + "icons": { + "x16": "https://www.gstatic.com/images/branding/product/1x/googleg_16dp.png", + "x32": "https://www.gstatic.com/images/branding/product/1x/googleg_32dp.png" + }, + "documentationLink": "https://developers.google.com/abusive-experience-report/", + "preferred": true + }, + { + "kind": "discovery#directoryItem", + "id": "admob:v1", + "name": "admob", + "version": "v1", + "title": "AdMob API", + "description": "The AdMob API allows publishers to programmatically get information about their AdMob account.", + "discoveryRestUrl": "https://admob.googleapis.com/$discovery/rest?version=v1", + "icons": { + "x16": "https://www.gstatic.com/images/branding/product/1x/googleg_16dp.png", + "x32": "https://www.gstatic.com/images/branding/product/1x/googleg_32dp.png" + }, + "documentationLink": "https://developers.google.com/admob/api/", + "preferred": true + }, + { + "kind": "discovery#directoryItem", + "id": "groupsmigration:v1", + "name": "groupsmigration", + "version": "v1", + "title": "Groups Migration API", + "description": "The Groups Migration API allows domain administrators to archive emails into Google groups.", + "discoveryRestUrl": "https://groupsmigration.googleapis.com/$discovery/rest?version=v1", + "icons": { + "x16": "https://www.gstatic.com/images/branding/product/1x/googleg_16dp.png", + "x32": "https://www.gstatic.com/images/branding/product/1x/googleg_32dp.png" + }, + "documentationLink": "https://developers.google.com/google-apps/groups-migration/", + "preferred": true + }, + { + "kind": "discovery#directoryItem", + "id": "tasks:v1", + "name": "tasks", + "version": "v1", + "title": "Google Tasks API", + "description": "The Google Tasks API lets you manage your tasks and task lists.", + "discoveryRestUrl": "https://tasks.googleapis.com/$discovery/rest?version=v1", + "icons": { + "x16": "https://www.gstatic.com/images/branding/product/1x/googleg_16dp.png", + "x32": "https://www.gstatic.com/images/branding/product/1x/googleg_32dp.png" + }, + "documentationLink": "https://developers.google.com/workspace/tasks/", + "preferred": true + }, + { + "kind": "discovery#directoryItem", + "id": "workflows:v1beta", + "name": "workflows", + "version": "v1beta", + "title": "Workflows API", + "description": "Manage workflow definitions. To execute workflows and manage executions, see the Workflows Executions API.", + "discoveryRestUrl": "https://workflows.googleapis.com/$discovery/rest?version=v1beta", + "icons": { + "x16": "https://www.gstatic.com/images/branding/product/1x/googleg_16dp.png", + "x32": "https://www.gstatic.com/images/branding/product/1x/googleg_32dp.png" + }, + "documentationLink": "https://cloud.google.com/workflows", + "preferred": false + }, + { + "kind": "discovery#directoryItem", + "id": "workflows:v1", + "name": "workflows", + "version": "v1", + "title": "Workflows API", + "description": "Manage workflow definitions. To execute workflows and manage executions, see the Workflows Executions API.", + "discoveryRestUrl": "https://workflows.googleapis.com/$discovery/rest?version=v1", + "icons": { + "x16": "https://www.gstatic.com/images/branding/product/1x/googleg_16dp.png", + "x32": "https://www.gstatic.com/images/branding/product/1x/googleg_32dp.png" + }, + "documentationLink": "https://cloud.google.com/workflows", + "preferred": true + } + ] +} \ No newline at end of file diff --git a/tests/src/google/discovery/fixtures/groupsmigration_v1.json b/tests/src/google/discovery/fixtures/groupsmigration_v1.json new file mode 100644 index 0000000..a159c8d --- /dev/null +++ b/tests/src/google/discovery/fixtures/groupsmigration_v1.json @@ -0,0 +1,171 @@ +{ + "title": "Groups Migration API", + "mtlsRootUrl": "https://groupsmigration.mtls.googleapis.com/", + "description": "The Groups Migration API allows domain administrators to archive emails into Google groups.", + "ownerDomain": "google.com", + "batchPath": "batch", + "kind": "discovery#restDescription", + "canonicalName": "Groups Migration", + "documentationLink": "https://developers.google.com/google-apps/groups-migration/", + "schemas": { + "Groups": { + "id": "Groups", + "type": "object", + "properties": { + "kind": { + "description": "The kind of insert resource this is.", + "type": "string" + }, + "responseCode": { + "type": "string", + "description": "The status of the insert request." + } + }, + "description": "JSON response template for groups migration API." + } + }, + "parameters": { + "quotaUser": { + "type": "string", + "location": "query", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters." + }, + "$.xgafv": { + "description": "V1 error format.", + "enum": [ + "1", + "2" + ], + "enumDescriptions": [ + "v1 error format", + "v2 error format" + ], + "type": "string", + "location": "query" + }, + "prettyPrint": { + "location": "query", + "default": "true", + "description": "Returns response with indentations and line breaks.", + "type": "boolean" + }, + "fields": { + "location": "query", + "description": "Selector specifying which fields to include in a partial response.", + "type": "string" + }, + "access_token": { + "location": "query", + "description": "OAuth access token.", + "type": "string" + }, + "callback": { + "location": "query", + "description": "JSONP", + "type": "string" + }, + "oauth_token": { + "type": "string", + "location": "query", + "description": "OAuth 2.0 token for the current user." + }, + "uploadType": { + "type": "string", + "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").", + "location": "query" + }, + "key": { + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "type": "string", + "location": "query" + }, + "upload_protocol": { + "type": "string", + "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").", + "location": "query" + }, + "alt": { + "location": "query", + "enumDescriptions": [ + "Responses with Content-Type of application/json", + "Media download with context-dependent Content-Type", + "Responses with Content-Type of application/x-protobuf" + ], + "description": "Data format for response.", + "type": "string", + "default": "json", + "enum": [ + "json", + "media", + "proto" + ] + } + }, + "discoveryVersion": "v1", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "baseUrl": "https://groupsmigration.googleapis.com/", + "servicePath": "", + "rootUrl": "https://groupsmigration.googleapis.com/", + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/apps.groups.migration": { + "description": "Upload messages to any Google group in your domain" + } + } + } + }, + "protocol": "rest", + "id": "groupsmigration:v1", + "name": "groupsmigration", + "version": "v1", + "basePath": "", + "resources": { + "archive": { + "methods": { + "insert": { + "flatPath": "groups/v1/groups/{groupId}/archive", + "response": { + "$ref": "Groups" + }, + "scopes": [ + "https://www.googleapis.com/auth/apps.groups.migration" + ], + "path": "groups/v1/groups/{groupId}/archive", + "supportsMediaUpload": true, + "mediaUpload": { + "accept": [ + "message/rfc822" + ], + "maxSize": "26214400", + "protocols": { + "simple": { + "multipart": true, + "path": "/upload/groups/v1/groups/{groupId}/archive" + } + } + }, + "id": "groupsmigration.archive.insert", + "description": "Inserts a new mail into the archive of the Google group.", + "parameters": { + "groupId": { + "type": "string", + "location": "path", + "required": true, + "description": "The group ID" + } + }, + "httpMethod": "POST", + "parameterOrder": [ + "groupId" + ] + } + } + } + }, + "revision": "20261019", + "ownerName": "Google" +} diff --git a/tests/src/google/discovery/fixtures/tasks_v1.json b/tests/src/google/discovery/fixtures/tasks_v1.json new file mode 100644 index 0000000..aa7550d --- /dev/null +++ b/tests/src/google/discovery/fixtures/tasks_v1.json @@ -0,0 +1,836 @@ +{ + "resources": { + "tasklists": { + "methods": { + "delete": { + "parameters": { + "tasklist": { + "required": true, + "type": "string", + "description": "Task list identifier.", + "location": "path" + } + }, + "description": "Deletes the authenticated user's specified task list. If the list contains assigned tasks, both the assigned tasks and the original tasks in the assignment surface (Docs, Chat Spaces) are deleted.", + "scopes": [ + "https://www.googleapis.com/auth/tasks" + ], + "parameterOrder": [ + "tasklist" + ], + "httpMethod": "DELETE", + "path": "tasks/v1/users/@me/lists/{tasklist}", + "flatPath": "tasks/v1/users/@me/lists/{tasklist}", + "id": "tasks.tasklists.delete" + }, + "get": { + "path": "tasks/v1/users/@me/lists/{tasklist}", + "parameters": { + "tasklist": { + "location": "path", + "description": "Task list identifier.", + "required": true, + "type": "string" + } + }, + "flatPath": "tasks/v1/users/@me/lists/{tasklist}", + "id": "tasks.tasklists.get", + "httpMethod": "GET", + "response": { + "$ref": "TaskList" + }, + "description": "Returns the authenticated user's specified task list.", + "scopes": [ + "https://www.googleapis.com/auth/tasks", + "https://www.googleapis.com/auth/tasks.readonly" + ], + "parameterOrder": [ + "tasklist" + ] + }, + "patch": { + "response": { + "$ref": "TaskList" + }, + "parameters": { + "tasklist": { + "description": "Task list identifier.", + "required": true, + "type": "string", + "location": "path" + } + }, + "scopes": [ + "https://www.googleapis.com/auth/tasks" + ], + "request": { + "$ref": "TaskList" + }, + "flatPath": "tasks/v1/users/@me/lists/{tasklist}", + "parameterOrder": [ + "tasklist" + ], + "description": "Updates the authenticated user's specified task list. This method supports patch semantics.", + "httpMethod": "PATCH", + "id": "tasks.tasklists.patch", + "path": "tasks/v1/users/@me/lists/{tasklist}" + }, + "update": { + "id": "tasks.tasklists.update", + "response": { + "$ref": "TaskList" + }, + "description": "Updates the authenticated user's specified task list.", + "request": { + "$ref": "TaskList" + }, + "parameterOrder": [ + "tasklist" + ], + "flatPath": "tasks/v1/users/@me/lists/{tasklist}", + "parameters": { + "tasklist": { + "description": "Task list identifier.", + "required": true, + "type": "string", + "location": "path" + } + }, + "httpMethod": "PUT", + "path": "tasks/v1/users/@me/lists/{tasklist}", + "scopes": [ + "https://www.googleapis.com/auth/tasks" + ] + }, + "list": { + "id": "tasks.tasklists.list", + "response": { + "$ref": "TaskLists" + }, + "flatPath": "tasks/v1/users/@me/lists", + "httpMethod": "GET", + "scopes": [ + "https://www.googleapis.com/auth/tasks", + "https://www.googleapis.com/auth/tasks.readonly" + ], + "path": "tasks/v1/users/@me/lists", + "parameters": { + "maxResults": { + "location": "query", + "format": "int32", + "description": "Maximum number of task lists returned on one page. Optional. The default is 1000 (max allowed: 1000).", + "type": "integer" + }, + "pageToken": { + "description": "Token specifying the result page to return. Optional.", + "location": "query", + "type": "string" + } + }, + "description": "Returns all the authenticated user's task lists. A user can have up to 2000 lists at a time.", + "parameterOrder": [] + }, + "insert": { + "response": { + "$ref": "TaskList" + }, + "scopes": [ + "https://www.googleapis.com/auth/tasks" + ], + "path": "tasks/v1/users/@me/lists", + "parameterOrder": [], + "description": "Creates a new task list and adds it to the authenticated user's task lists. A user can have up to 2000 lists at a time.", + "flatPath": "tasks/v1/users/@me/lists", + "id": "tasks.tasklists.insert", + "parameters": {}, + "request": { + "$ref": "TaskList" + }, + "httpMethod": "POST" + } + } + }, + "tasks": { + "methods": { + "move": { + "flatPath": "tasks/v1/lists/{tasklist}/tasks/{task}/move", + "parameterOrder": [ + "tasklist", + "task" + ], + "description": "Moves the specified task to another position in the destination task list. If the destination list is not specified, the task is moved within its current list. This can include putting it as a child task under a new parent and/or move it to a different position among its sibling tasks. A user can have up to 2,000 subtasks per task.", + "response": { + "$ref": "Task" + }, + "id": "tasks.tasks.move", + "path": "tasks/v1/lists/{tasklist}/tasks/{task}/move", + "parameters": { + "tasklist": { + "location": "path", + "type": "string", + "description": "Task list identifier.", + "required": true + }, + "destinationTasklist": { + "description": "Optional. Destination task list identifier. If set, the task is moved from tasklist to the destinationTasklist list. Otherwise the task is moved within its current list. Recurrent tasks cannot currently be moved between lists.", + "type": "string", + "location": "query" + }, + "task": { + "required": true, + "type": "string", + "description": "Task identifier.", + "location": "path" + }, + "parent": { + "type": "string", + "location": "query", + "description": "Optional. New parent task identifier. If the task is moved to the top level, this parameter is omitted. The task set as parent must exist in the task list and can not be hidden. Exceptions: 1. Assigned and repeating tasks cannot be set as parent tasks (have subtasks), or be moved under a parent task (become subtasks). 2. Tasks that are both completed and hidden cannot be nested, so the parent field must be empty." + }, + "previous": { + "type": "string", + "location": "query", + "description": "Optional. New previous sibling task identifier. If the task is moved to the first position among its siblings, this parameter is omitted. The task set as previous must exist in the task list and can not be hidden. Exceptions: 1. Tasks that are both completed and hidden can only be moved to position 0, so the previous field must be empty." + } + }, + "scopes": [ + "https://www.googleapis.com/auth/tasks" + ], + "httpMethod": "POST" + }, + "delete": { + "id": "tasks.tasks.delete", + "flatPath": "tasks/v1/lists/{tasklist}/tasks/{task}", + "description": "Deletes the specified task from the task list. If the task is assigned, both the assigned task and the original task (in Docs, Chat Spaces) are deleted. To delete the assigned task only, navigate to the assignment surface and unassign the task from there.", + "parameters": { + "tasklist": { + "required": true, + "location": "path", + "type": "string", + "description": "Task list identifier." + }, + "task": { + "required": true, + "description": "Task identifier.", + "type": "string", + "location": "path" + } + }, + "path": "tasks/v1/lists/{tasklist}/tasks/{task}", + "parameterOrder": [ + "tasklist", + "task" + ], + "scopes": [ + "https://www.googleapis.com/auth/tasks" + ], + "httpMethod": "DELETE" + }, + "update": { + "scopes": [ + "https://www.googleapis.com/auth/tasks" + ], + "flatPath": "tasks/v1/lists/{tasklist}/tasks/{task}", + "id": "tasks.tasks.update", + "httpMethod": "PUT", + "path": "tasks/v1/lists/{tasklist}/tasks/{task}", + "response": { + "$ref": "Task" + }, + "parameters": { + "tasklist": { + "required": true, + "location": "path", + "description": "Task list identifier.", + "type": "string" + }, + "task": { + "location": "path", + "required": true, + "description": "Task identifier.", + "type": "string" + } + }, + "description": "Updates the specified task.", + "request": { + "$ref": "Task" + }, + "parameterOrder": [ + "tasklist", + "task" + ] + }, + "list": { + "scopes": [ + "https://www.googleapis.com/auth/tasks", + "https://www.googleapis.com/auth/tasks.readonly" + ], + "description": "Returns all tasks in the specified task list. Doesn't return assigned tasks by default (from Docs, Chat Spaces). A user can have up to 20,000 non-hidden tasks per list and up to 100,000 tasks in total at a time.", + "httpMethod": "GET", + "parameterOrder": [ + "tasklist" + ], + "id": "tasks.tasks.list", + "parameters": { + "showAssigned": { + "location": "query", + "type": "boolean", + "description": "Optional. Flag indicating whether tasks assigned to the current user are returned in the result. Optional. The default is False." + }, + "dueMin": { + "description": "Lower bound for a task's due date (as a RFC 3339 timestamp) to filter by. Optional. The default is not to filter by due date.", + "type": "string", + "location": "query" + }, + "showCompleted": { + "location": "query", + "type": "boolean", + "description": "Flag indicating whether completed tasks are returned in the result. Note that showHidden must also be True to show tasks completed in first party clients, such as the web UI and Google's mobile apps. Optional. The default is True." + }, + "completedMin": { + "type": "string", + "description": "Lower bound for a task's completion date (as a RFC 3339 timestamp) to filter by. Optional. The default is not to filter by completion date.", + "location": "query" + }, + "maxResults": { + "description": "Maximum number of tasks returned on one page. Optional. The default is 20 (max allowed: 100).", + "type": "integer", + "location": "query", + "format": "int32" + }, + "dueMax": { + "location": "query", + "type": "string", + "description": "Upper bound for a task's due date (as a RFC 3339 timestamp) to filter by. Optional. The default is not to filter by due date." + }, + "showDeleted": { + "type": "boolean", + "description": "Flag indicating whether deleted tasks are returned in the result. Optional. The default is False.", + "location": "query" + }, + "showHidden": { + "location": "query", + "type": "boolean", + "description": "Flag indicating whether hidden tasks are returned in the result. Optional. The default is False." + }, + "updatedMin": { + "type": "string", + "description": "Lower bound for a task's last modification time (as a RFC 3339 timestamp) to filter by. Optional. The default is not to filter by last modification time.", + "location": "query" + }, + "pageToken": { + "type": "string", + "location": "query", + "description": "Token specifying the result page to return. Optional." + }, + "completedMax": { + "type": "string", + "description": "Upper bound for a task's completion date (as a RFC 3339 timestamp) to filter by. Optional. The default is not to filter by completion date.", + "location": "query" + }, + "tasklist": { + "location": "path", + "description": "Task list identifier.", + "required": true, + "type": "string" + } + }, + "flatPath": "tasks/v1/lists/{tasklist}/tasks", + "path": "tasks/v1/lists/{tasklist}/tasks", + "response": { + "$ref": "Tasks" + } + }, + "patch": { + "request": { + "$ref": "Task" + }, + "flatPath": "tasks/v1/lists/{tasklist}/tasks/{task}", + "path": "tasks/v1/lists/{tasklist}/tasks/{task}", + "response": { + "$ref": "Task" + }, + "parameterOrder": [ + "tasklist", + "task" + ], + "scopes": [ + "https://www.googleapis.com/auth/tasks" + ], + "description": "Updates the specified task. This method supports patch semantics.", + "parameters": { + "task": { + "location": "path", + "type": "string", + "description": "Task identifier.", + "required": true + }, + "tasklist": { + "type": "string", + "required": true, + "location": "path", + "description": "Task list identifier." + } + }, + "id": "tasks.tasks.patch", + "httpMethod": "PATCH" + }, + "insert": { + "path": "tasks/v1/lists/{tasklist}/tasks", + "scopes": [ + "https://www.googleapis.com/auth/tasks" + ], + "httpMethod": "POST", + "response": { + "$ref": "Task" + }, + "request": { + "$ref": "Task" + }, + "parameters": { + "previous": { + "location": "query", + "description": "Previous sibling task identifier. If the task is created at the first position among its siblings, this parameter is omitted. Optional.", + "type": "string" + }, + "parent": { + "description": "Parent task identifier. If the task is created at the top level, this parameter is omitted. An assigned task cannot be a parent task, nor can it have a parent. Setting the parent to an assigned task results in failure of the request. Optional.", + "type": "string", + "location": "query" + }, + "tasklist": { + "description": "Task list identifier.", + "required": true, + "type": "string", + "location": "path" + } + }, + "parameterOrder": [ + "tasklist" + ], + "description": "Creates a new task on the specified task list. Tasks assigned from Docs or Chat Spaces cannot be inserted from Tasks Public API; they can only be created by assigning them from Docs or Chat Spaces. A user can have up to 20,000 non-hidden tasks per list and up to 100,000 tasks in total at a time.", + "id": "tasks.tasks.insert", + "flatPath": "tasks/v1/lists/{tasklist}/tasks" + }, + "get": { + "flatPath": "tasks/v1/lists/{tasklist}/tasks/{task}", + "id": "tasks.tasks.get", + "parameters": { + "tasklist": { + "description": "Task list identifier.", + "required": true, + "type": "string", + "location": "path" + }, + "task": { + "location": "path", + "description": "Task identifier.", + "type": "string", + "required": true + } + }, + "response": { + "$ref": "Task" + }, + "httpMethod": "GET", + "scopes": [ + "https://www.googleapis.com/auth/tasks", + "https://www.googleapis.com/auth/tasks.readonly" + ], + "description": "Returns the specified task.", + "path": "tasks/v1/lists/{tasklist}/tasks/{task}", + "parameterOrder": [ + "tasklist", + "task" + ] + }, + "clear": { + "scopes": [ + "https://www.googleapis.com/auth/tasks" + ], + "parameters": { + "tasklist": { + "location": "path", + "description": "Task list identifier.", + "required": true, + "type": "string" + } + }, + "flatPath": "tasks/v1/lists/{tasklist}/clear", + "id": "tasks.tasks.clear", + "path": "tasks/v1/lists/{tasklist}/clear", + "httpMethod": "POST", + "description": "Clears all completed tasks from the specified task list. The affected tasks will be marked as 'hidden' and no longer be returned by default when retrieving all tasks for a task list.", + "parameterOrder": [ + "tasklist" + ] + } + } + } + }, + "ownerDomain": "google.com", + "baseUrl": "https://tasks.googleapis.com/", + "icons": { + "x32": "http://www.google.com/images/icons/product/search-32.gif", + "x16": "http://www.google.com/images/icons/product/search-16.gif" + }, + "protocol": "rest", + "fullyEncodeReservedExpansion": true, + "rootUrl": "https://tasks.googleapis.com/", + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/tasks": { + "description": "Create, edit, organize, and delete all your tasks" + }, + "https://www.googleapis.com/auth/tasks.readonly": { + "description": "View your tasks" + } + } + } + }, + "name": "tasks", + "kind": "discovery#restDescription", + "documentationLink": "https://developers.google.com/workspace/tasks/", + "basePath": "", + "servicePath": "", + "ownerName": "Google", + "batchPath": "batch", + "revision": "20261020", + "parameters": { + "oauth_token": { + "description": "OAuth 2.0 token for the current user.", + "type": "string", + "location": "query" + }, + "prettyPrint": { + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query", + "type": "boolean" + }, + "callback": { + "type": "string", + "description": "JSONP", + "location": "query" + }, + "$.xgafv": { + "description": "V1 error format.", + "location": "query", + "enumDescriptions": [ + "v1 error format", + "v2 error format" + ], + "enum": [ + "1", + "2" + ], + "type": "string" + }, + "uploadType": { + "type": "string", + "location": "query", + "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\")." + }, + "fields": { + "description": "Selector specifying which fields to include in a partial response.", + "location": "query", + "type": "string" + }, + "access_token": { + "type": "string", + "description": "OAuth access token.", + "location": "query" + }, + "key": { + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "type": "string", + "location": "query" + }, + "quotaUser": { + "location": "query", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.", + "type": "string" + }, + "alt": { + "type": "string", + "enumDescriptions": [ + "Responses with Content-Type of application/json", + "Media download with context-dependent Content-Type", + "Responses with Content-Type of application/x-protobuf" + ], + "location": "query", + "default": "json", + "enum": [ + "json", + "media", + "proto" + ], + "description": "Data format for response." + }, + "upload_protocol": { + "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").", + "location": "query", + "type": "string" + } + }, + "title": "Google Tasks API", + "mtlsRootUrl": "https://tasks.mtls.googleapis.com/", + "schemas": { + "SpaceInfo": { + "properties": { + "space": { + "description": "Output only. The Chat space where this task originates from. The format is \"spaces/{space}\".", + "type": "string", + "readOnly": true + } + }, + "type": "object", + "id": "SpaceInfo", + "description": "Information about the Chat Space where a task was assigned from." + }, + "TaskList": { + "id": "TaskList", + "type": "object", + "properties": { + "updated": { + "description": "Output only. Last modification time of the task list (as a RFC 3339 timestamp).", + "type": "string", + "readOnly": true + }, + "id": { + "type": "string", + "description": "Task list identifier." + }, + "selfLink": { + "readOnly": true, + "description": "Output only. URL pointing to this task list. Used to retrieve, update, or delete this task list.", + "type": "string" + }, + "etag": { + "type": "string", + "description": "ETag of the resource." + }, + "title": { + "description": "Title of the task list. Maximum length allowed: 1024 characters.", + "type": "string" + }, + "kind": { + "description": "Output only. Type of the resource. This is always \"tasks#taskList\".", + "readOnly": true, + "type": "string" + } + } + }, + "TaskLists": { + "id": "TaskLists", + "type": "object", + "properties": { + "kind": { + "type": "string", + "description": "Type of the resource. This is always \"tasks#taskLists\"." + }, + "etag": { + "type": "string", + "description": "ETag of the resource." + }, + "nextPageToken": { + "type": "string", + "description": "Token that can be used to request the next page of this result." + }, + "items": { + "items": { + "$ref": "TaskList" + }, + "description": "Collection of task lists.", + "type": "array" + } + } + }, + "AssignmentInfo": { + "type": "object", + "id": "AssignmentInfo", + "description": "Information about the source of the task assignment (Document, Chat Space).", + "properties": { + "surfaceType": { + "enum": [ + "CONTEXT_TYPE_UNSPECIFIED", + "GMAIL", + "DOCUMENT", + "SPACE" + ], + "enumDescriptions": [ + "Unknown value for this task's context.", + "The task is created from Gmail.", + "The task is assigned from a document.", + "The task is assigned from a Chat Space." + ], + "description": "Output only. The type of surface this assigned task originates from. Currently limited to DOCUMENT or SPACE.", + "readOnly": true, + "type": "string" + }, + "linkToTask": { + "description": "Output only. An absolute link to the original task in the surface of assignment (Docs, Chat spaces, etc.).", + "type": "string", + "readOnly": true + }, + "spaceInfo": { + "description": "Output only. Information about the Chat Space where this task originates from. This field is read-only.", + "readOnly": true, + "$ref": "SpaceInfo" + }, + "driveResourceInfo": { + "$ref": "DriveResourceInfo", + "description": "Output only. Information about the Drive file where this task originates from. Currently, the Drive file can only be a document. This field is read-only.", + "readOnly": true + } + } + }, + "Tasks": { + "properties": { + "items": { + "type": "array", + "description": "Collection of tasks.", + "items": { + "$ref": "Task" + } + }, + "kind": { + "description": "Type of the resource. This is always \"tasks#tasks\".", + "type": "string" + }, + "nextPageToken": { + "type": "string", + "description": "Token used to access the next page of this result." + }, + "etag": { + "type": "string", + "description": "ETag of the resource." + } + }, + "type": "object", + "id": "Tasks" + }, + "DriveResourceInfo": { + "properties": { + "driveFileId": { + "description": "Output only. Identifier of the file in the Drive API.", + "readOnly": true, + "type": "string" + }, + "resourceKey": { + "description": "Output only. Resource key required to access files shared via a shared link. Not required for all files. See also developers.google.com/drive/api/guides/resource-keys.", + "readOnly": true, + "type": "string" + } + }, + "type": "object", + "description": "Information about the Drive resource where a task was assigned from (the document, sheet, etc.).", + "id": "DriveResourceInfo" + }, + "Task": { + "id": "Task", + "type": "object", + "properties": { + "hidden": { + "type": "boolean", + "description": "Flag indicating whether the task is hidden. This is the case if the task had been marked completed when the task list was last cleared. The default is False. This field is read-only." + }, + "position": { + "description": "Output only. String indicating the position of the task among its sibling tasks under the same parent task or at the top level. If this string is greater than another task's corresponding position string according to lexicographical ordering, the task is positioned after the other task under the same parent task (or at the top level). Use the \"move\" method to move the task to another position.", + "type": "string", + "readOnly": true + }, + "webViewLink": { + "description": "Output only. An absolute link to the task in the Google Tasks Web UI.", + "type": "string", + "readOnly": true + }, + "due": { + "type": "string", + "description": "Scheduled date for the task (as an RFC 3339 timestamp). Optional. This represents the day that the task should be done, or that the task is visible on the calendar grid. It doesn't represent the deadline of the task. Only date information is recorded; the time portion of the timestamp is discarded when setting this field. It isn't possible to read or write the time that a task is scheduled for using the API." + }, + "completed": { + "description": "Completion date of the task (as a RFC 3339 timestamp). This field is omitted if the task has not been completed.", + "type": "string" + }, + "links": { + "type": "array", + "items": { + "properties": { + "link": { + "type": "string", + "description": "The URL." + }, + "type": { + "description": "Type of the link, e.g. \"email\", \"generic\", \"chat_message\", \"keep_note\".", + "type": "string" + }, + "description": { + "type": "string", + "description": "The description (might be empty)." + } + }, + "type": "object" + }, + "readOnly": true, + "description": "Output only. Collection of links. This collection is read-only." + }, + "status": { + "description": "Status of the task. This is either \"needsAction\" or \"completed\".", + "type": "string" + }, + "assignmentInfo": { + "description": "Output only. Context information for assigned tasks. A task can be assigned to a user, currently possible from surfaces like Docs and Chat Spaces. This field is populated for tasks assigned to the current user and identifies where the task was assigned from. This field is read-only.", + "$ref": "AssignmentInfo", + "readOnly": true + }, + "parent": { + "readOnly": true, + "description": "Output only. Parent task identifier. This field is omitted if it is a top-level task. Use the \"move\" method to move the task under a different parent or to the top level. A parent task can never be an assigned task (from Chat Spaces, Docs). This field is read-only.", + "type": "string" + }, + "deleted": { + "description": "Flag indicating whether the task has been deleted. For assigned tasks this field is read-only. They can only be deleted by calling tasks.delete, in which case both the assigned task and the original task (in Docs or Chat Spaces) are deleted. To delete the assigned task only, navigate to the assignment surface and unassign the task from there. The default is False.", + "type": "boolean" + }, + "id": { + "description": "Task identifier.", + "type": "string" + }, + "updated": { + "type": "string", + "readOnly": true, + "description": "Output only. Last modification time of the task (as a RFC 3339 timestamp)." + }, + "selfLink": { + "type": "string", + "readOnly": true, + "description": "Output only. URL pointing to this task. Used to retrieve, update, or delete this task." + }, + "kind": { + "readOnly": true, + "description": "Output only. Type of the resource. This is always \"tasks#task\".", + "type": "string" + }, + "notes": { + "type": "string", + "description": "Notes describing the task. Tasks assigned from Google Docs cannot have notes. Optional. Maximum length allowed: 8192 characters." + }, + "etag": { + "description": "ETag of the resource.", + "type": "string" + }, + "title": { + "description": "Title of the task. Maximum length allowed: 1024 characters.", + "type": "string" + } + } + } + }, + "discoveryVersion": "v1", + "id": "tasks:v1", + "canonicalName": "Tasks", + "description": "The Google Tasks API lets you manage your tasks and task lists.", + "version": "v1" +} diff --git a/tests/src/google/discovery/fixtures/workflows_v1.json b/tests/src/google/discovery/fixtures/workflows_v1.json new file mode 100644 index 0000000..10f7d3e --- /dev/null +++ b/tests/src/google/discovery/fixtures/workflows_v1.json @@ -0,0 +1,1071 @@ +{ + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "rootUrl": "https://workflows.googleapis.com/", + "baseUrl": "https://workflows.googleapis.com/", + "parameters": { + "access_token": { + "type": "string", + "description": "OAuth access token.", + "location": "query" + }, + "$.xgafv": { + "location": "query", + "enum": [ + "1", + "2" + ], + "description": "V1 error format.", + "enumDescriptions": [ + "v1 error format", + "v2 error format" + ], + "type": "string" + }, + "alt": { + "enum": [ + "json", + "media", + "proto" + ], + "enumDescriptions": [ + "Responses with Content-Type of application/json", + "Media download with context-dependent Content-Type", + "Responses with Content-Type of application/x-protobuf" + ], + "description": "Data format for response.", + "location": "query", + "default": "json", + "type": "string" + }, + "uploadType": { + "type": "string", + "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").", + "location": "query" + }, + "key": { + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "type": "string", + "location": "query" + }, + "callback": { + "type": "string", + "location": "query", + "description": "JSONP" + }, + "fields": { + "location": "query", + "description": "Selector specifying which fields to include in a partial response.", + "type": "string" + }, + "oauth_token": { + "description": "OAuth 2.0 token for the current user.", + "location": "query", + "type": "string" + }, + "prettyPrint": { + "type": "boolean", + "default": "true", + "description": "Returns response with indentations and line breaks.", + "location": "query" + }, + "upload_protocol": { + "location": "query", + "type": "string", + "description": "Upload protocol for media (e.g. \"raw\", \"multipart\")." + }, + "quotaUser": { + "location": "query", + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters." + } + }, + "fullyEncodeReservedExpansion": true, + "description": "Manage workflow definitions. To execute workflows and manage executions, see the Workflows Executions API.", + "name": "workflows", + "kind": "discovery#restDescription", + "version_module": true, + "title": "Workflows API", + "mtlsRootUrl": "https://workflows.mtls.googleapis.com/", + "schemas": { + "Empty": { + "id": "Empty", + "description": "A generic empty message that you can re-use to avoid defining duplicated empty messages in your APIs. A typical example is to use it as the request or the response type of an API method. For instance: service Foo { rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); }", + "properties": {}, + "type": "object" + }, + "Location": { + "id": "Location", + "description": "A resource that represents a Google Cloud location.", + "properties": { + "locationId": { + "type": "string", + "description": "The canonical id for this location. For example: `\"us-east1\"`." + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Cross-service attributes for the location. For example {\"cloud.googleapis.com/region\": \"us-east1\"}" + }, + "displayName": { + "description": "The friendly name for this location, typically a nearby city name. For example, \"Tokyo\".", + "type": "string" + }, + "name": { + "type": "string", + "description": "Resource name for the location, which may vary between implementations. For example: `\"projects/example-project/locations/us-east1\"`" + }, + "metadata": { + "type": "object", + "description": "Service-specific metadata. For example the available capacity at the given location.", + "additionalProperties": { + "type": "any", + "description": "Properties of the object. Contains field @type with type URL." + } + } + }, + "type": "object" + }, + "ListLocationsResponse": { + "properties": { + "locations": { + "description": "A list of locations that matches the specified filter in the request.", + "items": { + "$ref": "Location" + }, + "type": "array" + }, + "nextPageToken": { + "type": "string", + "description": "The standard List next-page token." + } + }, + "description": "The response message for Locations.ListLocations.", + "type": "object", + "id": "ListLocationsResponse" + }, + "OperationMetadata": { + "description": "Represents the metadata of the long-running operation.", + "type": "object", + "id": "OperationMetadata", + "properties": { + "apiVersion": { + "description": "API version used to start the operation.", + "type": "string" + }, + "endTime": { + "format": "google-datetime", + "description": "The time the operation finished running.", + "type": "string" + }, + "createTime": { + "type": "string", + "format": "google-datetime", + "description": "The time the operation was created." + }, + "verb": { + "type": "string", + "description": "Name of the verb executed by the operation." + }, + "target": { + "description": "Server-defined resource path for the target of the operation.", + "type": "string" + } + } + }, + "ListOperationsResponse": { + "type": "object", + "description": "The response message for Operations.ListOperations.", + "properties": { + "unreachable": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Unordered list. Unreachable resources. Populated when the request sets `ListOperationsRequest.return_partial_success` and reads across collections. For example, when attempting to list all resources across all supported locations." + }, + "nextPageToken": { + "description": "The standard List next-page token.", + "type": "string" + }, + "operations": { + "type": "array", + "items": { + "$ref": "Operation" + }, + "description": "A list of operations that matches the specified filter in the request." + } + }, + "id": "ListOperationsResponse" + }, + "ListWorkflowsResponse": { + "properties": { + "unreachable": { + "items": { + "type": "string" + }, + "description": "Unreachable resources.", + "type": "array" + }, + "nextPageToken": { + "type": "string", + "description": "A token, which can be sent as `page_token` to retrieve the next page. If this field is omitted, there are no subsequent pages." + }, + "workflows": { + "type": "array", + "items": { + "$ref": "Workflow" + }, + "description": "The workflows that match the request." + } + }, + "description": "Response for the ListWorkflows method.", + "type": "object", + "id": "ListWorkflowsResponse" + }, + "ListWorkflowRevisionsResponse": { + "properties": { + "nextPageToken": { + "description": "A token, which can be sent as `page_token` to retrieve the next page. If this field is omitted, there are no subsequent pages.", + "type": "string" + }, + "workflows": { + "description": "The revisions of the workflow, ordered in reverse chronological order.", + "type": "array", + "items": { + "$ref": "Workflow" + } + } + }, + "id": "ListWorkflowRevisionsResponse", + "description": "Response for the ListWorkflowRevisions method.", + "type": "object" + }, + "Operation": { + "description": "This resource represents a long-running operation that is the result of a network API call.", + "properties": { + "done": { + "type": "boolean", + "description": "If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available." + }, + "name": { + "description": "The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.", + "type": "string" + }, + "error": { + "$ref": "Status", + "description": "The error result of the operation in case of failure or cancellation." + }, + "response": { + "description": "The normal, successful response of the operation. If the original method returns no data on success, such as `Delete`, the response is `google.protobuf.Empty`. If the original method is standard `Get`/`Create`/`Update`, the response should be the resource. For other methods, the response should have the type `XxxResponse`, where `Xxx` is the original method name. For example, if the original method name is `TakeSnapshot()`, the inferred response type is `TakeSnapshotResponse`.", + "additionalProperties": { + "type": "any", + "description": "Properties of the object. Contains field @type with type URL." + }, + "type": "object" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "description": "Properties of the object. Contains field @type with type URL.", + "type": "any" + }, + "description": "Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any." + } + }, + "type": "object", + "id": "Operation" + }, + "StateError": { + "id": "StateError", + "description": "Describes an error related to the current state of the workflow.", + "type": "object", + "properties": { + "details": { + "type": "string", + "description": "Provides specifics about the error." + }, + "type": { + "type": "string", + "enum": [ + "TYPE_UNSPECIFIED", + "KMS_ERROR" + ], + "description": "The type of this state error.", + "enumDescriptions": [ + "No type specified.", + "Caused by an issue with KMS." + ] + } + } + }, + "Workflow": { + "id": "Workflow", + "properties": { + "executionHistoryLevel": { + "enum": [ + "EXECUTION_HISTORY_LEVEL_UNSPECIFIED", + "EXECUTION_HISTORY_BASIC", + "EXECUTION_HISTORY_DETAILED" + ], + "description": "Optional. Describes the execution history level to apply to this workflow.", + "type": "string", + "enumDescriptions": [ + "The default/unset value.", + "Enable execution history basic feature.", + "Enable execution history detailed feature." + ] + }, + "serviceAccount": { + "description": "The service account associated with the latest workflow version. This service account represents the identity of the workflow and determines what permissions the workflow has. Format: projects/{project}/serviceAccounts/{account} or {account} Using `-` as a wildcard for the `{project}` or not providing one at all will infer the project from the account. The `{account}` value can be the `email` address or the `unique_id` of the service account. If not provided, workflow will use the project's default service account. Modifying this field for an existing workflow results in a new workflow revision.", + "type": "string" + }, + "description": { + "type": "string", + "description": "Description of the workflow provided by the user. Must be at most 1000 Unicode characters long. This is a workflow-wide field and is not tied to a specific revision." + }, + "revisionCreateTime": { + "type": "string", + "description": "Output only. The timestamp for the latest revision of the workflow's creation.", + "readOnly": true, + "format": "google-datetime" + }, + "userEnvVars": { + "description": "Optional. User-defined environment variables associated with this workflow revision. This map has a maximum length of 20. Each string can take up to 4KiB. Keys cannot be empty strings and cannot start with \"GOOGLE\" or \"WORKFLOWS\".", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "callLogLevel": { + "description": "Optional. Describes the level of platform logging to apply to calls and call responses during executions of this workflow. If both the workflow and the execution specify a logging level, the execution level takes precedence.", + "enumDescriptions": [ + "No call logging level specified.", + "Log all call steps within workflows, all call returns, and all exceptions raised.", + "Log only exceptions that are raised from call steps within workflows.", + "Explicitly log nothing." + ], + "enum": [ + "CALL_LOG_LEVEL_UNSPECIFIED", + "LOG_ALL_CALLS", + "LOG_ERRORS_ONLY", + "LOG_NONE" + ], + "type": "string" + }, + "tags": { + "additionalProperties": { + "type": "string" + }, + "description": "Optional. Input only. Immutable. Tags associated with this workflow.", + "type": "object" + }, + "allKmsKeysVersions": { + "type": "array", + "readOnly": true, + "description": "Output only. A list of all KMS crypto key versions used to encrypt or decrypt the data associated with the workflow.", + "items": { + "type": "string" + } + }, + "createTime": { + "readOnly": true, + "description": "Output only. The timestamp for when the workflow was created. This is a workflow-wide field and is not tied to a specific revision.", + "format": "google-datetime", + "type": "string" + }, + "allKmsKeys": { + "description": "Output only. A list of all KMS crypto keys used to encrypt or decrypt the data associated with the workflow.", + "type": "array", + "items": { + "type": "string" + }, + "readOnly": true + }, + "revisionId": { + "description": "Output only. The revision of the workflow. A new revision of a workflow is created as a result of updating the following properties of a workflow: - Service account - Workflow code to be executed The format is \"000001-a4d\", where the first six characters define the zero-padded revision ordinal number. They are followed by a hyphen and three hexadecimal random characters.", + "readOnly": true, + "type": "string" + }, + "state": { + "enumDescriptions": [ + "Invalid state.", + "The workflow has been deployed successfully and is serving.", + "Workflow data is unavailable. See the `state_error` field." + ], + "type": "string", + "description": "Output only. State of the workflow deployment.", + "enum": [ + "STATE_UNSPECIFIED", + "ACTIVE", + "UNAVAILABLE" + ], + "readOnly": true + }, + "name": { + "type": "string", + "description": "The resource name of the workflow. Format: projects/{project}/locations/{location}/workflows/{workflow}. This is a workflow-wide field and is not tied to a specific revision." + }, + "labels": { + "description": "Labels associated with this workflow. Labels can contain at most 64 entries. Keys and values can be no longer than 63 characters and can only contain lowercase letters, numeric characters, underscores, and dashes. Label keys must start with a letter. International characters are allowed. This is a workflow-wide field and is not tied to a specific revision.", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "sourceContents": { + "type": "string", + "description": "Workflow code to be executed. The size limit is 128KB." + }, + "cryptoKeyName": { + "type": "string", + "description": "Optional. The resource name of a KMS crypto key used to encrypt or decrypt the data associated with the workflow. Format: projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey} Using `-` as a wildcard for the `{project}` or not providing one at all will infer the project from the account. If not provided, data associated with the workflow will not be CMEK-encrypted." + }, + "stateError": { + "readOnly": true, + "$ref": "StateError", + "description": "Output only. Error regarding the state of the workflow. For example, this field will have error details if the execution data is unavailable due to revoked KMS key permissions." + }, + "updateTime": { + "format": "google-datetime", + "readOnly": true, + "description": "Output only. The timestamp for when the workflow was last updated. This is a workflow-wide field and is not tied to a specific revision.", + "type": "string" + }, + "cryptoKeyVersion": { + "type": "string", + "readOnly": true, + "description": "Output only. The resource name of a KMS crypto key version used to encrypt or decrypt the data associated with the workflow. Format: projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey}/cryptoKeyVersions/{cryptoKeyVersion}" + } + }, + "type": "object", + "description": "Workflow program to be executed by Workflows." + }, + "Status": { + "type": "object", + "description": "The `Status` type defines a logical error model that is suitable for different programming environments, including REST APIs and RPC APIs. It is used by [gRPC](https://github.com/grpc). Each `Status` message contains three pieces of data: error code, error message, and error details. You can find out more about this error model and how to work with it in the [API Design Guide](https://cloud.google.com/apis/design/errors).", + "properties": { + "details": { + "description": "A list of messages that carry the error details. There is a common set of message types for APIs to use.", + "items": { + "type": "object", + "additionalProperties": { + "type": "any", + "description": "Properties of the object. Contains field @type with type URL." + } + }, + "type": "array" + }, + "code": { + "format": "int32", + "description": "The status code, which should be an enum value of google.rpc.Code.", + "type": "integer" + }, + "message": { + "type": "string", + "description": "A developer-facing error message, which should be in English. Any user-facing error message should be localized and sent in the google.rpc.Status.details field, or localized by the client." + } + }, + "id": "Status" + } + }, + "protocol": "rest", + "ownerName": "Google", + "canonicalName": "Workflows", + "basePath": "", + "revision": "20261007", + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/cloud-platform": { + "description": "See, edit, configure, and delete your Google Cloud data and see the email address for your Google Account." + } + } + } + }, + "discoveryVersion": "v1", + "servicePath": "", + "ownerDomain": "google.com", + "version": "v1", + "documentationLink": "https://cloud.google.com/workflows", + "resources": { + "projects": { + "resources": { + "locations": { + "methods": { + "list": { + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "parameterOrder": [ + "name" + ], + "id": "workflows.projects.locations.list", + "flatPath": "v1/projects/{projectsId}/locations", + "httpMethod": "GET", + "path": "v1/{+name}/locations", + "description": "Lists information about the supported locations for this service. This method lists locations based on the resource scope provided in the ListLocationsRequest.name field: * **Global locations**: If `name` is empty, the method lists the public locations available to all projects. * **Project-specific locations**: If `name` follows the format `projects/{project}`, the method lists locations visible to that specific project. This includes public, private, or other project-specific locations enabled for the project. For gRPC and client library implementations, the resource name is passed as the `name` field. For direct service calls, the resource name is incorporated into the request path based on the specific service implementation and version.", + "parameters": { + "extraLocationTypes": { + "location": "query", + "repeated": true, + "description": "Optional. Do not use this field unless explicitly documented otherwise. This is primarily for internal usage.", + "type": "string" + }, + "filter": { + "description": "A filter to narrow down results to a preferred subset. The filtering language accepts strings like `\"displayName=tokyo\"`, and is documented in more detail in [AIP-160](https://google.aip.dev/160).", + "location": "query", + "type": "string" + }, + "pageToken": { + "type": "string", + "description": "A page token received from the `next_page_token` field in the response. Send that page token to receive the subsequent page.", + "location": "query" + }, + "pageSize": { + "location": "query", + "format": "int32", + "description": "The maximum number of results to return. If not set, the service selects a default.", + "type": "integer" + }, + "name": { + "location": "path", + "required": true, + "type": "string", + "description": "The resource that owns the locations collection, if applicable.", + "pattern": "^projects/[^/]+$" + } + }, + "response": { + "$ref": "ListLocationsResponse" + } + }, + "get": { + "httpMethod": "GET", + "response": { + "$ref": "Location" + }, + "parameterOrder": [ + "name" + ], + "id": "workflows.projects.locations.get", + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "flatPath": "v1/projects/{projectsId}/locations/{locationsId}", + "description": "Gets information about a location.", + "parameters": { + "name": { + "description": "Resource name for the location.", + "required": true, + "location": "path", + "pattern": "^projects/[^/]+/locations/[^/]+$", + "type": "string" + } + }, + "path": "v1/{+name}" + } + }, + "resources": { + "operations": { + "methods": { + "get": { + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "path": "v1/{+name}", + "description": "Gets the latest state of a long-running operation. Clients can use this method to poll the operation result at intervals as recommended by the API service.", + "id": "workflows.projects.locations.operations.get", + "httpMethod": "GET", + "response": { + "$ref": "Operation" + }, + "parameters": { + "name": { + "location": "path", + "required": true, + "description": "The name of the operation resource.", + "type": "string", + "pattern": "^projects/[^/]+/locations/[^/]+/operations/[^/]+$" + } + }, + "parameterOrder": [ + "name" + ], + "flatPath": "v1/projects/{projectsId}/locations/{locationsId}/operations/{operationsId}" + }, + "list": { + "flatPath": "v1/projects/{projectsId}/locations/{locationsId}/operations", + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "httpMethod": "GET", + "description": "Lists operations that match the specified filter in the request. If the server doesn't support this method, it returns `UNIMPLEMENTED`.", + "id": "workflows.projects.locations.operations.list", + "parameterOrder": [ + "name" + ], + "response": { + "$ref": "ListOperationsResponse" + }, + "path": "v1/{+name}/operations", + "parameters": { + "pageSize": { + "format": "int32", + "description": "The standard list page size.", + "location": "query", + "type": "integer" + }, + "filter": { + "location": "query", + "type": "string", + "description": "The standard list filter." + }, + "pageToken": { + "location": "query", + "type": "string", + "description": "The standard list page token." + }, + "name": { + "required": true, + "location": "path", + "description": "The name of the operation's parent resource.", + "pattern": "^projects/[^/]+/locations/[^/]+$", + "type": "string" + }, + "returnPartialSuccess": { + "description": "When set to `true`, operations that are reachable are returned as normal, and those that are unreachable are returned in the ListOperationsResponse.unreachable field. This can only be `true` when reading across collections. For example, when `parent` is set to `\"projects/example/locations/-\"`. This field is not supported by default and will result in an `UNIMPLEMENTED` error if set unless explicitly documented otherwise in service or product specific documentation.", + "location": "query", + "type": "boolean" + } + } + }, + "delete": { + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "httpMethod": "DELETE", + "response": { + "$ref": "Empty" + }, + "path": "v1/{+name}", + "description": "Deletes a long-running operation. This method indicates that the client is no longer interested in the operation result. It does not cancel the operation. If the server doesn't support this method, it returns `google.rpc.Code.UNIMPLEMENTED`.", + "id": "workflows.projects.locations.operations.delete", + "flatPath": "v1/projects/{projectsId}/locations/{locationsId}/operations/{operationsId}", + "parameters": { + "name": { + "pattern": "^projects/[^/]+/locations/[^/]+/operations/[^/]+$", + "description": "The name of the operation resource to be deleted.", + "type": "string", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "name" + ] + } + } + }, + "workflows": { + "methods": { + "list": { + "path": "v1/{+parent}/workflows", + "parameters": { + "orderBy": { + "description": "Comma-separated list of fields that specify the order of the results. Default sorting order for a field is ascending. To specify descending order for a field, append a \"desc\" suffix. If not specified, the results are returned in an unspecified order.", + "type": "string", + "location": "query" + }, + "filter": { + "location": "query", + "description": "Filter to restrict results to specific workflows. For details, see AIP-160. For example, if you are using the Google APIs Explorer: `state=\"SUCCEEDED\"` or `createTime\u003e\"2023-08-01\" AND state=\"FAILED\"`", + "type": "string" + }, + "pageSize": { + "type": "integer", + "location": "query", + "format": "int32", + "description": "Maximum number of workflows to return per call. The service might return fewer than this value even if not at the end of the collection. If a value is not specified, a default value of 500 is used. The maximum permitted value is 1000 and values greater than 1000 are coerced down to 1000." + }, + "pageToken": { + "type": "string", + "description": "A page token, received from a previous `ListWorkflows` call. Provide this to retrieve the subsequent page. When paginating, all other parameters provided to `ListWorkflows` must match the call that provided the page token.", + "location": "query" + }, + "parent": { + "pattern": "^projects/[^/]+/locations/[^/]+$", + "type": "string", + "description": "Required. Project and location from which the workflows should be listed. Format: projects/{project}/locations/{location}", + "required": true, + "location": "path" + } + }, + "response": { + "$ref": "ListWorkflowsResponse" + }, + "parameterOrder": [ + "parent" + ], + "description": "Lists workflows in a given project and location. The default order is not specified.", + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "flatPath": "v1/projects/{projectsId}/locations/{locationsId}/workflows", + "httpMethod": "GET", + "id": "workflows.projects.locations.workflows.list" + }, + "get": { + "id": "workflows.projects.locations.workflows.get", + "flatPath": "v1/projects/{projectsId}/locations/{locationsId}/workflows/{workflowsId}", + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "description": "Gets details of a single workflow.", + "parameters": { + "revisionId": { + "type": "string", + "location": "query", + "description": "Optional. The revision of the workflow to retrieve. If the revision_id is empty, the latest revision is retrieved. The format is \"000001-a4d\", where the first six characters define the zero-padded decimal revision number. They are followed by a hyphen and three hexadecimal characters." + }, + "name": { + "pattern": "^projects/[^/]+/locations/[^/]+/workflows/[^/]+$", + "type": "string", + "description": "Required. Name of the workflow for which information should be retrieved. Format: projects/{project}/locations/{location}/workflows/{workflow}", + "required": true, + "location": "path" + } + }, + "path": "v1/{+name}", + "httpMethod": "GET", + "parameterOrder": [ + "name" + ], + "response": { + "$ref": "Workflow" + } + }, + "listRevisions": { + "httpMethod": "GET", + "flatPath": "v1/projects/{projectsId}/locations/{locationsId}/workflows/{workflowsId}:listRevisions", + "description": "Lists revisions for a given workflow.", + "response": { + "$ref": "ListWorkflowRevisionsResponse" + }, + "parameters": { + "pageSize": { + "location": "query", + "description": "The maximum number of revisions to return per page. If a value is not specified, a default value of 20 is used. The maximum permitted value is 100. Values greater than 100 are coerced down to 100.", + "type": "integer", + "format": "int32" + }, + "pageToken": { + "description": "The page token, received from a previous ListWorkflowRevisions call. Provide this to retrieve the subsequent page.", + "type": "string", + "location": "query" + }, + "name": { + "location": "path", + "required": true, + "pattern": "^projects/[^/]+/locations/[^/]+/workflows/[^/]+$", + "description": "Required. Workflow for which the revisions should be listed. Format: projects/{project}/locations/{location}/workflows/{workflow}", + "type": "string" + } + }, + "parameterOrder": [ + "name" + ], + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "id": "workflows.projects.locations.workflows.listRevisions", + "path": "v1/{+name}:listRevisions" + }, + "delete": { + "path": "v1/{+name}", + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "id": "workflows.projects.locations.workflows.delete", + "httpMethod": "DELETE", + "response": { + "$ref": "Operation" + }, + "parameters": { + "name": { + "location": "path", + "type": "string", + "required": true, + "description": "Required. Name of the workflow to be deleted. Format: projects/{project}/locations/{location}/workflows/{workflow}", + "pattern": "^projects/[^/]+/locations/[^/]+/workflows/[^/]+$" + } + }, + "parameterOrder": [ + "name" + ], + "description": "Deletes a workflow with the specified name. This method also cancels and deletes all running executions of the workflow.", + "flatPath": "v1/projects/{projectsId}/locations/{locationsId}/workflows/{workflowsId}" + }, + "create": { + "description": "Creates a new workflow. If a workflow with the specified name already exists in the specified project and location, the long running operation returns a ALREADY_EXISTS error.", + "response": { + "$ref": "Operation" + }, + "id": "workflows.projects.locations.workflows.create", + "httpMethod": "POST", + "request": { + "$ref": "Workflow" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "flatPath": "v1/projects/{projectsId}/locations/{locationsId}/workflows", + "path": "v1/{+parent}/workflows", + "parameters": { + "parent": { + "type": "string", + "pattern": "^projects/[^/]+/locations/[^/]+$", + "description": "Required. Project and location in which the workflow should be created. Format: projects/{project}/locations/{location}", + "location": "path", + "required": true + }, + "workflowId": { + "description": "Required. The ID of the workflow to be created. It has to fulfill the following requirements: * Must contain only letters, numbers, underscores and hyphens. * Must start with a letter. * Must be between 1-64 characters. * Must end with a number or a letter. * Must be unique within the customer project and location.", + "type": "string", + "location": "query" + } + }, + "parameterOrder": [ + "parent" + ] + }, + "patch": { + "parameters": { + "name": { + "description": "The resource name of the workflow. Format: projects/{project}/locations/{location}/workflows/{workflow}. This is a workflow-wide field and is not tied to a specific revision.", + "required": true, + "pattern": "^projects/[^/]+/locations/[^/]+/workflows/[^/]+$", + "type": "string", + "location": "path" + }, + "updateMask": { + "location": "query", + "description": "List of fields to be updated. If not present, the entire workflow will be updated.", + "type": "string", + "format": "google-fieldmask" + } + }, + "request": { + "$ref": "Workflow" + }, + "response": { + "$ref": "Operation" + }, + "httpMethod": "PATCH", + "description": "Updates an existing workflow. Running this method has no impact on already running executions of the workflow. A new revision of the workflow might be created as a result of a successful update operation. In that case, the new revision is used in new workflow executions.", + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "flatPath": "v1/projects/{projectsId}/locations/{locationsId}/workflows/{workflowsId}", + "parameterOrder": [ + "name" + ], + "id": "workflows.projects.locations.workflows.patch", + "path": "v1/{+name}" + } + } + } + } + } + } + } + }, + "id": "workflows:v1", + "batchPath": "batch", + "endpoints": [ + { + "location": "asia-south1", + "description": "Regional Endpoint", + "endpointUrl": "https://workflows.asia-south1.rep.googleapis.com/" + }, + { + "endpointUrl": "https://workflows.asia-southeast2.rep.googleapis.com/", + "location": "asia-southeast2", + "description": "Regional Endpoint" + }, + { + "description": "Regional Endpoint", + "location": "asia-southeast3", + "endpointUrl": "https://workflows.asia-southeast3.rep.googleapis.com/" + }, + { + "location": "australia-southeast1", + "description": "Regional Endpoint", + "endpointUrl": "https://workflows.australia-southeast1.rep.googleapis.com/" + }, + { + "location": "australia-southeast2", + "description": "Regional Endpoint", + "endpointUrl": "https://workflows.australia-southeast2.rep.googleapis.com/" + }, + { + "description": "Regional Endpoint", + "location": "europe-central2", + "endpointUrl": "https://workflows.europe-central2.rep.googleapis.com/" + }, + { + "endpointUrl": "https://workflows.europe-north1.rep.googleapis.com/", + "location": "europe-north1", + "description": "Regional Endpoint" + }, + { + "endpointUrl": "https://workflows.europe-north2.rep.googleapis.com/", + "location": "europe-north2", + "description": "Regional Endpoint" + }, + { + "location": "europe-southwest1", + "description": "Regional Endpoint", + "endpointUrl": "https://workflows.europe-southwest1.rep.googleapis.com/" + }, + { + "description": "Regional Endpoint", + "endpointUrl": "https://workflows.europe-west1.rep.googleapis.com/", + "location": "europe-west1" + }, + { + "location": "europe-west10", + "endpointUrl": "https://workflows.europe-west10.rep.googleapis.com/", + "description": "Regional Endpoint" + }, + { + "description": "Regional Endpoint", + "location": "europe-west12", + "endpointUrl": "https://workflows.europe-west12.rep.googleapis.com/" + }, + { + "endpointUrl": "https://workflows.europe-west2.rep.googleapis.com/", + "location": "europe-west2", + "description": "Regional Endpoint" + }, + { + "endpointUrl": "https://workflows.europe-west3.rep.googleapis.com/", + "description": "Regional Endpoint", + "location": "europe-west3" + }, + { + "endpointUrl": "https://workflows.europe-west4.rep.googleapis.com/", + "location": "europe-west4", + "description": "Regional Endpoint" + }, + { + "endpointUrl": "https://workflows.europe-west6.rep.googleapis.com/", + "description": "Regional Endpoint", + "location": "europe-west6" + }, + { + "endpointUrl": "https://workflows.europe-west8.rep.googleapis.com/", + "location": "europe-west8", + "description": "Regional Endpoint" + }, + { + "endpointUrl": "https://workflows.europe-west9.rep.googleapis.com/", + "description": "Regional Endpoint", + "location": "europe-west9" + }, + { + "endpointUrl": "https://workflows.me-central1.rep.googleapis.com/", + "description": "Regional Endpoint", + "location": "me-central1" + }, + { + "location": "me-central2", + "endpointUrl": "https://workflows.me-central2.rep.googleapis.com/", + "description": "Regional Endpoint" + }, + { + "description": "Regional Endpoint", + "endpointUrl": "https://workflows.me-west1.rep.googleapis.com/", + "location": "me-west1" + }, + { + "location": "northamerica-northeast1", + "description": "Regional Endpoint", + "endpointUrl": "https://workflows.northamerica-northeast1.rep.googleapis.com/" + }, + { + "description": "Regional Endpoint", + "location": "northamerica-northeast2", + "endpointUrl": "https://workflows.northamerica-northeast2.rep.googleapis.com/" + }, + { + "description": "Regional Endpoint", + "location": "northamerica-south1", + "endpointUrl": "https://workflows.northamerica-south1.rep.googleapis.com/" + }, + { + "description": "Regional Endpoint", + "location": "southamerica-east1", + "endpointUrl": "https://workflows.southamerica-east1.rep.googleapis.com/" + }, + { + "location": "southamerica-west1", + "description": "Regional Endpoint", + "endpointUrl": "https://workflows.southamerica-west1.rep.googleapis.com/" + }, + { + "location": "us-central1", + "endpointUrl": "https://workflows.us-central1.rep.googleapis.com/", + "description": "Regional Endpoint" + }, + { + "description": "Regional Endpoint", + "location": "us-east1", + "endpointUrl": "https://workflows.us-east1.rep.googleapis.com/" + }, + { + "description": "Regional Endpoint", + "location": "us-east4", + "endpointUrl": "https://workflows.us-east4.rep.googleapis.com/" + }, + { + "location": "us-east5", + "description": "Regional Endpoint", + "endpointUrl": "https://workflows.us-east5.rep.googleapis.com/" + }, + { + "description": "Regional Endpoint", + "endpointUrl": "https://workflows.us-south1.rep.googleapis.com/", + "location": "us-south1" + }, + { + "endpointUrl": "https://workflows.us-west1.rep.googleapis.com/", + "description": "Regional Endpoint", + "location": "us-west1" + }, + { + "endpointUrl": "https://workflows.us-west2.rep.googleapis.com/", + "location": "us-west2", + "description": "Regional Endpoint" + }, + { + "location": "us-west3", + "endpointUrl": "https://workflows.us-west3.rep.googleapis.com/", + "description": "Regional Endpoint" + }, + { + "location": "us-west4", + "endpointUrl": "https://workflows.us-west4.rep.googleapis.com/", + "description": "Regional Endpoint" + }, + { + "description": "Regional Endpoint", + "location": "us-west8", + "endpointUrl": "https://workflows.us-west8.rep.googleapis.com/" + } + ] +} diff --git a/tests/src/google/discovery/test_discovery.cpp b/tests/src/google/discovery/test_discovery.cpp new file mode 100644 index 0000000..935dbf7 --- /dev/null +++ b/tests/src/google/discovery/test_discovery.cpp @@ -0,0 +1,502 @@ +// Discovery engine: deterministic local tests. The documents under +// fixtures/ are REAL Discovery documents saved verbatim from Google on +// 2026-10-25 (never edited); the engine's behavior against all 310 +// reachable documents is checked by tools/check_discovery_directory.py. +// MockTransport only records what the engine sends; no Google reply is +// scripted to prove a behavior. +#include +#include + +#include "framework.h" +#include "clients/transports/mock_transport.h" +#include "google/credentials.h" +#include "google/discovery/caps.h" +#include "google/discovery/engine.h" +#include "google/discovery/validate.h" + +using namespace molo::google; +using namespace molo::google::discovery; +using HttpMethod = molo::clients::Method; +using molo::clients::body::Json; +using molo::clients::transports::MockTransport; + +namespace { + +const char* kFixtures = "tests/src/google/discovery/fixtures"; + +// Counts fetches so cache behavior is observable. +struct CountingSource : DocumentSource { + FileDocumentSource inner{kFixtures}; + int fetches = 0; + std::string fetch(const std::string& url, std::size_t max) override { + ++fetches; + return inner.fetch(url, max); + } +}; + +struct Rig { + MockTransport* mock; + std::shared_ptr client; + std::shared_ptr catalog; + std::unique_ptr engine; + Rig(bool key, bool oauth) { + auto t = std::make_unique(); + mock = t.get(); + client = std::make_shared(std::move(t), + key ? Credentials::api_key("KEYVALUE") : nullptr, + oauth ? Credentials::bearer("TOKENVALUE") : nullptr); + catalog = std::make_shared(std::make_shared(kFixtures)); + engine = std::make_unique(client, catalog); + } + // Answers the next request to `path` so the request can be inspected. + void accept(HttpMethod m, const std::string& path) { + molo::clients::HeaderMap h; + h.set("Content-Type", "application/json"); + mock->expect_response(m, path, molo::clients::ClientResponse(200, h, "{}")); + } +}; + +template +bool invalid(F&& f, const std::string& needle = "") { + try { + f(); + } catch (const GoogleError& e) { + return e.http_status() == 400 && e.status() == "INVALID_ARGUMENT" && + e.message().find(needle) != std::string::npos; + } + return false; +} + +Json obj(std::initializer_list> kv) { + Json o = Json::object(); + for (const auto& [k, v] : kv) o.set(k, v); + return o; +} + +// A minimal but structurally real document, for shapes no fixture holds. +Json doc_with(const Json& methods, const std::string& root = "https://x.googleapis.com/") { + Json d = obj({{"kind", Json("discovery#restDescription")}, {"protocol", Json("rest")}, + {"id", Json("x:v1")}, {"name", Json("x")}, {"version", Json("v1")}, + {"rootUrl", Json(root)}, {"servicePath", Json("")}, + {"schemas", Json::object()}}); + d.set("methods", methods); + return d; +} + +Json method(const std::string& id, const std::string& verb, const std::string& path, + const Json& params = Json::object()) { + return obj({{"id", Json(id)}, {"httpMethod", Json(verb)}, {"path", Json(path)}, + {"parameters", params}}); +} + +const Method& m_of(const Api& a, const char* id) { + const Method* m = a.find(id); + if (!m) FAIL(std::string("missing method ") + id); + return *m; +} + +} // namespace + +TEST(discovery, caps_are_hard_ceilings) { + static_assert(DiscoveryCaps::kMaxPages <= 50); + static_assert(DiscoveryCaps::kMaxDocBytes >= 6419010, "must hold compute v1"); + static_assert(DiscoveryCaps::kMinRefetch.count() >= 60); + ASSERT_EQ(DiscoveryCaps::kMaxItems, std::size_t{10000}); +} + +TEST(discovery, directory_lists_preferred_and_finds_versions) { + CountingSource src; + Directory d = Directory::parse(src.fetch(kDirectoryUrl, 1 << 20)); + ASSERT_EQ(d.items().size(), std::size_t{6}); + ASSERT_EQ(d.find("workflows")->version, std::string("v1")); + ASSERT_EQ(d.find("workflows", "v1beta")->id, std::string("workflows:v1beta")); + ASSERT_TRUE(d.find("workflows", "v9") == nullptr); + ASSERT_TRUE(d.find("nope") == nullptr); + ASSERT_EQ(d.search("WORK", 10).size(), std::size_t{1}); // preferred only + ASSERT_EQ(d.search("work", 10, false).size(), std::size_t{2}); + ASSERT_EQ(d.search("", 2).size(), std::size_t{2}); +} + +TEST(discovery, directory_rejects_hostile_items) { + ASSERT_THROW(Directory::parse("{}"), GoogleError); + ASSERT_THROW(Directory::parse("not json"), GoogleError); + std::string evil = + R"({"kind":"discovery#directoryList","items":[{"id":"a:v1","name":"a","version":"v1","discoveryRestUrl":"https://evil.example/doc"}]})"; + ASSERT_THROW(Directory::parse(evil), GoogleError); +} + +TEST(discovery, host_allowlist) { + ASSERT_TRUE(is_google_host_url("https://tasks.googleapis.com/")); + ASSERT_TRUE(is_google_host_url("https://workflows.us-central1.rep.googleapis.com/")); + ASSERT_FALSE(is_google_host_url("http://tasks.googleapis.com/")); + ASSERT_FALSE(is_google_host_url("https://googleapis.com/")); + ASSERT_FALSE(is_google_host_url("https://evilgoogleapis.com/")); + ASSERT_FALSE(is_google_host_url("https://x.googleapis.com.evil.example/")); + ASSERT_FALSE(is_google_host_url("https://user@x.googleapis.com/")); + ASSERT_FALSE(is_google_host_url("https://x.googleapis.com:8443/")); +} + +TEST(discovery, http_source_refuses_foreign_hosts_before_traffic) { + auto t = std::make_unique(); + MockTransport* mock = t.get(); + HttpDocumentSource src(std::move(t)); + ASSERT_TRUE(invalid([&] { src.fetch("https://evil.example/x", 1000); }, "googleapis.com")); + ASSERT_EQ(mock->send_count(), std::size_t{0}); +} + +TEST(discovery, parses_real_documents) { + Rig r(true, true); + auto tasks = r.catalog->api("tasks"); + ASSERT_EQ(tasks->id(), std::string("tasks:v1")); + ASSERT_EQ(tasks->methods().size(), std::size_t{14}); + const Method& list = m_of(*tasks, "tasks.tasks.list"); + ASSERT_TRUE(tasks->is_pageable(list)); + ASSERT_EQ(tasks->items_key(list), std::string("items")); + ASSERT_TRUE(list.needs_oauth()); + ASSERT_TRUE(list.support == Support::kFull); + auto wf = r.catalog->api("workflows:v1"); + ASSERT_TRUE(wf->endpoints().size() > 10); + ASSERT_TRUE(wf->is_operation_schema("Operation")); + ASSERT_EQ(wf->operation_getters().size(), std::size_t{1}); +} + +TEST(discovery, classifies_limited_shapes_from_real_documents) { + Rig r(true, true); + auto gm = r.catalog->api("groupsmigration"); + const Method& ins = m_of(*gm, "groupsmigration.archive.insert"); + ASSERT_TRUE(ins.support == Support::kLimited); + ASSERT_TRUE(ins.media.supported && ins.media.simple); + auto ad = r.catalog->api("admob"); + ASSERT_TRUE(m_of(*ad, "admob.accounts.mediationReport.generate").support == Support::kLimited); + ASSERT_TRUE(m_of(*ad, "admob.accounts.get").support == Support::kFull); // streamingType NONE +} + +TEST(discovery, classifies_rejected_shapes) { + Json bidi = method("x.a", "POST", "v1/a"); + bidi.set("streamingType", Json("BIDI")); + Json objp = method("x.b", "GET", "v1/b", obj({{"ext", obj({{"type", Json("object")}, {"location", Json("query")}})}})); + Json noparam = method("x.c", "GET", "v1/{id}/c"); + Json head = method("x.d", "HEAD", "v1/d"); + Api a = Api::parse(doc_with(obj({{"a", bidi}, {"b", objp}, {"c", noparam}, {"d", head}}))); + for (const char* id : {"x.a", "x.b", "x.c", "x.d"}) { + ASSERT_TRUE(m_of(a, id).support == Support::kRejected); + ASSERT_FALSE(m_of(a, id).notes.empty()); + } + CallArgs args; + ASSERT_TRUE(invalid([&] { Planner::plan(a, m_of(a, "x.a"), args); }, "cannot be called")); +} + +TEST(discovery, document_validation_rejects_untrusted_shapes) { + ASSERT_THROW(Api::parse(doc_with(Json::object(), "https://evil.example/")), GoogleError); + ASSERT_THROW(Api::parse(doc_with(Json::object(), "http://x.googleapis.com/")), GoogleError); + Json d = doc_with(Json::object()); + d.set("kind", Json("other")); + ASSERT_THROW(Api::parse(d), GoogleError); + Json ep = doc_with(Json::object()); + Json row = obj({{"location", Json("l")}, {"endpointUrl", Json("https://evil.example/")}}); + Json rows = Json::array(); + rows.push(row); + ep.set("endpoints", rows); + ASSERT_THROW(Api::parse(ep), GoogleError); +} + +TEST(discovery, plan_builds_the_url_and_picks_credentials) { + Rig r(true, true); + CallArgs a; + a.params.set("name", Json("sites/example.com")); + PreparedCall p = r.engine->plan("abusiveexperiencereport", "abusiveexperiencereport.sites.get", a); + ASSERT_EQ(p.endpoint.base_url, std::string("https://abusiveexperiencereport.googleapis.com")); + ASSERT_EQ(p.path, std::string("/v1/sites/example.com")); + ASSERT_TRUE(p.endpoint.prefer_key); // no scopes declared: the key + a.auth = AuthMode::kOAuth; + ASSERT_FALSE(r.engine->plan("abusiveexperiencereport", "abusiveexperiencereport.sites.get", a).endpoint.prefer_key); + CallArgs t; + t.params.set("tasklist", Json("L1")); + ASSERT_FALSE(r.engine->plan("tasks", "tasks.tasks.list", t).endpoint.prefer_key); // scoped: OAuth + t.auth = AuthMode::kKey; + ASSERT_TRUE(r.engine->plan("tasks", "tasks.tasks.list", t).endpoint.prefer_key); + ASSERT_THROW(auth_mode_from("magic"), GoogleError); + ASSERT_TRUE(auth_mode_from("") == AuthMode::kAuto); +} + +TEST(discovery, path_values_are_encoded_and_reserved_expansion_keeps_slashes) { + Rig r(true, true); + CallArgs t; + t.params.set("tasklist", Json("a b/c?x")); + ASSERT_EQ(r.engine->plan("tasks", "tasks.tasks.list", t).path, + std::string("/tasks/v1/lists/a%20b%2Fc%3Fx/tasks")); + CallArgs w; + w.params.set("name", Json("projects/p1/locations/us-central1/workflows/w 1")); + ASSERT_EQ(r.engine->plan("workflows", "workflows.projects.locations.workflows.get", w).path, + std::string("/v1/projects/p1/locations/us-central1/workflows/w%201")); +} + +TEST(discovery, dot_segments_and_patterns_are_rejected) { + Rig r(true, true); + CallArgs t; + t.params.set("tasklist", Json("..")); + ASSERT_TRUE(invalid([&] { r.engine->plan("tasks", "tasks.tasks.list", t); }, "dot segment")); + CallArgs w; + w.params.set("name", Json("projects/p/../x")); + ASSERT_TRUE(invalid([&] { r.engine->plan("workflows", "workflows.projects.locations.workflows.get", w); })); + w.params.set("name", Json("nonsense")); + ASSERT_TRUE(invalid([&] { r.engine->plan("workflows", "workflows.projects.locations.workflows.get", w); }, "does not match")); +} + +TEST(discovery, parameter_validation_before_traffic) { + Rig r(true, true); + CallArgs t; + t.params.set("tasklist", Json("L1")); + ASSERT_TRUE(invalid([&] { r.engine->plan("tasks", "tasks.tasks.list", CallArgs{}); }, "required parameter tasklist")); + t.params.set("nope", Json(1)); + ASSERT_TRUE(invalid([&] { r.engine->plan("tasks", "tasks.tasks.list", t); }, "no parameter 'nope'")); + t.params = obj({{"tasklist", Json("L1")}, {"maxResults", Json("abc")}}); + ASSERT_TRUE(invalid([&] { r.engine->plan("tasks", "tasks.tasks.list", t); }, "integer")); + t.params = obj({{"tasklist", Json("L1")}, {"showCompleted", Json("yes")}}); + ASSERT_TRUE(invalid([&] { r.engine->plan("tasks", "tasks.tasks.list", t); }, "true or false")); + t.params = obj({{"tasklist", Json("L1")}, {"maxResults", Json(20)}, {"showCompleted", Json(true)}, + {"dueMin", Json("2026-10-25T10:00:00Z")}}); + PreparedCall p = r.engine->plan("tasks", "tasks.tasks.list", t); + ASSERT_EQ(p.path, std::string("/tasks/v1/lists/L1/tasks?dueMin=2026-10-25T10%3A00%3A00Z&maxResults=20&showCompleted=true")); + ASSERT_EQ(r.mock->send_count(), std::size_t{0}); +} + +TEST(discovery, reserved_global_parameters_are_refused) { + Rig r(true, true); + for (const char* name : {"key", "access_token", "oauth_token", "alt", "callback", "uploadType"}) { + CallArgs a; + a.params = obj({{"tasklist", Json("L1")}, {name, Json("x")}}); + ASSERT_TRUE(invalid([&] { r.engine->plan("tasks", "tasks.tasks.list", a); }, name)); + } + CallArgs ok; + ok.params = obj({{"tasklist", Json("L1")}, {"fields", Json("items(id)")}, {"quotaUser", Json("u")}}); + ASSERT_TRUE(r.engine->plan("tasks", "tasks.tasks.list", ok).path.find("fields=items%28id%29") != std::string::npos); +} + +TEST(discovery, repeated_enum_and_range_checks) { + Param rep; + rep.name = "r"; + rep.type = "string"; + rep.location = "query"; + rep.repeated = true; + rep.enum_values = {"A", "B"}; + ASSERT_TRUE(invalid([&] { ParamValidator::values(rep, Json::object()); }, "scalar")); + Json arr = Json::array(); + arr.push(Json("A")); + arr.push(Json("B")); + ASSERT_EQ(ParamValidator::values(rep, arr).size(), std::size_t{2}); + ASSERT_EQ(ParamValidator::values(rep, Json("A")).size(), std::size_t{1}); + arr.push(Json("C")); + ASSERT_TRUE(invalid([&] { ParamValidator::values(rep, arr); }, "enum")); + Json many = Json::array(); + rep.enum_values.clear(); + for (int i = 0; i < 101; ++i) many.push(Json("v")); + ASSERT_TRUE(invalid([&] { ParamValidator::values(rep, many); }, "more than")); + Param n; + n.name = "n"; + n.type = "integer"; + n.format = "int32"; + n.location = "query"; + n.minimum = 1; + n.maximum = 100; + ASSERT_EQ(ParamValidator::values(n, Json(50))[0], std::string("50")); + ASSERT_TRUE(invalid([&] { ParamValidator::values(n, Json(0)); }, "minimum")); + ASSERT_TRUE(invalid([&] { ParamValidator::values(n, Json(101)); }, "maximum")); + ASSERT_TRUE(invalid([&] { ParamValidator::values(n, Json(1.5)); }, "integer")); + Param big; + big.name = "b"; + big.type = "string"; + big.format = "int64"; + big.location = "query"; + ASSERT_EQ(ParamValidator::values(big, Json("9007199254740993"))[0], std::string("9007199254740993")); + ASSERT_TRUE(invalid([&] { ParamValidator::values(big, Json("1x")); }, "64-bit")); + Param s; + s.name = "s"; + s.type = "string"; + s.location = "query"; + ASSERT_TRUE(invalid([&] { ParamValidator::values(s, Json(std::string(2049, 'a'))); }, "cap")); + ASSERT_TRUE(invalid([&] { ParamValidator::values(s, Json("a\nb")); }, "control")); + ASSERT_TRUE(invalid([&] { ParamValidator::values(s, Json()); }, "null")); +} + +TEST(discovery, repeated_parameter_from_a_real_document) { + Rig r(true, true); + CallArgs a; + a.params = obj({{"name", Json("projects/p1")}}); + Json types = Json::array(); + types.push(Json("a")); + types.push(Json("b")); + a.params.set("extraLocationTypes", types); + PreparedCall p = r.engine->plan("workflows", "workflows.projects.locations.list", a); + ASSERT_EQ(p.path, std::string("/v1/projects/p1/locations?extraLocationTypes=a&extraLocationTypes=b")); +} + +TEST(discovery, body_is_validated_against_the_schema) { + Rig r(true, true); + auto make = [&](Json body) { + CallArgs a; + a.params = obj({{"parent", Json("projects/p1/locations/us-central1")}, {"workflowId", Json("w1")}}); + a.body = std::move(body); + return r.engine->plan("workflows", "workflows.projects.locations.workflows.create", a); + }; + PreparedCall ok = make(obj({{"description", Json("d")}, {"labels", obj({{"k", Json("v")}})}, + {"callLogLevel", Json("LOG_ALL_CALLS")}})); + ASSERT_TRUE(ok.has_body); + ASSERT_TRUE(ok.http == HttpMethod::kPost); + ASSERT_TRUE(ok.operation); + ASSERT_TRUE(invalid([&] { make(obj({{"bogus", Json(1)}})); }, "no property bogus")); + ASSERT_TRUE(invalid([&] { make(obj({{"description", Json(5)}})); }, "must be a string")); + ASSERT_TRUE(invalid([&] { make(obj({{"callLogLevel", Json("LOUD")}})); }, "enum")); + ASSERT_TRUE(invalid([&] { make(obj({{"labels", obj({{"k", Json(1)}})}})); }, "/labels/k")); + ASSERT_TRUE(invalid([&] { make(Json("text")); }, "must be an object")); + Json deep = Json::object(); + ASSERT_TRUE(make(deep).has_body); // empty body is {} and valid +} + +TEST(discovery, body_rules_follow_the_method_declaration) { + Rig r(true, true); + CallArgs g; + g.params = obj({{"name", Json("projects/p1/locations/us-central1/workflows/w")}}); + g.body = obj({}); + ASSERT_TRUE(invalid([&] { r.engine->plan("workflows", "workflows.projects.locations.workflows.get", g); }, "no request body")); + CallArgs c; + c.params = obj({{"tasklist", Json("L1")}}); + PreparedCall clear = r.engine->plan("tasks", "tasks.tasks.clear", c); // POST without a request schema + ASSERT_TRUE(clear.has_body); + ASSERT_EQ(clear.body.dump(), std::string("{}")); +} + +TEST(discovery, regional_endpoints_come_from_the_document) { + Rig r(true, true); + CallArgs a; + a.params = obj({{"name", Json("projects/p1/locations/us-central1/workflows/w")}}); + a.location = "us-central1"; + PreparedCall p = r.engine->plan("workflows", "workflows.projects.locations.workflows.get", a); + ASSERT_EQ(p.endpoint.base_url, std::string("https://workflows.us-central1.rep.googleapis.com")); + a.location = "mars-north1"; + ASSERT_TRUE(invalid([&] { r.engine->plan("workflows", "workflows.projects.locations.workflows.get", a); }, "no endpoint for location")); + CallArgs t; + t.params = obj({{"tasklist", Json("L")}}); + t.location = "us-central1"; + ASSERT_TRUE(invalid([&] { r.engine->plan("tasks", "tasks.tasks.list", t); }, "no regional endpoints")); +} + +TEST(discovery, paging_arguments_are_checked_before_traffic) { + Rig r(true, true); + CallArgs a; + a.params = obj({{"name", Json("sites/example.com")}}); + a.max_pages = 3; + ASSERT_TRUE(invalid([&] { r.engine->call("abusiveexperiencereport", "abusiveexperiencereport.sites.get", a); }, "not pageable")); + a.max_pages = 0; + ASSERT_TRUE(invalid([&] { r.engine->call("abusiveexperiencereport", "abusiveexperiencereport.sites.get", a); }, "max_pages")); + a.max_pages = DiscoveryCaps::kMaxPages + 1; + ASSERT_TRUE(invalid([&] { r.engine->call("abusiveexperiencereport", "abusiveexperiencereport.sites.get", a); }, "max_pages")); + ASSERT_EQ(r.mock->send_count(), std::size_t{0}); +} + +TEST(discovery, unknown_api_and_method_are_named) { + Rig r(true, true); + try { + r.engine->plan("nosuchapi", "x.y", CallArgs{}); + FAIL("expected throw"); + } catch (const GoogleError& e) { + ASSERT_EQ(e.http_status(), 404); + } + try { + r.engine->plan("tasks", "tasks.nope", CallArgs{}); + FAIL("expected throw"); + } catch (const GoogleError& e) { + ASSERT_EQ(e.http_status(), 404); + } + ASSERT_TRUE(invalid([&] { r.engine->plan("../etc", "x", CallArgs{}); }, "name or name:version")); +} + +TEST(discovery, no_credential_is_503_after_validation) { + Rig r(false, false); + CallArgs a; + a.params = obj({{"tasklist", Json("L1")}}); + try { + r.engine->call("tasks", "tasks.tasks.list", a); + FAIL("expected throw"); + } catch (const NotConfiguredError& e) { + ASSERT_EQ(e.http_status(), 503); + } + ASSERT_TRUE(invalid([&] { r.engine->call("tasks", "tasks.tasks.list", CallArgs{}); }, "required")); + ASSERT_EQ(r.mock->send_count(), std::size_t{0}); +} + +TEST(discovery, call_sends_the_planned_request_with_the_chosen_credential) { + Rig r(true, true); + r.accept(HttpMethod::kPost, "/tasks/v1/lists/L1/tasks"); + CallArgs a; + a.params = obj({{"tasklist", Json("L1")}}); + a.body = obj({{"title", Json("write tests")}}); + CallResult res = r.engine->call("tasks", "tasks.tasks.insert", a); + ASSERT_EQ(res.pages, std::size_t{1}); + auto seen = r.mock->seen(); + ASSERT_EQ(seen.size(), std::size_t{1}); + ASSERT_EQ(seen[0].method, std::string("POST")); + ASSERT_EQ(seen[0].body, std::string("{\"title\":\"write tests\"}")); + ASSERT_EQ(seen[0].headers.get("Authorization").value_or(""), std::string("Bearer TOKENVALUE")); + + Rig k(true, true); + k.accept(HttpMethod::kGet, "/v1/sites/example.com"); + CallArgs g; + g.params = obj({{"name", Json("sites/example.com")}}); + k.engine->call("abusiveexperiencereport", "abusiveexperiencereport.sites.get", g); + ASSERT_EQ(k.mock->seen()[0].headers.get("x-goog-api-key").value_or(""), std::string("KEYVALUE")); +} + +TEST(discovery, catalog_caches_expires_and_honors_the_refetch_floor) { + auto src = std::make_shared(); + auto now = std::make_shared( + std::chrono::steady_clock::time_point{} + std::chrono::hours(1000)); + Catalog cat(src, [now] { return *now; }); + cat.api("tasks"); + ASSERT_EQ(src->fetches, 2); // directory + document + cat.api("tasks"); + cat.api("tasks:v1"); + ASSERT_EQ(src->fetches, 2); + ASSERT_EQ(cat.stats().cached, std::size_t{1}); + // forgetting inside the floor still serves the cached document + cat.forget("tasks"); + *now += std::chrono::seconds(30); + cat.api("tasks"); + ASSERT_EQ(src->fetches, 2); + // past the floor the forgotten document is fetched again + *now += std::chrono::seconds(31); + cat.api("tasks"); + ASSERT_EQ(src->fetches, 3); + // past the max age, directory and document are both refetched + *now += DiscoveryCaps::kMaxAge + std::chrono::seconds(1); + cat.api("tasks"); + ASSERT_EQ(src->fetches, 5); +} + +TEST(discovery, catalog_rejects_a_document_that_is_not_the_one_listed) { + struct Swap : DocumentSource { + FileDocumentSource inner{kFixtures}; + std::string fetch(const std::string& url, std::size_t max) override { + if (url.find("tasks") != std::string::npos && url.find("directory") == std::string::npos && + url != kDirectoryUrl) + return inner.fetch("https://workflows.googleapis.com/$discovery/rest?version=v1", max); + return inner.fetch(url, max); + } + }; + Catalog cat(std::make_shared()); + ASSERT_THROW(cat.api("tasks"), GoogleError); +} + +TEST(discovery, list_and_describe_report_the_declaration) { + Rig r(true, true); + Json apis = r.engine->list_apis("task", 10); + ASSERT_EQ(apis.as_array().size(), std::size_t{1}); + Json ms = r.engine->list_methods("tasks", "tasks.tasks.", 3); + ASSERT_EQ(ms.find("methods")->as_array().size(), std::size_t{3}); + ASSERT_TRUE(ms.find("truncated")->as_bool()); + Json d = r.engine->describe("tasks", "tasks.tasks.list"); + ASSERT_TRUE(d.find("pageable")->as_bool()); + ASSERT_EQ(d.find("itemsKey")->as_string(), std::string("items")); + Json wf = r.engine->describe("workflows", "workflows.projects.locations.workflows.delete"); + ASSERT_TRUE(wf.find("operation")->as_bool()); +} diff --git a/tests/src/google/discovery/test_discovery_live.cpp b/tests/src/google/discovery/test_discovery_live.cpp new file mode 100644 index 0000000..3b8bb8a --- /dev/null +++ b/tests/src/google/discovery/test_discovery_live.cpp @@ -0,0 +1,138 @@ +// Opt-in live checks (MOLO_LIVE_NET=1): the real directory, real documents +// fetched over HTTPS with no credential, and read-only calls on the project +// behind GOOGLE_ACCESS_TOKEN (GOOGLE_CLOUD_PROJECT, default lisa-448004). +// Nothing is created, changed or deleted. A disabled API is reported loudly. +#include +#include + +#include "framework.h" +#include "google/credentials.h" +#include "google/discovery/caps.h" +#include "google/discovery/engine.h" + +using namespace molo::google; +using namespace molo::google::discovery; +using molo::clients::body::Json; + +namespace { +bool net() { return std::getenv("MOLO_LIVE_NET") != nullptr; } +bool authed() { return net() && !Credentials::env("GOOGLE_ACCESS_TOKEN").empty(); } +std::string project() { + std::string p = Credentials::env("GOOGLE_CLOUD_PROJECT"); + return p.empty() ? "lisa-448004" : p; +} +void skip(const char* name, const char* why) { + std::cout << "SKIP " << name << " (" << why << ")\n"; +} +Engine make_engine(std::shared_ptr& cat) { + cat = std::make_shared(std::make_shared()); + return Engine(GoogleClient::from_env(), cat); +} +} // namespace + +TEST(discovery_live, directory_and_documents_from_google) { + if (!net()) return skip("discovery_live.directory_and_documents_from_google", "set MOLO_LIVE_NET=1"); + Catalog cat(std::make_shared()); + auto dir = cat.directory(); + std::cout << " directory: " << dir->items().size() << " items, " << dir->preferred_count() + << " preferred\n"; + ASSERT_TRUE(dir->preferred_count() >= 300); + for (const char* name : {"serviceusage", "cloudresourcemanager", "tasks", "workflows"}) { + auto api = cat.api(name); + std::cout << " " << api->id() << ": " << api->methods().size() << " methods, " + << api->endpoints().size() << " regional endpoints, " << api->raw_bytes << " bytes\n"; + ASSERT_TRUE(api->methods().size() > 5); + } + ASSERT_EQ(cat.stats().fetches, std::size_t{5}); +} + +TEST(discovery_live, sizes_of_the_largest_documents_fit_the_caps) { + if (!net()) return skip("discovery_live.sizes_of_the_largest_documents_fit_the_caps", "set MOLO_LIVE_NET=1"); + Catalog cat(std::make_shared()); + for (const char* name : {"compute", "aiplatform"}) { + auto api = cat.api(name); + std::cout << " " << api->id() << ": " << api->methods().size() << " methods, " << api->raw_bytes + << " bytes\n"; + ASSERT_TRUE(api->raw_bytes <= DiscoveryCaps::kMaxDocBytes); + ASSERT_TRUE(api->methods().size() <= DiscoveryCaps::kMaxMethods); + } +} + +TEST(discovery_live, read_only_calls_page_and_get) { + if (!authed()) return skip("discovery_live.read_only_calls_page_and_get", "needs MOLO_LIVE_NET=1 and GOOGLE_ACCESS_TOKEN"); + std::shared_ptr cat; + Engine e = make_engine(cat); + try { + CallArgs get; + get.params.set("name", Json("projects/" + project() + "/services/serviceusage.googleapis.com")); + CallResult p = e.call("serviceusage", "serviceusage.services.get", get); + std::cout << " serviceusage.services.get: " << p.body.dump().substr(0, 120) << "\n"; + ASSERT_TRUE(p.body.find("name") != nullptr); + + CallArgs list; + list.params = Json::object(); + list.params.set("parent", Json("projects/" + project())); + list.params.set("pageSize", Json(3)); + list.params.set("filter", Json("state:ENABLED")); + list.max_pages = 3; + CallResult s = e.call("serviceusage", "serviceusage.services.list", list); + std::cout << " serviceusage.services.list: pages=" << s.pages << " items=" + << s.body.find("services")->as_array().size() << " truncated=" << s.truncated << "\n"; + ASSERT_EQ(s.items_key, std::string("services")); + ASSERT_TRUE(s.pages >= 2); + ASSERT_TRUE(s.body.find("services")->as_array().size() > 3); + ASSERT_TRUE(s.truncated || s.body.find("nextPageToken") == nullptr); + } catch (const GoogleError& err) { + if (err.http_status() == 403 || err.status() == "PERMISSION_DENIED") + std::cout << " API NOT ENABLED or permission denied: " << err.message() << "\n"; + throw; + } +} + +TEST(discovery_live, operation_reply_is_detected_and_a_missing_one_is_4xx) { + if (!authed()) return skip("discovery_live.operation_reply_is_detected_and_a_missing_one_is_4xx", "needs MOLO_LIVE_NET=1 and GOOGLE_ACCESS_TOKEN"); + std::shared_ptr cat; + Engine e = make_engine(cat); + CallArgs a; + a.params.set("name", Json("operations/does-not-exist-0000")); + PreparedCall p = e.plan("serviceusage", "serviceusage.operations.get", a); + ASSERT_TRUE(p.operation); + try { + e.call("serviceusage", "serviceusage.operations.get", a); + FAIL("expected a 4xx"); + } catch (const GoogleError& err) { + std::cout << " serviceusage.operations.get(missing): " << err.http_status() << " " << err.status() << "\n"; + ASSERT_TRUE(err.http_status() == 404 || err.http_status() == 400); // Google rejects the name shape with 400 + } +} + +TEST(discovery_live, completed_operation_with_wait_returns_without_polling) { + if (!authed()) return skip("discovery_live.completed_operation_with_wait_returns_without_polling", "needs MOLO_LIVE_NET=1 and GOOGLE_ACCESS_TOKEN"); + std::shared_ptr cat; + Engine e = make_engine(cat); + // Find a finished operation of the project, then fetch it again with wait. + CallArgs list; + list.params.set("name", Json("operations")); + list.params.set("pageSize", Json(5)); + list.params.set("filter", Json("done=true")); + try { + CallResult l = e.call("serviceusage", "serviceusage.operations.list", list); + const Json* ops = l.body.find("operations"); + if (!ops || ops->as_array().empty()) { + std::cout << " no finished operations visible to this identity: wait path not exercised\n"; + return; + } + std::string name = ops->as_array()[0].find("name")->as_string(); + CallArgs get; + get.params.set("name", Json(name)); + get.wait = true; + CallResult g = e.call("serviceusage", "serviceusage.operations.get", get); + std::cout << " waited on " << name << ": done=" << g.body.find("done")->as_bool() << "\n"; + ASSERT_TRUE(g.operation_waited); + ASSERT_TRUE(g.body.find("done")->as_bool()); + } catch (const GoogleError& err) { + std::cout << " NOT EXERCISED (wait polling needs a finished operation): " << err.http_status() << " " << err.status() << ": " << err.message() << "\n"; + if (err.http_status() == 403 || err.http_status() == 400) return; // loud, not a failure of the engine + throw; + } +}