#pragma once // Computer use: browser actions in a computer use sandbox (navigate, click, // type, screenshot) over the sandbox data plane, as Google documents it: // POST /cdp with {"command": "", "params": {...}} and a signed // JWT (see data_plane.h). The commands are Chrome DevTools Protocol names. // Tabs and the CDP WebSocket are exposed through the same connector. #include #include #include "google/sandbox/cdp_session.h" #include "google/sandbox/data_plane_connector.h" #include "google/sandbox/sandbox_service.h" namespace molo::google::sandbox { struct ActionResult { clients::body::Json result; // The CDP result object ({} when none). int status = 0; // HTTP status of the data plane call. std::string screenshot_png; // Raw PNG bytes, set by screenshot() only. }; class ComputerUse : public SandboxService { public: // connector null: one built from DataPlaneOptions::from_env(). explicit ComputerUse(std::shared_ptr client, std::shared_ptr connector = nullptr); ActionResult navigate(const std::string& sandbox_name, const std::string& url) const; // Left click at viewport coordinates: mousePressed then mouseReleased. ActionResult click(const std::string& sandbox_name, int x, int y) const; ActionResult type_text(const std::string& sandbox_name, const std::string& text) const; ActionResult screenshot(const std::string& sandbox_name) const; // Any CDP command (Domain.method) on the active tab. ActionResult command(const std::string& sandbox_name, const std::string& cdp_method, const clients::body::Json& params) const; // Data plane routes. clients::body::Json health(const std::string& sandbox_name) const; clients::body::Json list_tabs(const std::string& sandbox_name) const; clients::body::Json create_tab(const std::string& sandbox_name) const; clients::body::Json activate_tab(const std::string& sandbox_name, const std::string& tab_id) const; clients::body::Json close_tab(const std::string& sandbox_name, const std::string& tab_id) const; // The CDP WebSocket (not yet connected; call connect()). std::unique_ptr open_cdp(const std::string& sandbox_name, CdpOptions options = {}) const; static void validate_url(const std::string& url); // http(s) only, capped static void validate_point(int x, int y); static void validate_text(const std::string& text); // CDP command rows (compared to the CDP names in tests). static clients::body::Json navigate_params(const std::string& url); static clients::body::Json mouse_params(const char* type, int x, int y); static clients::body::Json insert_text_params(const std::string& text); static clients::body::Json screenshot_params(); // Pulls the result object out of a /cdp reply: {"result": {...}} or the // object itself (the reply shape is unverified, SPEC section 7). static clients::body::Json result_of(const DataPlaneReply& reply); // Base64 PNG in "data" of a captureScreenshot result; empty when absent or // over SandboxCaps::kMaxScreenshotBytes. static std::string screenshot_of(const clients::body::Json& result); private: ActionResult send(const std::string& sandbox_name, const std::string& cdp_method, const clients::body::Json& params) const; std::shared_ptr connector_; }; } // namespace molo::google::sandbox