#pragma once // Computer use: browser actions in a computer use sandbox (navigate, click, // type, screenshot). Docs describe the action set and that each action is a // Chrome DevTools Protocol command ({"command","params"}); they send it to // the sandbox data plane with a signed JWT. This class carries the same CDP // commands as an application/json Chunk through sandboxEnvironments.execute, // the only sandbox call in the REST surface. Whether the server routes that // Chunk to the browser is unverified (SPEC Unverified). #include #include "google/sandbox/sandbox_service.h" namespace molo::google::sandbox { struct ActionResult { clients::body::Json result; // The CDP result object ({} when none). ExecutionResult exec; // Text, files and skip reasons of the reply. std::string screenshot_png; // Raw PNG bytes, set by screenshot() only. }; class ComputerUse : public SandboxService { public: explicit ComputerUse(std::shared_ptr client); ActionResult navigate(const std::string& sandbox_name, const std::string& url) const; // Left click at viewport coordinates: mousePressed then mouseReleased. ActionResult click(const std::string& sandbox_name, int x, int y) const; ActionResult type_text(const std::string& sandbox_name, const std::string& text) const; ActionResult screenshot(const std::string& sandbox_name) const; static void validate_url(const std::string& url); // http(s) only, capped static void validate_point(int x, int y); static void validate_text(const std::string& text); // CDP command rows (compared to the CDP names in tests). static clients::body::Json navigate_params(const std::string& url); static clients::body::Json mouse_params(const char* type, int x, int y); static clients::body::Json insert_text_params(const std::string& text); private: ActionResult send(const std::string& sandbox_name, const std::string& command, const clients::body::Json& params) const; }; } // namespace molo::google::sandbox