2026-10-24 04:36:39 +02:00
|
|
|
#pragma once
|
|
|
|
|
|
|
|
|
|
// Computer use: browser actions in a computer use sandbox (navigate, click,
|
2026-10-24 05:30:44 +02:00
|
|
|
// type, screenshot) over the sandbox data plane, as Google documents it:
|
|
|
|
|
// POST /cdp with {"command": "<CDP method>", "params": {...}} and a signed
|
|
|
|
|
// JWT (see data_plane.h). The commands are Chrome DevTools Protocol names.
|
|
|
|
|
// Tabs and the CDP WebSocket are exposed through the same connector.
|
2026-10-24 04:36:39 +02:00
|
|
|
|
2026-10-24 05:30:44 +02:00
|
|
|
#include <memory>
|
2026-10-24 04:36:39 +02:00
|
|
|
#include <string>
|
|
|
|
|
|
2026-10-24 05:30:44 +02:00
|
|
|
#include "google/sandbox/cdp_session.h"
|
|
|
|
|
#include "google/sandbox/data_plane_connector.h"
|
2026-10-24 04:36:39 +02:00
|
|
|
#include "google/sandbox/sandbox_service.h"
|
|
|
|
|
|
|
|
|
|
namespace molo::google::sandbox {
|
|
|
|
|
|
|
|
|
|
struct ActionResult {
|
|
|
|
|
clients::body::Json result; // The CDP result object ({} when none).
|
2026-10-24 05:30:44 +02:00
|
|
|
int status = 0; // HTTP status of the data plane call.
|
2026-10-24 04:36:39 +02:00
|
|
|
std::string screenshot_png; // Raw PNG bytes, set by screenshot() only.
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
class ComputerUse : public SandboxService {
|
|
|
|
|
public:
|
2026-10-24 05:30:44 +02:00
|
|
|
// connector null: one built from DataPlaneOptions::from_env().
|
|
|
|
|
explicit ComputerUse(std::shared_ptr<GoogleClient> client,
|
|
|
|
|
std::shared_ptr<DataPlaneConnector> connector = nullptr);
|
2026-10-24 04:36:39 +02:00
|
|
|
|
|
|
|
|
ActionResult navigate(const std::string& sandbox_name, const std::string& url) const;
|
|
|
|
|
// Left click at viewport coordinates: mousePressed then mouseReleased.
|
|
|
|
|
ActionResult click(const std::string& sandbox_name, int x, int y) const;
|
|
|
|
|
ActionResult type_text(const std::string& sandbox_name, const std::string& text) const;
|
|
|
|
|
ActionResult screenshot(const std::string& sandbox_name) const;
|
2026-10-24 05:30:44 +02:00
|
|
|
// Any CDP command (Domain.method) on the active tab.
|
|
|
|
|
ActionResult command(const std::string& sandbox_name, const std::string& cdp_method,
|
|
|
|
|
const clients::body::Json& params) const;
|
|
|
|
|
|
|
|
|
|
// Data plane routes.
|
|
|
|
|
clients::body::Json health(const std::string& sandbox_name) const;
|
|
|
|
|
clients::body::Json list_tabs(const std::string& sandbox_name) const;
|
|
|
|
|
clients::body::Json create_tab(const std::string& sandbox_name) const;
|
|
|
|
|
clients::body::Json activate_tab(const std::string& sandbox_name, const std::string& tab_id) const;
|
|
|
|
|
clients::body::Json close_tab(const std::string& sandbox_name, const std::string& tab_id) const;
|
|
|
|
|
// The CDP WebSocket (not yet connected; call connect()).
|
|
|
|
|
std::unique_ptr<CdpSession> open_cdp(const std::string& sandbox_name,
|
|
|
|
|
CdpOptions options = {}) const;
|
2026-10-24 04:36:39 +02:00
|
|
|
|
|
|
|
|
static void validate_url(const std::string& url); // http(s) only, capped
|
|
|
|
|
static void validate_point(int x, int y);
|
|
|
|
|
static void validate_text(const std::string& text);
|
|
|
|
|
|
|
|
|
|
// CDP command rows (compared to the CDP names in tests).
|
|
|
|
|
static clients::body::Json navigate_params(const std::string& url);
|
|
|
|
|
static clients::body::Json mouse_params(const char* type, int x, int y);
|
|
|
|
|
static clients::body::Json insert_text_params(const std::string& text);
|
2026-10-24 05:30:44 +02:00
|
|
|
static clients::body::Json screenshot_params();
|
|
|
|
|
// Pulls the result object out of a /cdp reply: {"result": {...}} or the
|
|
|
|
|
// object itself (the reply shape is unverified, SPEC section 7).
|
|
|
|
|
static clients::body::Json result_of(const DataPlaneReply& reply);
|
|
|
|
|
// Base64 PNG in "data" of a captureScreenshot result; empty when absent or
|
|
|
|
|
// over SandboxCaps::kMaxScreenshotBytes.
|
|
|
|
|
static std::string screenshot_of(const clients::body::Json& result);
|
2026-10-24 04:36:39 +02:00
|
|
|
|
|
|
|
|
private:
|
2026-10-24 05:30:44 +02:00
|
|
|
ActionResult send(const std::string& sandbox_name, const std::string& cdp_method,
|
2026-10-24 04:36:39 +02:00
|
|
|
const clients::body::Json& params) const;
|
2026-10-24 05:30:44 +02:00
|
|
|
|
|
|
|
|
std::shared_ptr<DataPlaneConnector> connector_;
|
2026-10-24 04:36:39 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
} // namespace molo::google::sandbox
|