whatsapp: fix link/login 401, add image send, contact sync and search; harden failover
Root causes, each reproduced against the live server or endpoint: - Server-only JIDs decoded as "@s.whatsapp.net" and pairing replies copied them into `to`, so the device was never registered and every login returned 401. Decoder/encoder fixed, replies use s.whatsapp.net, ADV hosted prefixes follow whatsmeow. The link wizard now waits for the restart, logs in, uploads prekeys and retries once with a fresh device on 401/403. - Closed sockets made every receive loop spin until its deadline (the 7 minute CPU burn). Loops stop on a closed stream. - whatsapp_send had no image argument and send_image was a stub. Added WaMedia (byte-exact against 8 wa-go golden vectors), media_conn upload, image send and the DeviceSentMessage copy for the user's other devices. - Phone contacts never reached the client: added app-state sync (WaAppState, WaReceiver, LTHash and MAC checks verified against a whatsmeow-built oracle) and the historySyncConfig declaration at registration. - Contact lookup: FTS5 trigram plus fuzzy ranking, ambiguity is reported and never guessed, global search lists contacts. Send caps (10/min, 200/day) are enforced and survive unlink. Failover and agent: - Every failed backend candidate is printed the moment it fails. - muse runs with low reasoning effort, an empty preset and one stable workspace (36.5s down to 2.8s on the real prompt); Ctrl-C no longer counts as a backend failure; CLI backends use the circuit breaker; opencode runs are capped at 30s. - Orphan tool results are dropped before sending (devplace HTTP 400, reproduced); compaction no longer starts its window on a tool result. - Terminal art pasted as a prompt is refused before any model call. Tests: whatsapp_test, muse_flags_test, llm_history_test; full suite 40 steps (agent.e2e reads live tmux panes and is flaky, green on re-run). Research archived under research/whatsapp-automation. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -4,8 +4,13 @@
|
||||
// checks of the wire codec, crypto, Noise handshake and Signal session. The
|
||||
// live pairing/send paths need a phone and are not covered here. See
|
||||
// docs/whatsapp.md.
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "json.h"
|
||||
@@ -15,6 +20,10 @@
|
||||
#include "wa_noise.h"
|
||||
#include "wa_proto.h"
|
||||
#include "wa_signal.h"
|
||||
#include "wa_appstate.h"
|
||||
#include "wa_media.h"
|
||||
#include "wa_receiver.h"
|
||||
#include "wa_store.h"
|
||||
|
||||
using namespace detz::wa;
|
||||
using detz::json::Value;
|
||||
@@ -362,15 +371,306 @@ static void test_device() {
|
||||
check(ub && !ub->paired(), "unpaired device serialize/deserialize round-trip");
|
||||
}
|
||||
|
||||
// Regression (docs/whatsapp.md section 14): a server-only JID decoded to "@s.whatsapp.net", and the
|
||||
// pairing reply copied it as `to`, so the server refused pair-device-sign and login ended in 401.
|
||||
static void test_server_jid() {
|
||||
WaNode n("iq");
|
||||
n.attrs["to"] = "s.whatsapp.net";
|
||||
n.attrs["from"] = "31630045827@s.whatsapp.net";
|
||||
Bytes wire{0}; // 1-byte flags header (0 = uncompressed), as WaClient::send_node frames it
|
||||
Bytes body = WaBinary::encode(n);
|
||||
wire.insert(wire.end(), body.begin(), body.end());
|
||||
WaNode back = WaBinary::decode(WaBinary::unpack(wire));
|
||||
check(back.attr("to") == "s.whatsapp.net", "server-only JID round-trips as s.whatsapp.net");
|
||||
check(back.attr("from") == "31630045827@s.whatsapp.net", "user JID round-trips unchanged");
|
||||
}
|
||||
|
||||
// WaMedia against wa-go's golden vectors (Baileys-generated, MIT): every IV, derived key,
|
||||
// ciphertext+MAC and both hashes must match byte for byte.
|
||||
static void test_media() {
|
||||
Value d = Value::loads(slurp(path("media_vectors.json")));
|
||||
Bytes media_key = fromhex(d.get("mediaKeyHex").str());
|
||||
int ok = 0, total = 0;
|
||||
for (const auto& c : d.get("cases").items()) {
|
||||
const Value& pt = d.get("plaintexts").get(c.get("plaintext").str());
|
||||
Bytes plain = fromhex(pt.get("plaintextHex").str());
|
||||
std::string info = c.get("info").str();
|
||||
WaMedia::Keys k = WaMedia::expand(media_key, info);
|
||||
WaMedia::Encrypted e = WaMedia::encrypt(plain, info, media_key);
|
||||
bool keys_ok = tohex(k.iv) == c.get("iv").str() && tohex(k.cipher_key) == c.get("cipherKey").str() &&
|
||||
tohex(k.mac_key) == c.get("macKey").str() && tohex(k.ref_key) == c.get("refKey").str();
|
||||
bool blob_ok = tohex(e.data_to_upload) == c.get("ciphertextHex").str();
|
||||
bool hash_ok = tohex(e.file_sha256) == c.get("fileSha256").str() && tohex(e.file_enc_sha256) == c.get("fileEncSha256").str();
|
||||
bool len_ok = e.file_length == plain.size();
|
||||
++total;
|
||||
if (keys_ok && blob_ok && hash_ok && len_ok) ++ok;
|
||||
else std::printf(" mismatch: %s %s keys=%d blob=%d hash=%d len=%d\n", c.get("plaintext").str().c_str(), info.c_str(), keys_ok, blob_ok, hash_ok, len_ok);
|
||||
}
|
||||
check(total > 0 && ok == total, "media encrypt matches wa-go golden vectors (" + std::to_string(ok) + "/" + std::to_string(total) + ")");
|
||||
check(WaMedia::token(WaCrypto::sha256(Bytes{'a'})).size() == 44, "upload token is base64url with padding (44 chars for 32 bytes)");
|
||||
}
|
||||
|
||||
|
||||
// ---- app-state sync (docs/whatsapp.md section 14) ----
|
||||
// Oracle: tests/whatsapp_data/appstate_vectors.json, generated by appstate_oracle.go with whatsmeow's own
|
||||
// lthash, hkdfutil and protobuf packages. Nothing here is produced by the code under test.
|
||||
|
||||
static std::string make_temp_home() {
|
||||
std::string tmpl = "/tmp/wa_test_XXXXXX";
|
||||
char* d = ::mkdtemp(tmpl.data());
|
||||
return d ? std::string(d) : std::string();
|
||||
}
|
||||
|
||||
static void test_lthash_and_keys() {
|
||||
std::printf("[appstate: keys and LTHash]\n");
|
||||
Value v = Value::loads(slurp(path("appstate_vectors.json")));
|
||||
Bytes key_data = fromhex(v.get("key_data").str());
|
||||
WaAppStateKeys k = WaAppStateKeys::expand(key_data);
|
||||
std::string want = v.get("expanded_keys").str();
|
||||
std::string got = tohex(k.index) + tohex(k.value_encryption) + tohex(k.value_mac) + tohex(k.snapshot_mac) + tohex(k.patch_mac);
|
||||
check(got == want, "expanded app-state keys match whatsmeow expandAppStateKeys (160 bytes)");
|
||||
|
||||
// LTHash: add then remove is the identity; the order of items does not matter.
|
||||
Bytes a(32, 1), b(32, 2), c(32, 3);
|
||||
Bytes st = WaLtHash::zero();
|
||||
WaLtHash::subtract_then_add(st, {}, {a, b});
|
||||
WaLtHash::subtract_then_add(st, {a}, {c});
|
||||
Bytes st2 = WaLtHash::zero();
|
||||
WaLtHash::subtract_then_add(st2, {}, {c});
|
||||
WaLtHash::subtract_then_add(st2, {}, {b});
|
||||
check(st == st2, "LTHash: add a,b; remove a, add c equals add c; remove b (order-free, add/remove cancel)");
|
||||
Bytes st3 = WaLtHash::zero();
|
||||
WaLtHash::subtract_then_add(st3, {}, {a});
|
||||
WaLtHash::subtract_then_add(st3, {a}, {});
|
||||
check(st3 == WaLtHash::zero(), "LTHash: add then remove returns the zero state");
|
||||
}
|
||||
|
||||
// Fake collection server: serves one snapshot and the given patches, and answers the external blob download.
|
||||
struct FakeCollection {
|
||||
Bytes snapshot;
|
||||
std::vector<Bytes> patches;
|
||||
int iq_calls = 0;
|
||||
WaNode operator()(WaNode iq) {
|
||||
(void)iq;
|
||||
WaNode col("collection");
|
||||
col.attrs["name"] = WaAppStateSync::kContactsCollection;
|
||||
WaNode::Children kids;
|
||||
if (iq_calls == 0) {
|
||||
WaNode snap("snapshot");
|
||||
snap.content = Bytes{0x0a, 0x00}; // stands in for an ExternalBlobReference; the download hook returns the body
|
||||
kids.push_back(snap);
|
||||
}
|
||||
WaNode patches_node("patches");
|
||||
WaNode::Children pk;
|
||||
for (const auto& p : patches) {
|
||||
WaNode pn("patch");
|
||||
pn.content = p;
|
||||
pk.push_back(pn);
|
||||
}
|
||||
patches_node.content = pk;
|
||||
kids.push_back(patches_node);
|
||||
col.content = kids;
|
||||
++iq_calls;
|
||||
WaNode sync("sync");
|
||||
sync.content = WaNode::Children{col};
|
||||
WaNode reply("iq");
|
||||
reply.content = WaNode::Children{sync};
|
||||
return reply;
|
||||
}
|
||||
};
|
||||
|
||||
static WaAppStateSync::DownloadFn snapshot_download(const Bytes& snapshot) {
|
||||
return [snapshot](const WaExternalBlobRef&, std::string&) -> std::optional<Bytes> { return snapshot; };
|
||||
}
|
||||
|
||||
static void test_appstate_sync() {
|
||||
std::printf("[appstate: snapshot and patch sync]\n");
|
||||
Value v = Value::loads(slurp(path("appstate_vectors.json")));
|
||||
Bytes key_id = fromhex(v.get("key_id").str());
|
||||
Bytes key_data = fromhex(v.get("key_data").str());
|
||||
Bytes snapshot = fromhex(v.get("snapshot").str());
|
||||
Bytes patch = fromhex(v.get("patch").str());
|
||||
Bytes tampered = fromhex(v.get("patch_tampered").str());
|
||||
std::string name = WaAppStateSync::kContactsCollection;
|
||||
std::vector<std::string> logs;
|
||||
auto log = [&](const std::string& s) { logs.push_back(s); };
|
||||
|
||||
// 1. good chain: snapshot then one patch
|
||||
{
|
||||
std::string home = make_temp_home();
|
||||
WaStore store(home, "sync");
|
||||
store.put_app_key(key_id, key_data, 1767225600123);
|
||||
FakeCollection col;
|
||||
col.patches = {patch};
|
||||
WaAppStateSync sync(store, log);
|
||||
auto rep = sync.sync(name, col, snapshot_download(snapshot));
|
||||
check(rep.error.empty() && rep.rejected == 0, "good snapshot+patch chain verifies (snapshot MAC, patch MAC, value and index MACs)");
|
||||
check(rep.patches_applied == 1 && rep.mutations_applied == 10, "applied 1 patch with 10 mutations (4 snapshot + 6 patch)");
|
||||
check(rep.contacts_saved == 6 && rep.contacts_removed == 1 && rep.contacts_unnamed == 1,
|
||||
"contacts: 6 saved (3 snapshot, 3 patch), 1 removed, 1 unnamed skipped");
|
||||
auto contacts = store.contacts();
|
||||
std::map<std::string, std::string> got(contacts.begin(), contacts.end());
|
||||
std::map<std::string, std::string> want = {
|
||||
{"31611111111@s.whatsapp.net", "Nigel Dokter (work)"},
|
||||
{"31622222222@s.whatsapp.net", "Ann"},
|
||||
{"31644444444@s.whatsapp.net", "Joop Visser"},
|
||||
};
|
||||
check(got == want, "wa_contact holds exactly the named contacts after snapshot + patch (C removed, unnamed F and the pushName setting not stored)");
|
||||
auto ver = store.app_state_version(name);
|
||||
check(ver && ver->version == 2 && tohex(ver->hash) == v.get("patch_hash_hex").str(),
|
||||
"stored version 2 and LTHash state equal whatsmeow's state after the patch");
|
||||
// a second sync from the stored version is a no-op on the server side
|
||||
FakeCollection none;
|
||||
none.iq_calls = 1; // the server sends no snapshot once the client is at version 2
|
||||
auto rep2 = sync.sync(name, none, snapshot_download(snapshot));
|
||||
check(rep2.error.empty() && rep2.patches_applied == 0 && rep2.version == 2, "a re-sync from version 2 with no new patches changes nothing");
|
||||
std::filesystem::remove_all(home);
|
||||
}
|
||||
|
||||
// 2. tampered patch: MAC mismatch, the patch is not stored, the snapshot before it is
|
||||
{
|
||||
std::string home = make_temp_home();
|
||||
WaStore store(home, "tamper");
|
||||
store.put_app_key(key_id, key_data, 1767225600123);
|
||||
FakeCollection col;
|
||||
col.patches = {tampered};
|
||||
WaAppStateSync sync(store, log);
|
||||
auto rep = sync.sync(name, col, snapshot_download(snapshot));
|
||||
check(rep.rejected == 1 && rep.patches_applied == 0, "tampered patch is rejected (value MAC) and not applied");
|
||||
auto ver = store.app_state_version(name);
|
||||
check(ver && ver->version == 1, "after a rejected patch the stored version stays at the last verified state (1)");
|
||||
std::filesystem::remove_all(home);
|
||||
}
|
||||
|
||||
// 3. missing key: nothing is stored, the key id is reported for a key request
|
||||
{
|
||||
std::string home = make_temp_home();
|
||||
WaStore store(home, "nokey");
|
||||
FakeCollection col;
|
||||
WaAppStateSync sync(store, log);
|
||||
auto rep = sync.sync(name, col, snapshot_download(snapshot));
|
||||
check(rep.missing_key_ids.size() == 1 && rep.missing_key_ids[0] == key_id && rep.contacts_saved == 0,
|
||||
"missing app-state key: key id reported, no data stored");
|
||||
check(!store.app_state_version(name).has_value(), "missing key leaves the collection version unset");
|
||||
std::filesystem::remove_all(home);
|
||||
}
|
||||
|
||||
// 4. key share message parses to the oracle key; key request encodes a parseable Message
|
||||
{
|
||||
Bytes ks = fromhex(v.get("key_share_message").str());
|
||||
WaProtocolMessage pm = parse_protocol_message(ks);
|
||||
check(pm.type == 6 && pm.key_share.size() == 1 && pm.key_share[0].key_id == key_id && pm.key_share[0].key_data == key_data &&
|
||||
pm.key_share[0].timestamp == 1767225600123,
|
||||
"APP_STATE_SYNC_KEY_SHARE parses: key id, 32-byte key data, timestamp (whatsmeow proto)");
|
||||
Bytes kr = encode_app_state_key_request({key_id});
|
||||
WaProtocolMessage req = parse_protocol_message(kr);
|
||||
check(req.type == 7 && req.key_share.empty() && tohex(kr) == v.get("key_request_message").str(),
|
||||
"APP_STATE_SYNC_KEY_REQUEST encodes byte-exact to whatsmeow's Message");
|
||||
}
|
||||
}
|
||||
|
||||
static void test_receiver() {
|
||||
std::printf("[appstate: key share receipt]\n");
|
||||
// Alice (the phone) starts a session with our device with X3DH and sends the key share as pkmsg.
|
||||
WaDevice bob = WaDevice::generate();
|
||||
WaCrypto::KeyPair alice_id = WaCrypto::generate_x25519();
|
||||
WaCrypto::KeyPair alice_base = WaCrypto::generate_x25519();
|
||||
WaCrypto::KeyPair one_time = WaCrypto::generate_x25519();
|
||||
std::string home = make_temp_home();
|
||||
WaStore store(home, "recv");
|
||||
store.put_prekey(77, Bytes(one_time.priv.begin(), one_time.priv.end()));
|
||||
|
||||
WaSession::InitiatorParams p;
|
||||
p.local_identity = alice_id;
|
||||
p.local_base = alice_base;
|
||||
p.remote_identity = signal_pub(bob.identity_key.pub);
|
||||
p.remote_signed_pre = signal_pub(bob.signed_pre_key.pair.pub);
|
||||
p.remote_pre_key = signal_pub(one_time.pub);
|
||||
p.has_pre_key = true;
|
||||
WaSession s = WaSession::initiate(p, WaCrypto::generate_x25519());
|
||||
s.set_pending(1234, signal_pub(alice_base.pub), bob.signed_pre_key.key_id, true, 77);
|
||||
Bytes plain = fromhex(Value::loads(slurp(path("appstate_vectors.json"))).get("key_share_message").str());
|
||||
CiphertextMessage ct = s.encrypt(WaCrypto::pkcs7_pad(plain));
|
||||
WaNode enc("enc");
|
||||
enc.attrs["v"] = "2";
|
||||
enc.attrs["type"] = ct.type;
|
||||
enc.content = ct.serialized;
|
||||
|
||||
WaReceiver rx(store);
|
||||
auto res = rx.decrypt(enc, "15550001111@s.whatsapp.net", bob);
|
||||
check(ct.type == "pkmsg" && res.ok && res.message == plain, "pkmsg from the phone decrypts to the key-share Message (X3DH responder + unpad)");
|
||||
check(store.load_session("15550001111.0").has_value(), "the new session is stored under the phone's signal address");
|
||||
Bytes bad = ct.serialized;
|
||||
bad.back() ^= 0x01;
|
||||
WaNode enc_bad("enc");
|
||||
enc_bad.attrs["type"] = ct.type;
|
||||
enc_bad.content = bad;
|
||||
auto corrupt = rx.decrypt(enc_bad, "15550001111@s.whatsapp.net", bob);
|
||||
check(!corrupt.ok, "a corrupted pkmsg is refused with a reason, not decrypted");
|
||||
std::filesystem::remove_all(home);
|
||||
}
|
||||
|
||||
// The phone only pushes history and app-state keys to a companion that declares historySyncConfig
|
||||
// (DeviceProps field 5), so registration must carry it.
|
||||
static void test_device_props() {
|
||||
WaDeviceProps dp;
|
||||
Bytes with = dp.encode();
|
||||
dp.history_sync_config = false;
|
||||
Bytes without = dp.encode();
|
||||
check(with.size() > without.size() + 20, "DeviceProps carries historySyncConfig by default");
|
||||
ProtoReader r(with);
|
||||
ProtoReader::Field f;
|
||||
bool found = false;
|
||||
while (r.next(f)) found = found || (f.field == 5 && f.wire == 2);
|
||||
check(found, "DeviceProps field 5 (historySyncConfig) is length-delimited");
|
||||
}
|
||||
|
||||
// Contact search: the user's phone has "Nigel Dokter"; the user types "Nigel", "Niggel", "dokter" or the number.
|
||||
static void test_contact_search() {
|
||||
std::string home = "build/probe/wa_contact_test_home";
|
||||
std::filesystem::remove_all(home);
|
||||
WaStore store(home, "t");
|
||||
store.put_contact("31612345001@s.whatsapp.net", "Nigel Dokter");
|
||||
store.put_contact("31612345002@s.whatsapp.net", "Nina Smit");
|
||||
store.put_contact("31612345003@s.whatsapp.net", "Peter de Vries");
|
||||
auto top = [&](const std::string& q) {
|
||||
auto h = store.search_contacts(q, 3, 0.3);
|
||||
return h.empty() ? std::string("(none)") : h[0].name;
|
||||
};
|
||||
check(top("Nigel") == "Nigel Dokter", "contact search: first name");
|
||||
check(top("Niggel") == "Nigel Dokter", "contact search: typo (Niggel -> Nigel Dokter)");
|
||||
check(top("nigel dokter") == "Nigel Dokter", "contact search: full name, any case");
|
||||
check(top("dokter") == "Nigel Dokter", "contact search: surname only");
|
||||
check(top("12345003") == "Peter de Vries", "contact search: number fragment");
|
||||
check(store.search_contacts("zzzzqq", 3, 0.3).empty(), "contact search: nonsense matches nothing");
|
||||
store.remove_contact("31612345001@s.whatsapp.net");
|
||||
check(top("Nigel") != "Nigel Dokter", "contact search: removed contact is gone from the index");
|
||||
WaStore reopened(home, "t");
|
||||
check(reopened.search_contacts("Nina", 3, 0.3).size() >= 1, "contact search: index survives reopening the database");
|
||||
// send log: the floor under the rate caps
|
||||
check(store.sends_since(60) == 0, "send log starts empty");
|
||||
store.record_send("31612345002@s.whatsapp.net");
|
||||
store.record_send("31612345002@s.whatsapp.net");
|
||||
check(store.sends_since(60) == 2 && store.sends_since(86400) == 2, "send log counts sends in the window");
|
||||
std::filesystem::remove_all(home);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
DIR = argc > 1 ? argv[1] : "tests/whatsapp_data";
|
||||
test_contact_search();
|
||||
test_device_props();
|
||||
test_crypto();
|
||||
test_codec();
|
||||
test_media();
|
||||
test_server_jid();
|
||||
test_client_payload();
|
||||
test_noise();
|
||||
test_signal();
|
||||
test_device();
|
||||
test_hostile();
|
||||
test_lthash_and_keys();
|
||||
test_appstate_sync();
|
||||
test_receiver();
|
||||
std::printf("%s\n", g_fail == 0 ? "whatsapp: ALL OK" : "whatsapp: FAILURES");
|
||||
return g_fail == 0 ? 0 : 1;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user