#include "google/adk/yaml.h" #include #include #include #include "google/adk/caps.h" #include "google/adk/error.h" namespace molo::google::adk::yaml { using clients::body::Json; namespace { struct Line { int indent = 0; std::string text; // after the indent, right-trimmed std::string raw; // the full line (for block strings) int no = 0; // 1-based }; [[noreturn]] void fail(int line, const std::string& what) { throw AdkError::invalid("yaml line " + std::to_string(line) + ": " + what); } std::string rtrim(std::string s) { while (!s.empty() && (s.back() == ' ' || s.back() == '\t')) s.pop_back(); return s; } bool is_dash(const std::string& t) { return !t.empty() && t[0] == '-' && (t.size() == 1 || t[1] == ' '); } void append_utf8(std::string& out, std::uint32_t cp) { if (cp < 0x80) { out.push_back(static_cast(cp)); } else if (cp < 0x800) { out.push_back(static_cast(0xC0 | (cp >> 6))); out.push_back(static_cast(0x80 | (cp & 0x3F))); } else if (cp < 0x10000) { out.push_back(static_cast(0xE0 | (cp >> 12))); out.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); out.push_back(static_cast(0x80 | (cp & 0x3F))); } else { out.push_back(static_cast(0xF0 | (cp >> 18))); out.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); out.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); out.push_back(static_cast(0x80 | (cp & 0x3F))); } } bool all_digits(const std::string& s, std::size_t from) { if (from >= s.size()) return false; for (std::size_t k = from; k < s.size(); ++k) if (s[k] < '0' || s[k] > '9') return false; return true; } // YAML 1.2 core schema subset for an unquoted scalar. Json resolve_plain(const std::string& s) { if (s.empty() || s == "~" || s == "null" || s == "Null" || s == "NULL") return Json(); if (s == "true" || s == "True" || s == "TRUE") return Json(true); if (s == "false" || s == "False" || s == "FALSE") return Json(false); std::size_t from = (s[0] == '-') ? 1 : 0; if (all_digits(s, from) && s.size() - from <= 15) return Json(std::stod(s)); std::size_t dot = s.find('.'); if (dot != std::string::npos && dot > from && s.size() <= 32) { std::size_t e = s.find_first_of("eE", dot); std::string frac = s.substr(dot + 1, e == std::string::npos ? std::string::npos : e - dot - 1); bool ok = all_digits(s.substr(0, dot), from) && all_digits(frac, 0); if (ok && e != std::string::npos) { std::size_t q = e + 1; if (q < s.size() && (s[q] == '+' || s[q] == '-')) ++q; ok = all_digits(s, q); } if (ok) return Json(std::stod(s)); } return Json(s); } class Parser { public: explicit Parser(const std::string& text) { split(text); } Json run() { skip(); if (i_ >= lines_.size()) return Json(); Json v = node(0, 1); skip(); if (i_ < lines_.size()) fail(lines_[i_].no, "unexpected content (bad indentation or a second document)"); return v; } private: std::vector lines_; std::size_t i_ = 0; std::size_t nodes_ = 0; void split(const std::string& text) { std::size_t pos = 0; int no = 0; bool first_sig = true; while (pos <= text.size()) { std::size_t nl = text.find('\n', pos); std::string raw = text.substr(pos, nl == std::string::npos ? std::string::npos : nl - pos); pos = nl == std::string::npos ? text.size() + 1 : nl + 1; ++no; if (!raw.empty() && raw.back() == '\r') raw.pop_back(); if (lines_.size() >= AdkCaps::kMaxYamlLines) fail(no, "over " + std::to_string(AdkCaps::kMaxYamlLines) + " lines"); for (unsigned char c : raw) if ((c < 0x20 && c != '\t') || c == 0x7F) fail(no, "control character"); Line l; l.no = no; l.raw = raw; std::size_t sp = 0; while (sp < raw.size() && raw[sp] == ' ') ++sp; if (sp < raw.size() && raw[sp] == '\t') fail(no, "tab in indentation"); l.indent = static_cast(sp); l.text = rtrim(raw.substr(sp)); if (!l.text.empty() && l.text[0] != '#') { if (l.indent == 0 && (l.text == "---" || l.text.rfind("--- ", 0) == 0)) { if (!first_sig) fail(no, "multiple documents are not supported"); if (l.text != "---") fail(no, "content after --- is not supported"); first_sig = false; l.text.clear(); } else if (l.indent == 0 && (l.text == "..." || l.text[0] == '%')) { fail(no, "document end marker or directive is not supported"); } else { first_sig = false; } } lines_.push_back(std::move(l)); } } void skip() { while (i_ < lines_.size() && (lines_[i_].text.empty() || lines_[i_].text[0] == '#')) ++i_; } void count(int no) { if (++nodes_ > AdkCaps::kMaxYamlNodes) fail(no, "over the node cap"); } static std::string strip_comment(const std::string& t) { if (!t.empty() && t[0] == '#') return ""; std::size_t p = t.find(" #"); return rtrim(p == std::string::npos ? t : t.substr(0, p)); } // Parses a quoted scalar starting at t[0]; sets `end` past the closing quote. std::string quoted(const std::string& t, std::size_t& end, int no) { char q = t[0]; std::string out; for (std::size_t p = 1; p < t.size(); ++p) { char c = t[p]; if (q == '\'') { if (c == '\'') { if (p + 1 < t.size() && t[p + 1] == '\'') { out.push_back('\''); ++p; continue; } end = p + 1; return out; } out.push_back(c); continue; } if (c == '"') { end = p + 1; return out; } if (c != '\\') { out.push_back(c); continue; } if (++p >= t.size()) break; switch (t[p]) { case 'n': out.push_back('\n'); break; case 't': out.push_back('\t'); break; case 'r': out.push_back('\r'); break; case 'b': out.push_back('\b'); break; case 'f': out.push_back('\f'); break; case '0': out.push_back('\0'); break; case '/': out.push_back('/'); break; case '\\': out.push_back('\\'); break; case '"': out.push_back('"'); break; case 'u': { auto hex4 = [&](std::size_t at) -> std::uint32_t { if (at + 4 > t.size()) fail(no, "short \\u escape"); std::uint32_t v = 0; for (std::size_t k = 0; k < 4; ++k) { char h = t[at + k]; v <<= 4; if (h >= '0' && h <= '9') v |= static_cast(h - '0'); else if (h >= 'a' && h <= 'f') v |= static_cast(h - 'a' + 10); else if (h >= 'A' && h <= 'F') v |= static_cast(h - 'A' + 10); else fail(no, "bad \\u escape"); } return v; }; std::uint32_t cp = hex4(p + 1); p += 4; if (cp >= 0xD800 && cp < 0xDC00) { if (p + 2 < t.size() && t[p + 1] == '\\' && t[p + 2] == 'u') { std::uint32_t lo = hex4(p + 3); if (lo < 0xDC00 || lo > 0xDFFF) fail(no, "bad surrogate pair"); cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); p += 6; } else { fail(no, "lone surrogate"); } } else if (cp >= 0xDC00 && cp <= 0xDFFF) { fail(no, "lone surrogate"); } append_utf8(out, cp); break; } default: fail(no, std::string("unknown escape \\") + t[p]); } } fail(no, "unterminated quoted string"); } static bool block_header(const std::string& t) { if (t.empty() || (t[0] != '|' && t[0] != '>')) return false; std::string rest = t.substr(1); std::size_t k = 0; while (k < rest.size() && (rest[k] == '-' || rest[k] == '+')) ++k; if (k > 1) return false; std::string tail = rest.substr(k); return tail.empty() || tail[0] == ' ' || tail[0] == '#'; } // One inline value (after "key:" or "- ", or a lone scalar line). Json scalar(const std::string& text, int no, int parent_indent) { count(no); std::string t = text; if (t.empty() || t[0] == '#') return Json(); char c = t[0]; if (c == '"' || c == '\'') { std::size_t end = 0; std::string s = quoted(t, end, no); std::string rest = rtrim(t.substr(end)); std::size_t k = 0; while (k < rest.size() && rest[k] == ' ') ++k; if (k < rest.size() && !(rest[k] == '#' && k > 0)) fail(no, "content after a quoted string"); return Json(s); } if (c == '|' || c == '>') { if (!block_header(t)) fail(no, "unsupported block scalar header (indentation indicators are rejected)"); return block(t, parent_indent); } t = strip_comment(t); if (t.empty()) return Json(); switch (t[0]) { case '&': fail(no, "anchors are not supported"); case '*': fail(no, "aliases are not supported"); case '!': fail(no, "tags are not supported"); case '%': fail(no, "directives are not supported"); case '?': fail(no, "complex keys are not supported"); case '@': case '`': fail(no, "reserved indicator"); case '[': case '{': if (t == "[]") return Json::array(); if (t == "{}") return Json::object(); fail(no, "flow collections are not supported (only [] and {})"); default: break; } if (is_dash(t)) fail(no, "sequence item where a value was expected"); if (t.find(": ") != std::string::npos || t.back() == ':') fail(no, "nested mapping on one line is not supported"); return resolve_plain(t); } static bool blank(const Line& l) { return l.raw.find_first_not_of(' ') == std::string::npos; } Json block(const std::string& header, int parent_indent) { bool literal = header[0] == '|'; char chomp = 'c'; if (header.size() > 1 && header[1] == '-') chomp = 's'; if (header.size() > 1 && header[1] == '+') chomp = 'k'; int bi = -1; for (std::size_t k = i_; k < lines_.size(); ++k) { if (blank(lines_[k])) continue; bi = lines_[k].indent; break; } std::vector ls; if (bi > parent_indent) { std::size_t j = i_; for (; j < lines_.size(); ++j) { const Line& l = lines_[j]; if (blank(l)) { ls.emplace_back(); continue; } if (l.indent < bi) break; ls.push_back(l.raw.substr(static_cast(bi))); } i_ = j; } std::size_t trailing = 0; while (!ls.empty() && ls.back().empty()) { ls.pop_back(); ++trailing; } std::string out; if (literal) { for (std::size_t k = 0; k < ls.size(); ++k) { if (k) out.push_back('\n'); out += ls[k]; } } else { for (std::size_t k = 0; k < ls.size(); ++k) { if (k == 0) { out = ls[k]; } else if (ls[k].empty()) { out.push_back('\n'); } else if (ls[k - 1].empty()) { out += ls[k]; } else if (ls[k][0] == ' ' || ls[k - 1][0] == ' ') { out += "\n" + ls[k]; } else { out += " " + ls[k]; } } } if (!ls.empty()) { if (chomp == 'c') out.push_back('\n'); else if (chomp == 'k') out += std::string(1 + trailing, '\n'); } else if (chomp == 'k') { out = std::string(trailing, '\n'); } return Json(out); } // Splits "key: rest". False when the line is not a map entry. bool entry(const std::string& t, std::string& key, std::string& rest, int no) { if (t.empty()) return false; char c = t[0]; std::size_t colon = std::string::npos; if (c == '"' || c == '\'') { std::size_t end = 0; key = quoted(t, end, no); std::size_t p = end; while (p < t.size() && t[p] == ' ') ++p; if (p >= t.size() || t[p] != ':' || (p + 1 < t.size() && t[p + 1] != ' ')) return false; colon = p; } else { for (std::size_t p = 0; p < t.size(); ++p) { if (t[p] == ':' && (p + 1 == t.size() || t[p + 1] == ' ')) { colon = p; break; } if (t[p] == ' ' && p + 1 < t.size() && t[p + 1] == '#') return false; } if (colon == std::string::npos || colon == 0) return false; key = rtrim(t.substr(0, colon)); switch (key[0]) { case '&': fail(no, "anchors are not supported"); case '*': fail(no, "aliases are not supported"); case '!': fail(no, "tags are not supported"); case '?': fail(no, "complex keys are not supported"); case '[': case '{': fail(no, "flow keys are not supported"); default: break; } } std::size_t r = colon + 1; while (r < t.size() && t[r] == ' ') ++r; rest = t.substr(r); return true; } Json node(int min_indent, int depth) { skip(); if (depth > AdkCaps::kMaxYamlDepth) fail(i_ < lines_.size() ? lines_[i_].no : 0, "nesting deeper than the cap"); if (i_ >= lines_.size() || lines_[i_].indent < min_indent) return Json(); const Line l = lines_[i_]; if (is_dash(l.text)) return seq(l.indent, depth); std::string k, r; if (entry(l.text, k, r, l.no)) return map(l.indent, depth); ++i_; return scalar(l.text, l.no, min_indent - 1); } Json seq(int ind, int depth) { Json out = Json::array(); for (;;) { skip(); if (i_ >= lines_.size() || lines_[i_].indent < ind) break; Line& l = lines_[i_]; if (l.indent > ind) fail(l.no, "unexpected indentation"); if (!is_dash(l.text)) break; count(l.no); std::size_t sp = 1; while (sp < l.text.size() && l.text[sp] == ' ') ++sp; std::string content = l.text.substr(sp); if (content.empty() || content[0] == '#') { ++i_; skip(); if (i_ < lines_.size() && lines_[i_].indent > ind) out.push(node(ind + 1, depth + 1)); else out.push(Json()); } else { l.indent = ind + static_cast(sp); l.text = content; out.push(node(ind + 1, depth + 1)); } } return out; } Json map(int ind, int depth) { Json out = Json::object(); std::set seen; for (;;) { skip(); if (i_ >= lines_.size() || lines_[i_].indent < ind) break; const Line l = lines_[i_]; if (l.indent > ind) fail(l.no, "unexpected indentation"); if (is_dash(l.text)) fail(l.no, "sequence item where a key was expected"); std::string key, rest; if (!entry(l.text, key, rest, l.no)) fail(l.no, "expected 'key: value'"); if (key == "<<") fail(l.no, "merge keys are not supported"); if (key.empty() || key.size() > 256) fail(l.no, "key empty or over 256 bytes"); if (!seen.insert(key).second) fail(l.no, "duplicate key '" + key + "'"); count(l.no); ++i_; Json value; if (rest.empty() || rest[0] == '#') { skip(); if (i_ < lines_.size() && lines_[i_].indent > ind) value = node(ind + 1, depth + 1); else if (i_ < lines_.size() && lines_[i_].indent == ind && is_dash(lines_[i_].text)) value = seq(ind, depth + 1); } else { value = scalar(rest, l.no, ind); } out.set(key, std::move(value)); } return out; } }; // ---- emitter ---------------------------------------------------------------- bool plain_ok(const std::string& s) { if (s.empty()) return false; for (unsigned char c : s) if (c < 0x20 || c == 0x7F) return false; if (s.front() == ' ' || s.back() == ' ') return false; if (std::string("-?:,[]{}#&*!|>'\"%@`").find(s.front()) != std::string::npos) return false; if (s.find(": ") != std::string::npos || s.find(" #") != std::string::npos || s.back() == ':') return false; return resolve_plain(s).is_string(); } std::string quote(const std::string& s) { return "\"" + clients::body::json_escape(s) + "\""; } // Block form: multi-line, \n and printable only, first non-empty line does // not start with a space, no whitespace-only lines. bool block_ok(const std::string& s) { if (s.find('\n') == std::string::npos) return false; bool content = false; for (unsigned char c : s) { if (c == '\n') continue; content = true; if ((c < 0x20 && c != '\t') || c == 0x7F) return false; } if (!content) return false; std::size_t k = 0; while (s[k] == '\n') ++k; if (s[k] == ' ' || s[k] == '\t') return false; std::size_t start = 0; for (;;) { std::size_t nl = s.find('\n', start); std::string line = s.substr(start, nl == std::string::npos ? std::string::npos : nl - start); if (!line.empty() && line.find_first_not_of(' ') == std::string::npos) return false; if (nl == std::string::npos) break; start = nl + 1; } return true; } std::string scalar_text(const Json& v) { if (v.is_null()) return "null"; if (v.is_bool()) return v.as_bool() ? "true" : "false"; if (v.is_number()) return v.dump(); return plain_ok(v.as_string()) ? v.as_string() : quote(v.as_string()); } void emit_block_string(const std::string& s, int indent, std::string& out) { std::size_t trail = 0; while (trail < s.size() && s[s.size() - 1 - trail] == '\n') ++trail; out += trail == 0 ? "|-" : (trail == 1 ? "|" : "|+"); out += "\n"; std::string body = s.substr(0, s.size() - trail); std::string pad(static_cast(indent), ' '); std::size_t start = 0; for (;;) { std::size_t nl = body.find('\n', start); std::string line = body.substr(start, nl == std::string::npos ? std::string::npos : nl - start); out += line.empty() ? "\n" : pad + line + "\n"; if (nl == std::string::npos) break; start = nl + 1; } if (trail > 1) out += std::string(trail - 1, '\n'); } bool is_container(const Json& v) { return v.is_object() || v.is_array(); } bool is_empty_container(const Json& v) { return (v.is_object() && v.as_object().empty()) || (v.is_array() && v.as_array().empty()); } std::string empty_text(const Json& v) { return v.is_object() ? "{}" : "[]"; } void emit(const Json& v, int indent, std::string& out); void emit_map(const Json& v, int indent, std::string& out) { std::string pad(static_cast(indent), ' '); for (const auto& [k, child] : v.as_object()) { out += pad + (plain_ok(k) ? k : quote(k)) + ":"; if (is_empty_container(child)) { out += " " + empty_text(child) + "\n"; } else if (is_container(child)) { out += "\n"; emit(child, indent + 2, out); } else if (child.is_string() && block_ok(child.as_string())) { out += " "; emit_block_string(child.as_string(), indent + 2, out); } else { out += " " + scalar_text(child) + "\n"; } } } void emit_seq(const Json& v, int indent, std::string& out) { std::string pad(static_cast(indent), ' '); for (const Json& item : v.as_array()) { if (is_empty_container(item)) { out += pad + "- " + empty_text(item) + "\n"; } else if (is_container(item)) { std::string tmp; emit(item, indent + 2, tmp); out += pad + "- " + tmp.substr(static_cast(indent) + 2); } else if (item.is_string() && block_ok(item.as_string())) { out += pad + "- "; emit_block_string(item.as_string(), indent + 2, out); } else { out += pad + "- " + scalar_text(item) + "\n"; } } } void emit(const Json& v, int indent, std::string& out) { if (v.is_object()) emit_map(v, indent, out); else if (v.is_array()) emit_seq(v, indent, out); else out += std::string(static_cast(indent), ' ') + scalar_text(v) + "\n"; } } // namespace Json parse(const std::string& text) { if (text.size() > AdkCaps::kMaxDefinitionBytes) throw AdkError::invalid("yaml over " + std::to_string(AdkCaps::kMaxDefinitionBytes) + " bytes"); return Parser(text).run(); } std::string dump(const Json& value) { if (is_empty_container(value)) return empty_text(value) + "\n"; std::string out; emit(value, 0, out); return out; } } // namespace molo::google::adk::yaml