Files
moloapp/src/google/adk/yaml.cpp
T

582 lines
20 KiB
C++

#include "google/adk/yaml.h"
#include <cstdint>
#include <set>
#include <vector>
#include "google/adk/caps.h"
#include "google/adk/error.h"
namespace molo::google::adk::yaml {
using clients::body::Json;
namespace {
struct Line {
int indent = 0;
std::string text; // after the indent, right-trimmed
std::string raw; // the full line (for block strings)
int no = 0; // 1-based
};
[[noreturn]] void fail(int line, const std::string& what) {
throw AdkError::invalid("yaml line " + std::to_string(line) + ": " + what);
}
std::string rtrim(std::string s) {
while (!s.empty() && (s.back() == ' ' || s.back() == '\t')) s.pop_back();
return s;
}
bool is_dash(const std::string& t) {
return !t.empty() && t[0] == '-' && (t.size() == 1 || t[1] == ' ');
}
void append_utf8(std::string& out, std::uint32_t cp) {
if (cp < 0x80) {
out.push_back(static_cast<char>(cp));
} else if (cp < 0x800) {
out.push_back(static_cast<char>(0xC0 | (cp >> 6)));
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
} else if (cp < 0x10000) {
out.push_back(static_cast<char>(0xE0 | (cp >> 12)));
out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
} else {
out.push_back(static_cast<char>(0xF0 | (cp >> 18)));
out.push_back(static_cast<char>(0x80 | ((cp >> 12) & 0x3F)));
out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
}
}
bool all_digits(const std::string& s, std::size_t from) {
if (from >= s.size()) return false;
for (std::size_t k = from; k < s.size(); ++k)
if (s[k] < '0' || s[k] > '9') return false;
return true;
}
// YAML 1.2 core schema subset for an unquoted scalar.
Json resolve_plain(const std::string& s) {
if (s.empty() || s == "~" || s == "null" || s == "Null" || s == "NULL") return Json();
if (s == "true" || s == "True" || s == "TRUE") return Json(true);
if (s == "false" || s == "False" || s == "FALSE") return Json(false);
std::size_t from = (s[0] == '-') ? 1 : 0;
if (all_digits(s, from) && s.size() - from <= 15) return Json(std::stod(s));
std::size_t dot = s.find('.');
if (dot != std::string::npos && dot > from && s.size() <= 32) {
std::size_t e = s.find_first_of("eE", dot);
std::string frac = s.substr(dot + 1, e == std::string::npos ? std::string::npos : e - dot - 1);
bool ok = all_digits(s.substr(0, dot), from) && all_digits(frac, 0);
if (ok && e != std::string::npos) {
std::size_t q = e + 1;
if (q < s.size() && (s[q] == '+' || s[q] == '-')) ++q;
ok = all_digits(s, q);
}
if (ok) return Json(std::stod(s));
}
return Json(s);
}
class Parser {
public:
explicit Parser(const std::string& text) { split(text); }
Json run() {
skip();
if (i_ >= lines_.size()) return Json();
Json v = node(0, 1);
skip();
if (i_ < lines_.size())
fail(lines_[i_].no, "unexpected content (bad indentation or a second document)");
return v;
}
private:
std::vector<Line> lines_;
std::size_t i_ = 0;
std::size_t nodes_ = 0;
void split(const std::string& text) {
std::size_t pos = 0;
int no = 0;
bool first_sig = true;
while (pos <= text.size()) {
std::size_t nl = text.find('\n', pos);
std::string raw = text.substr(pos, nl == std::string::npos ? std::string::npos : nl - pos);
pos = nl == std::string::npos ? text.size() + 1 : nl + 1;
++no;
if (!raw.empty() && raw.back() == '\r') raw.pop_back();
if (lines_.size() >= AdkCaps::kMaxYamlLines)
fail(no, "over " + std::to_string(AdkCaps::kMaxYamlLines) + " lines");
for (unsigned char c : raw)
if ((c < 0x20 && c != '\t') || c == 0x7F) fail(no, "control character");
Line l;
l.no = no;
l.raw = raw;
std::size_t sp = 0;
while (sp < raw.size() && raw[sp] == ' ') ++sp;
if (sp < raw.size() && raw[sp] == '\t') fail(no, "tab in indentation");
l.indent = static_cast<int>(sp);
l.text = rtrim(raw.substr(sp));
if (!l.text.empty() && l.text[0] != '#') {
if (l.indent == 0 && (l.text == "---" || l.text.rfind("--- ", 0) == 0)) {
if (!first_sig) fail(no, "multiple documents are not supported");
if (l.text != "---") fail(no, "content after --- is not supported");
first_sig = false;
l.text.clear();
} else if (l.indent == 0 && (l.text == "..." || l.text[0] == '%')) {
fail(no, "document end marker or directive is not supported");
} else {
first_sig = false;
}
}
lines_.push_back(std::move(l));
}
}
void skip() {
while (i_ < lines_.size() && (lines_[i_].text.empty() || lines_[i_].text[0] == '#')) ++i_;
}
void count(int no) {
if (++nodes_ > AdkCaps::kMaxYamlNodes) fail(no, "over the node cap");
}
static std::string strip_comment(const std::string& t) {
if (!t.empty() && t[0] == '#') return "";
std::size_t p = t.find(" #");
return rtrim(p == std::string::npos ? t : t.substr(0, p));
}
// Parses a quoted scalar starting at t[0]; sets `end` past the closing quote.
std::string quoted(const std::string& t, std::size_t& end, int no) {
char q = t[0];
std::string out;
for (std::size_t p = 1; p < t.size(); ++p) {
char c = t[p];
if (q == '\'') {
if (c == '\'') {
if (p + 1 < t.size() && t[p + 1] == '\'') {
out.push_back('\'');
++p;
continue;
}
end = p + 1;
return out;
}
out.push_back(c);
continue;
}
if (c == '"') {
end = p + 1;
return out;
}
if (c != '\\') {
out.push_back(c);
continue;
}
if (++p >= t.size()) break;
switch (t[p]) {
case 'n': out.push_back('\n'); break;
case 't': out.push_back('\t'); break;
case 'r': out.push_back('\r'); break;
case 'b': out.push_back('\b'); break;
case 'f': out.push_back('\f'); break;
case '0': out.push_back('\0'); break;
case '/': out.push_back('/'); break;
case '\\': out.push_back('\\'); break;
case '"': out.push_back('"'); break;
case 'u': {
auto hex4 = [&](std::size_t at) -> std::uint32_t {
if (at + 4 > t.size()) fail(no, "short \\u escape");
std::uint32_t v = 0;
for (std::size_t k = 0; k < 4; ++k) {
char h = t[at + k];
v <<= 4;
if (h >= '0' && h <= '9') v |= static_cast<std::uint32_t>(h - '0');
else if (h >= 'a' && h <= 'f') v |= static_cast<std::uint32_t>(h - 'a' + 10);
else if (h >= 'A' && h <= 'F') v |= static_cast<std::uint32_t>(h - 'A' + 10);
else fail(no, "bad \\u escape");
}
return v;
};
std::uint32_t cp = hex4(p + 1);
p += 4;
if (cp >= 0xD800 && cp < 0xDC00) {
if (p + 2 < t.size() && t[p + 1] == '\\' && t[p + 2] == 'u') {
std::uint32_t lo = hex4(p + 3);
if (lo < 0xDC00 || lo > 0xDFFF) fail(no, "bad surrogate pair");
cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00);
p += 6;
} else {
fail(no, "lone surrogate");
}
} else if (cp >= 0xDC00 && cp <= 0xDFFF) {
fail(no, "lone surrogate");
}
append_utf8(out, cp);
break;
}
default: fail(no, std::string("unknown escape \\") + t[p]);
}
}
fail(no, "unterminated quoted string");
}
static bool block_header(const std::string& t) {
if (t.empty() || (t[0] != '|' && t[0] != '>')) return false;
std::string rest = t.substr(1);
std::size_t k = 0;
while (k < rest.size() && (rest[k] == '-' || rest[k] == '+')) ++k;
if (k > 1) return false;
std::string tail = rest.substr(k);
return tail.empty() || tail[0] == ' ' || tail[0] == '#';
}
// One inline value (after "key:" or "- ", or a lone scalar line).
Json scalar(const std::string& text, int no, int parent_indent) {
count(no);
std::string t = text;
if (t.empty() || t[0] == '#') return Json();
char c = t[0];
if (c == '"' || c == '\'') {
std::size_t end = 0;
std::string s = quoted(t, end, no);
std::string rest = rtrim(t.substr(end));
std::size_t k = 0;
while (k < rest.size() && rest[k] == ' ') ++k;
if (k < rest.size() && !(rest[k] == '#' && k > 0)) fail(no, "content after a quoted string");
return Json(s);
}
if (c == '|' || c == '>') {
if (!block_header(t)) fail(no, "unsupported block scalar header (indentation indicators are rejected)");
return block(t, parent_indent);
}
t = strip_comment(t);
if (t.empty()) return Json();
switch (t[0]) {
case '&': fail(no, "anchors are not supported");
case '*': fail(no, "aliases are not supported");
case '!': fail(no, "tags are not supported");
case '%': fail(no, "directives are not supported");
case '?': fail(no, "complex keys are not supported");
case '@':
case '`': fail(no, "reserved indicator");
case '[':
case '{':
if (t == "[]") return Json::array();
if (t == "{}") return Json::object();
fail(no, "flow collections are not supported (only [] and {})");
default: break;
}
if (is_dash(t)) fail(no, "sequence item where a value was expected");
if (t.find(": ") != std::string::npos || t.back() == ':')
fail(no, "nested mapping on one line is not supported");
return resolve_plain(t);
}
static bool blank(const Line& l) { return l.raw.find_first_not_of(' ') == std::string::npos; }
Json block(const std::string& header, int parent_indent) {
bool literal = header[0] == '|';
char chomp = 'c';
if (header.size() > 1 && header[1] == '-') chomp = 's';
if (header.size() > 1 && header[1] == '+') chomp = 'k';
int bi = -1;
for (std::size_t k = i_; k < lines_.size(); ++k) {
if (blank(lines_[k])) continue;
bi = lines_[k].indent;
break;
}
std::vector<std::string> ls;
if (bi > parent_indent) {
std::size_t j = i_;
for (; j < lines_.size(); ++j) {
const Line& l = lines_[j];
if (blank(l)) {
ls.emplace_back();
continue;
}
if (l.indent < bi) break;
ls.push_back(l.raw.substr(static_cast<std::size_t>(bi)));
}
i_ = j;
}
std::size_t trailing = 0;
while (!ls.empty() && ls.back().empty()) {
ls.pop_back();
++trailing;
}
std::string out;
if (literal) {
for (std::size_t k = 0; k < ls.size(); ++k) {
if (k) out.push_back('\n');
out += ls[k];
}
} else {
for (std::size_t k = 0; k < ls.size(); ++k) {
if (k == 0) {
out = ls[k];
} else if (ls[k].empty()) {
out.push_back('\n');
} else if (ls[k - 1].empty()) {
out += ls[k];
} else if (ls[k][0] == ' ' || ls[k - 1][0] == ' ') {
out += "\n" + ls[k];
} else {
out += " " + ls[k];
}
}
}
if (!ls.empty()) {
if (chomp == 'c') out.push_back('\n');
else if (chomp == 'k') out += std::string(1 + trailing, '\n');
} else if (chomp == 'k') {
out = std::string(trailing, '\n');
}
return Json(out);
}
// Splits "key: rest". False when the line is not a map entry.
bool entry(const std::string& t, std::string& key, std::string& rest, int no) {
if (t.empty()) return false;
char c = t[0];
std::size_t colon = std::string::npos;
if (c == '"' || c == '\'') {
std::size_t end = 0;
key = quoted(t, end, no);
std::size_t p = end;
while (p < t.size() && t[p] == ' ') ++p;
if (p >= t.size() || t[p] != ':' || (p + 1 < t.size() && t[p + 1] != ' ')) return false;
colon = p;
} else {
for (std::size_t p = 0; p < t.size(); ++p) {
if (t[p] == ':' && (p + 1 == t.size() || t[p + 1] == ' ')) {
colon = p;
break;
}
if (t[p] == ' ' && p + 1 < t.size() && t[p + 1] == '#') return false;
}
if (colon == std::string::npos || colon == 0) return false;
key = rtrim(t.substr(0, colon));
switch (key[0]) {
case '&': fail(no, "anchors are not supported");
case '*': fail(no, "aliases are not supported");
case '!': fail(no, "tags are not supported");
case '?': fail(no, "complex keys are not supported");
case '[':
case '{': fail(no, "flow keys are not supported");
default: break;
}
}
std::size_t r = colon + 1;
while (r < t.size() && t[r] == ' ') ++r;
rest = t.substr(r);
return true;
}
Json node(int min_indent, int depth) {
skip();
if (depth > AdkCaps::kMaxYamlDepth)
fail(i_ < lines_.size() ? lines_[i_].no : 0, "nesting deeper than the cap");
if (i_ >= lines_.size() || lines_[i_].indent < min_indent) return Json();
const Line l = lines_[i_];
if (is_dash(l.text)) return seq(l.indent, depth);
std::string k, r;
if (entry(l.text, k, r, l.no)) return map(l.indent, depth);
++i_;
return scalar(l.text, l.no, min_indent - 1);
}
Json seq(int ind, int depth) {
Json out = Json::array();
for (;;) {
skip();
if (i_ >= lines_.size() || lines_[i_].indent < ind) break;
Line& l = lines_[i_];
if (l.indent > ind) fail(l.no, "unexpected indentation");
if (!is_dash(l.text)) break;
count(l.no);
std::size_t sp = 1;
while (sp < l.text.size() && l.text[sp] == ' ') ++sp;
std::string content = l.text.substr(sp);
if (content.empty() || content[0] == '#') {
++i_;
skip();
if (i_ < lines_.size() && lines_[i_].indent > ind) out.push(node(ind + 1, depth + 1));
else out.push(Json());
} else {
l.indent = ind + static_cast<int>(sp);
l.text = content;
out.push(node(ind + 1, depth + 1));
}
}
return out;
}
Json map(int ind, int depth) {
Json out = Json::object();
std::set<std::string> seen;
for (;;) {
skip();
if (i_ >= lines_.size() || lines_[i_].indent < ind) break;
const Line l = lines_[i_];
if (l.indent > ind) fail(l.no, "unexpected indentation");
if (is_dash(l.text)) fail(l.no, "sequence item where a key was expected");
std::string key, rest;
if (!entry(l.text, key, rest, l.no)) fail(l.no, "expected 'key: value'");
if (key == "<<") fail(l.no, "merge keys are not supported");
if (key.empty() || key.size() > 256) fail(l.no, "key empty or over 256 bytes");
if (!seen.insert(key).second) fail(l.no, "duplicate key '" + key + "'");
count(l.no);
++i_;
Json value;
if (rest.empty() || rest[0] == '#') {
skip();
if (i_ < lines_.size() && lines_[i_].indent > ind) value = node(ind + 1, depth + 1);
else if (i_ < lines_.size() && lines_[i_].indent == ind && is_dash(lines_[i_].text))
value = seq(ind, depth + 1);
} else {
value = scalar(rest, l.no, ind);
}
out.set(key, std::move(value));
}
return out;
}
};
// ---- emitter ----------------------------------------------------------------
bool plain_ok(const std::string& s) {
if (s.empty()) return false;
for (unsigned char c : s)
if (c < 0x20 || c == 0x7F) return false;
if (s.front() == ' ' || s.back() == ' ') return false;
if (std::string("-?:,[]{}#&*!|>'\"%@`").find(s.front()) != std::string::npos) return false;
if (s.find(": ") != std::string::npos || s.find(" #") != std::string::npos || s.back() == ':')
return false;
return resolve_plain(s).is_string();
}
std::string quote(const std::string& s) { return "\"" + clients::body::json_escape(s) + "\""; }
// Block form: multi-line, \n and printable only, first non-empty line does
// not start with a space, no whitespace-only lines.
bool block_ok(const std::string& s) {
if (s.find('\n') == std::string::npos) return false;
bool content = false;
for (unsigned char c : s) {
if (c == '\n') continue;
content = true;
if ((c < 0x20 && c != '\t') || c == 0x7F) return false;
}
if (!content) return false;
std::size_t k = 0;
while (s[k] == '\n') ++k;
if (s[k] == ' ' || s[k] == '\t') return false;
std::size_t start = 0;
for (;;) {
std::size_t nl = s.find('\n', start);
std::string line = s.substr(start, nl == std::string::npos ? std::string::npos : nl - start);
if (!line.empty() && line.find_first_not_of(' ') == std::string::npos) return false;
if (nl == std::string::npos) break;
start = nl + 1;
}
return true;
}
std::string scalar_text(const Json& v) {
if (v.is_null()) return "null";
if (v.is_bool()) return v.as_bool() ? "true" : "false";
if (v.is_number()) return v.dump();
return plain_ok(v.as_string()) ? v.as_string() : quote(v.as_string());
}
void emit_block_string(const std::string& s, int indent, std::string& out) {
std::size_t trail = 0;
while (trail < s.size() && s[s.size() - 1 - trail] == '\n') ++trail;
out += trail == 0 ? "|-" : (trail == 1 ? "|" : "|+");
out += "\n";
std::string body = s.substr(0, s.size() - trail);
std::string pad(static_cast<std::size_t>(indent), ' ');
std::size_t start = 0;
for (;;) {
std::size_t nl = body.find('\n', start);
std::string line = body.substr(start, nl == std::string::npos ? std::string::npos : nl - start);
out += line.empty() ? "\n" : pad + line + "\n";
if (nl == std::string::npos) break;
start = nl + 1;
}
if (trail > 1) out += std::string(trail - 1, '\n');
}
bool is_container(const Json& v) { return v.is_object() || v.is_array(); }
bool is_empty_container(const Json& v) {
return (v.is_object() && v.as_object().empty()) || (v.is_array() && v.as_array().empty());
}
std::string empty_text(const Json& v) { return v.is_object() ? "{}" : "[]"; }
void emit(const Json& v, int indent, std::string& out);
void emit_map(const Json& v, int indent, std::string& out) {
std::string pad(static_cast<std::size_t>(indent), ' ');
for (const auto& [k, child] : v.as_object()) {
out += pad + (plain_ok(k) ? k : quote(k)) + ":";
if (is_empty_container(child)) {
out += " " + empty_text(child) + "\n";
} else if (is_container(child)) {
out += "\n";
emit(child, indent + 2, out);
} else if (child.is_string() && block_ok(child.as_string())) {
out += " ";
emit_block_string(child.as_string(), indent + 2, out);
} else {
out += " " + scalar_text(child) + "\n";
}
}
}
void emit_seq(const Json& v, int indent, std::string& out) {
std::string pad(static_cast<std::size_t>(indent), ' ');
for (const Json& item : v.as_array()) {
if (is_empty_container(item)) {
out += pad + "- " + empty_text(item) + "\n";
} else if (is_container(item)) {
std::string tmp;
emit(item, indent + 2, tmp);
out += pad + "- " + tmp.substr(static_cast<std::size_t>(indent) + 2);
} else if (item.is_string() && block_ok(item.as_string())) {
out += pad + "- ";
emit_block_string(item.as_string(), indent + 2, out);
} else {
out += pad + "- " + scalar_text(item) + "\n";
}
}
}
void emit(const Json& v, int indent, std::string& out) {
if (v.is_object()) emit_map(v, indent, out);
else if (v.is_array()) emit_seq(v, indent, out);
else out += std::string(static_cast<std::size_t>(indent), ' ') + scalar_text(v) + "\n";
}
} // namespace
Json parse(const std::string& text) {
if (text.size() > AdkCaps::kMaxDefinitionBytes)
throw AdkError::invalid("yaml over " + std::to_string(AdkCaps::kMaxDefinitionBytes) + " bytes");
return Parser(text).run();
}
std::string dump(const Json& value) {
if (is_empty_container(value)) return empty_text(value) + "\n";
std::string out;
emit(value, 0, out);
return out;
}
} // namespace molo::google::adk::yaml