More stuff for working with strings and bytes!

- "\x" escape sequence to put byte values in strings: "\x34"
- String.byteAt(index) gets value of byte in string.
- String.bytes returns a raw sequence of bytes for a string.
- String.codePointAt(index) gets the code point at an offset as a raw number.

src/vm/wren_compiler.c

@@ -622,15 +622,15 @@ static void addStringChar(Parser* parser, char c)
   wrenByteBufferWrite(parser->vm, &parser->string, c);
 }
 
-// Reads a four hex digit Unicode escape sequence in a string literal.
-static void readUnicodeEscape(Parser* parser)
+// Reads [digits] hex digits in a string literal and returns their number value.
+static int readHexEscape(Parser* parser, int digits, const char* description)
 {
   int value = 0;
-  for (int i = 0; i < 4; i++)
+  for (int i = 0; i < digits; i++)
   {
     if (peekChar(parser) == '"' || peekChar(parser) == '\0')
     {
-      lexError(parser, "Incomplete Unicode escape sequence.");
+      lexError(parser, "Incomplete %s escape sequence.", description);
 
       // Don't consume it if it isn't expected. Keeps us from reading past the
       // end of an unterminated string.
@@ -641,13 +641,21 @@ static void readUnicodeEscape(Parser* parser)
     int digit = readHexDigit(parser);
     if (digit == -1)
     {
-      lexError(parser, "Invalid Unicode escape sequence.");
+      lexError(parser, "Invalid %s escape sequence.", description);
       break;
     }
 
     value = (value * 16) | digit;
   }
 
+  return value;
+}
+
+// Reads a four hex digit Unicode escape sequence in a string literal.
+static void readUnicodeEscape(Parser* parser)
+{
+  int value = readHexEscape(parser, 4, "Unicode");
+
   // Grow the buffer enough for the encoded result.
   int numBytes = wrenUtf8NumBytes(value);
   if (numBytes != 0)
@@ -696,9 +704,13 @@ static void readString(Parser* parser)
         case 'n':  addStringChar(parser, '\n'); break;
         case 'r':  addStringChar(parser, '\r'); break;
         case 't':  addStringChar(parser, '\t'); break;
-        case 'v':  addStringChar(parser, '\v'); break;
         case 'u':  readUnicodeEscape(parser); break;
           // TODO: 'U' for 8 octet Unicode escapes.
+        case 'v':  addStringChar(parser, '\v'); break;
+        case 'x':
+          addStringChar(parser, (uint8_t)readHexEscape(parser, 2, "byte"));
+          break;
+
         default:
           lexError(parser, "Invalid escape character '%c'.",
                    *(parser->currentChar - 1));

src/vm/wren_core.c

@@ -138,7 +138,19 @@ static const char* libSource =
 "  }\n"
 "}\n"
 "\n"
-"class String is Sequence {}\n"
+"class String is Sequence {\n"
+"  bytes { new StringByteSequence(this) }\n"
+"}\n"
+"\n"
+"class StringByteSequence is Sequence {\n"
+"  new(string) {\n"
+"    _string = string\n"
+"  }\n"
+"\n"
+"  [index] { _string.byteAt(index) }\n"
+"  iterate(iterator) { _string.iterateByte_(iterator) }\n"
+"  iteratorValue(iterator) { _string.byteAt(iterator) }\n"
+"}\n"
 "\n"
 "class List is Sequence {\n"
 "  addAll(other) {\n"
@@ -307,7 +319,7 @@ static uint32_t calculateRange(WrenVM* vm, Value* args, ObjRange* range,
                                uint32_t* length, int* step)
 {
   *step = 0;
-  
+
   // Corner case: an empty range at zero is allowed on an empty sequence.
   // This way, list[0..-1] and list[0...list.count] can be used to copy a list
   // even when empty.
@@ -1225,7 +1237,33 @@ DEF_PRIMITIVE(string_fromCodePoint)
     RETURN_ERROR("Code point cannot be greater than 0x10ffff.");
   }
 
-  RETURN_VAL(wrenStringFromCodePoint(vm, (int)AS_NUM(args[1])));
+  RETURN_VAL(wrenStringFromCodePoint(vm, codePoint));
+}
+
+DEF_PRIMITIVE(string_byteAt)
+{
+  ObjString* string = AS_STRING(args[0]);
+
+  uint32_t index = validateIndex(vm, args, string->length, 1, "Index");
+  if (index == UINT32_MAX) return PRIM_ERROR;
+
+  RETURN_NUM((uint8_t)string->value[index]);
+}
+
+DEF_PRIMITIVE(string_codePointAt)
+{
+  ObjString* string = AS_STRING(args[0]);
+
+  uint32_t index = validateIndex(vm, args, string->length, 1, "Index");
+  if (index == UINT32_MAX) return PRIM_ERROR;
+
+  // If we are in the middle of a UTF-8 sequence, indicate that.
+  const uint8_t* bytes = (uint8_t*)string->value;
+  if ((bytes[index] & 0xc0) == 0x80) RETURN_NUM(-1);
+
+  // Decode the UTF-8 sequence.
+  RETURN_NUM(wrenUtf8Decode((uint8_t*)string->value + index,
+                            string->length - index));
 }
 
 DEF_PRIMITIVE(string_contains)
@@ -1294,6 +1332,29 @@ DEF_PRIMITIVE(string_iterate)
   RETURN_NUM(index);
 }
 
+DEF_PRIMITIVE(string_iterateByte)
+{
+  ObjString* string = AS_STRING(args[0]);
+
+  // If we're starting the iteration, return the first index.
+  if (IS_NULL(args[1]))
+  {
+    if (string->length == 0) RETURN_FALSE;
+    RETURN_NUM(0);
+  }
+
+  if (!validateInt(vm, args, 1, "Iterator")) return PRIM_ERROR;
+
+  if (AS_NUM(args[1]) < 0) RETURN_FALSE;
+  uint32_t index = (uint32_t)AS_NUM(args[1]);
+
+  // Advance to the next byte.
+  index++;
+  if (index >= string->length) RETURN_FALSE;
+
+  RETURN_NUM(index);
+}
+
 DEF_PRIMITIVE(string_iteratorValue)
 {
   ObjString* string = AS_STRING(args[0]);
@@ -1533,11 +1594,14 @@ void wrenInitializeCore(WrenVM* vm)
   PRIMITIVE(vm->stringClass->obj.classObj, "fromCodePoint(_)", string_fromCodePoint);
   PRIMITIVE(vm->stringClass, "+(_)", string_plus);
   PRIMITIVE(vm->stringClass, "[_]", string_subscript);
+  PRIMITIVE(vm->stringClass, "byteAt(_)", string_byteAt);
+  PRIMITIVE(vm->stringClass, "codePointAt(_)", string_codePointAt);
   PRIMITIVE(vm->stringClass, "contains(_)", string_contains);
   PRIMITIVE(vm->stringClass, "count", string_count);
   PRIMITIVE(vm->stringClass, "endsWith(_)", string_endsWith);
   PRIMITIVE(vm->stringClass, "indexOf(_)", string_indexOf);
   PRIMITIVE(vm->stringClass, "iterate(_)", string_iterate);
+  PRIMITIVE(vm->stringClass, "iterateByte_(_)", string_iterateByte);
   PRIMITIVE(vm->stringClass, "iteratorValue(_)", string_iteratorValue);
   PRIMITIVE(vm->stringClass, "startsWith(_)", string_startsWith);
   PRIMITIVE(vm->stringClass, "toString", string_toString);

src/vm/wren_utils.c

@@ -79,14 +79,14 @@ void wrenUtf8Encode(int value, uint8_t* bytes)
   }
   else if (value <= 0x7ff)
   {
-    // Two byte sequence: 110xxxxx	 10xxxxxx.
+    // Two byte sequence: 110xxxxx 10xxxxxx.
     *bytes = 0xc0 | ((value & 0x7c0) >> 6);
     bytes++;
     *bytes = 0x80 | (value & 0x3f);
   }
   else if (value <= 0xffff)
   {
-    // Three byte sequence: 1110xxxx	 10xxxxxx 10xxxxxx.
+    // Three byte sequence: 1110xxxx 10xxxxxx 10xxxxxx.
     *bytes = 0xe0 | ((value & 0xf000) >> 12);
     bytes++;
     *bytes = 0x80 | ((value & 0xfc0) >> 6);
@@ -110,3 +110,52 @@ void wrenUtf8Encode(int value, uint8_t* bytes)
     ASSERT(false, "Invalid UTF-8 value.");
   }
 }
+
+int wrenUtf8Decode(const uint8_t* bytes, uint32_t length)
+{
+  // Single byte (i.e. fits in ASCII).
+  if (*bytes <= 0x7f) return *bytes;
+
+  int value;
+  uint32_t remainingBytes;
+  if ((*bytes & 0xe0) == 0xc0)
+  {
+    // Two byte sequence: 110xxxxx 10xxxxxx.
+    value = *bytes & 0x1f;
+    remainingBytes = 1;
+  }
+  else if ((*bytes & 0xf0) == 0xe0)
+  {
+    // Three byte sequence: 1110xxxx	 10xxxxxx 10xxxxxx.
+    value = *bytes & 0x0f;
+    remainingBytes = 2;
+  }
+  else if ((*bytes & 0xf8) == 0xf0)
+  {
+    // Four byte sequence: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx.
+    value = *bytes & 0x07;
+    remainingBytes = 3;
+  }
+  else
+  {
+    // Invalid UTF-8 sequence.
+    return -1;
+  }
+
+  // Don't read past the end of the buffer on truncated UTF-8.
+  // TODO: Test this.
+  if (remainingBytes > length - 1) return -1;
+
+  while (remainingBytes > 0)
+  {
+    bytes++;
+    remainingBytes--;
+
+    // Remaining bytes must be of form 10xxxxxx.
+    if ((*bytes & 0xc0) != 0x80) return -1;
+
+    value = value << 6 | (*bytes & 0x3f);
+  }
+
+  return value;
+}

src/vm/wren_utils.h

@@ -90,4 +90,8 @@ int wrenUtf8NumBytes(int value);
 // enough to hold the encoded result.
 void wrenUtf8Encode(int value, uint8_t* bytes);
 
+// Decodes the UTF-8 sequence in [bytes] (which has max [length]), returning
+// the code point.
+int wrenUtf8Decode(const uint8_t* bytes, uint32_t length);
+
 #endif