String.fromCodePoint(). Fix #219.

This commit is contained in:
Bob Nystrom
2015-03-27 07:43:36 -07:00
parent 373770a8d5
commit 7d45dda383
23 changed files with 166 additions and 43 deletions
+11 -32
View File
@@ -648,39 +648,18 @@ static void readUnicodeEscape(Parser* parser)
value = (value * 16) | digit;
}
ByteBuffer* buffer = &parser->string;
// Grow the buffer enough for the encoded result.
int numBytes = wrenUtf8NumBytes(value);
if (numBytes != 0)
{
// TODO: Function to grow buffer in one allocation.
for (int i = 0; i < numBytes; i++)
{
wrenByteBufferWrite(parser->vm, &parser->string, 0);
}
// UTF-8 encode the value.
if (value <= 0x7f)
{
// Single byte (i.e. fits in ASCII).
wrenByteBufferWrite(parser->vm, buffer, value & 0x7f);
}
else if (value <= 0x7ff)
{
// Two byte sequence: 110xxxxx 10xxxxxx.
wrenByteBufferWrite(parser->vm, buffer, 0xc0 | ((value & 0x7c0) >> 6));
wrenByteBufferWrite(parser->vm, buffer, 0x80 | (value & 0x3f));
}
else if (value <= 0xffff)
{
// Three byte sequence: 1110xxxx 10xxxxxx 10xxxxxx.
wrenByteBufferWrite(parser->vm, buffer, 0xe0 | ((value & 0xf000) >> 12));
wrenByteBufferWrite(parser->vm, buffer, 0x80 | ((value & 0xfc0) >> 6));
wrenByteBufferWrite(parser->vm, buffer, 0x80 | (value & 0x3f));
}
else if (value <= 0x10ffff)
{
// Four byte sequence: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx.
wrenByteBufferWrite(parser->vm, buffer, 0xf0 | ((value & 0x1c0000) >> 18));
wrenByteBufferWrite(parser->vm, buffer, 0x80 | ((value & 0x3f000) >> 12));
wrenByteBufferWrite(parser->vm, buffer, 0x80 | ((value & 0xfc0) >> 6));
wrenByteBufferWrite(parser->vm, buffer, 0x80 | (value & 0x3f));
}
else
{
// Invalid Unicode value. See: http://tools.ietf.org/html/rfc3629
// TODO: Error.
wrenUtf8Encode(value,
parser->string.data + parser->string.count - numBytes);
}
}
+18
View File
@@ -1211,6 +1211,23 @@ DEF_PRIMITIVE(range_toString)
RETURN_VAL(result);
}
DEF_PRIMITIVE(string_fromCodePoint)
{
if (!validateInt(vm, args, 1, "Code point")) return PRIM_ERROR;
int codePoint = (int)AS_NUM(args[1]);
if (codePoint < 0)
{
RETURN_ERROR("Code point cannot be negative.");
}
else if (codePoint > 0x10ffff)
{
RETURN_ERROR("Code point cannot be greater than 0x10ffff.");
}
RETURN_VAL(wrenStringFromCodePoint(vm, (int)AS_NUM(args[1])));
}
DEF_PRIMITIVE(string_contains)
{
if (!validateString(vm, args, 1, "Argument")) return PRIM_ERROR;
@@ -1513,6 +1530,7 @@ void wrenInitializeCore(WrenVM* vm)
wrenInterpret(vm, "", libSource);
vm->stringClass = AS_CLASS(wrenFindVariable(vm, "String"));
PRIMITIVE(vm->stringClass->obj.classObj, "fromCodePoint(_)", string_fromCodePoint);
PRIMITIVE(vm->stringClass, "+(_)", string_plus);
PRIMITIVE(vm->stringClass, "[_]", string_subscript);
PRIMITIVE(vm->stringClass, "contains(_)", string_contains);
+52
View File
@@ -58,3 +58,55 @@ int wrenSymbolTableFind(SymbolTable* symbols, const char* name, size_t length)
return -1;
}
int wrenUtf8NumBytes(int value)
{
ASSERT(value >= 0, "Cannot encode a negative value.");
if (value <= 0x7f) return 1;
if (value <= 0x7ff) return 2;
if (value <= 0xffff) return 3;
if (value <= 0x10ffff) return 4;
return 0;
}
void wrenUtf8Encode(int value, uint8_t* bytes)
{
if (value <= 0x7f)
{
// Single byte (i.e. fits in ASCII).
*bytes = value & 0x7f;
}
else if (value <= 0x7ff)
{
// Two byte sequence: 110xxxxx 10xxxxxx.
*bytes = 0xc0 | ((value & 0x7c0) >> 6);
bytes++;
*bytes = 0x80 | (value & 0x3f);
}
else if (value <= 0xffff)
{
// Three byte sequence: 1110xxxx 10xxxxxx 10xxxxxx.
*bytes = 0xe0 | ((value & 0xf000) >> 12);
bytes++;
*bytes = 0x80 | ((value & 0xfc0) >> 6);
bytes++;
*bytes = 0x80 | (value & 0x3f);
}
else if (value <= 0x10ffff)
{
// Four byte sequence: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx.
*bytes = 0xf0 | ((value & 0x1c0000) >> 18);
bytes++;
*bytes = 0x80 | ((value & 0x3f000) >> 12);
bytes++;
*bytes = 0x80 | ((value & 0xfc0) >> 6);
bytes++;
*bytes = 0x80 | (value & 0x3f);
}
else
{
// Invalid Unicode value. See: http://tools.ietf.org/html/rfc3629
ASSERT(false, "Invalid UTF-8 value.");
}
}
+9
View File
@@ -81,4 +81,13 @@ int wrenSymbolTableEnsure(WrenVM* vm, SymbolTable* symbols,
// Looks up name in the symbol table. Returns its index if found or -1 if not.
int wrenSymbolTableFind(SymbolTable* symbols, const char* name, size_t length);
// Returns the number of bytes needed to encode [value] in UTF-8.
//
// Returns 0 if [value] is too large to encode.
int wrenUtf8NumBytes(int value);
// Encodes value as a series of bytes in [bytes], which is assumed to be large
// enough to hold the encoded result.
void wrenUtf8Encode(int value, uint8_t* bytes);
#endif
+13
View File
@@ -650,6 +650,19 @@ Value wrenNumToString(WrenVM* vm, double value)
return wrenNewString(vm, buffer, length);
}
Value wrenStringFromCodePoint(WrenVM* vm, int value)
{
int length = wrenUtf8NumBytes(value);
ASSERT(length != 0, "Value out of range.");
ObjString* string = allocateString(vm, length);
wrenUtf8Encode(value, (uint8_t*)string->value);
hashString(string);
return OBJ_VAL(string);
}
Value wrenStringFormat(WrenVM* vm, const char* format, ...)
{
va_list argList;
+3
View File
@@ -683,6 +683,9 @@ Value wrenNumToString(WrenVM* vm, double value);
// @ - A Wren string object.
Value wrenStringFormat(WrenVM* vm, const char* format, ...);
// Creates a new string containing the UTF-8 encoding of [value].
Value wrenStringFromCodePoint(WrenVM* vm, int value);
// Creates a new string containing the code point in [string] starting at byte
// [index]. If [index] points into the middle of a UTF-8 sequence, returns an
// empty string.