retoor/wren
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simple string interpolation.

Browse Source 
This allows "%(...)" inside a string literal to interpolate the
stringified result of an expression.

It doesn't support custom interpolators or format strings, but we can
consider extending that later.
This commit is contained in:
Bob Nystrom
2015-11-11 07:55:48 -08:00
parent 71575d9179
commit 78655c68b0
41 changed files with 321 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vm/wren_compiler.c
+145 -36
View File
@@ -24,17 +24,23 @@
// Note that this limitation is also explicit in the bytecode. Since
// `CODE_LOAD_LOCAL` and `CODE_STORE_LOCAL` use a single argument byte to
// identify the local, only 256 can be in scope at one time.
#define MAX_LOCALS (256)
#define MAX_LOCALS 256
// The maximum number of upvalues (i.e. variables from enclosing functions)
// that a function can close over.
#define MAX_UPVALUES (256)
#define MAX_UPVALUES 256
// The maximum number of distinct constants that a function can contain. This
// value is explicit in the bytecode since `CODE_CONSTANT` only takes a single
// two-byte argument.
#define MAX_CONSTANTS (1 << 16)
// The maximum depth that interpolation can nest. For example, this string has
// three levels:
//
// "outside %(one + "%(two + "%(three)")")"
#define MAX_INTERPOLATION_NESTING 8
typedef enum
{
TOKEN_LEFT_PAREN,
@@ -95,7 +101,24 @@ typedef enum
TOKEN_STATIC_FIELD,
TOKEN_NAME,
TOKEN_NUMBER,
// A string literal without any interpolation, or the last section of a
// string following the last interpolated expression.
TOKEN_STRING,
// A portion of a string literal preceding an interpolated expression. This
// string:
//
// "a %(b) c %(d) e"
//
// is tokenized to:
//
// TOKEN_INTERPOLATION "a "
// TOKEN_NAME b
// TOKEN_INTERPOLATION " c "
// TOKEN_NAME d
// TOKEN_STRING " e"
TOKEN_INTERPOLATION,
TOKEN_LINE,
@@ -115,6 +138,9 @@ typedef struct
// The 1-based line where the token appears.
int line;
// The parsed value if the token is a literal.
Value value;
} Token;
typedef struct
@@ -141,7 +167,23 @@ typedef struct
// The most recently consumed/advanced token.
Token previous;
// Tracks the lexing state when tokenizing interpolated strings.
//
// Interpolated strings make the lexer not strictly regular: we don't know
// whether a ")" should be treated as a RIGHT_PAREN token or as ending an
// interpolated expression unless we know whether we are inside a string
// interpolation and how many unmatched "(" there are. This is particularly
// complex because interpolation can nest:
//
// " %( " %( inner ) " ) "
//
// This tracks that state. The parser maintains a stack of ints, one for each
// level of current interpolation nesting. Each value is the number of
// unmatched "(" that are waiting to be closed.
int parens[MAX_INTERPOLATION_NESTING];
int numParens;
// If subsequent newline tokens should be discarded.
bool skipNewlines;
@@ -150,9 +192,6 @@ typedef struct
// If a syntax or compile error has occurred.
bool hasError;
// The parsed value if the current token is a literal.
Value value;
} Parser;
typedef struct
@@ -520,7 +559,7 @@ static void makeToken(Parser* parser, TokenType type)
parser->current.start = parser->tokenStart;
parser->current.length = (int)(parser->currentChar - parser->tokenStart);
parser->current.line = parser->currentLine;
// Make line tokens appear on the line containing the "\n".
if (type == TOKEN_LINE) parser->current.line--;
}
@@ -596,13 +635,13 @@ static void makeNumber(Parser* parser, bool isHex)
// We don't check that the entire token is consumed because we've already
// scanned it ourselves and know it's valid.
parser->value = NUM_VAL(isHex ? strtol(parser->tokenStart, NULL, 16)
: strtod(parser->tokenStart, NULL));
parser->current.value = NUM_VAL(isHex ? strtol(parser->tokenStart, NULL, 16)
: strtod(parser->tokenStart, NULL));
if (errno == ERANGE)
{
lexError(parser, "Number literal was too large.");
parser->value = NUM_VAL(0);
parser->current.value = NUM_VAL(0);
}
makeToken(parser, TOKEN_NUMBER);
@@ -720,8 +759,9 @@ static void readUnicodeEscape(Parser* parser, ByteBuffer* string, int length)
static void readString(Parser* parser)
{
ByteBuffer string;
TokenType type = TOKEN_STRING;
wrenByteBufferInit(&string);
for (;;)
{
char c = nextChar(parser);
@@ -737,12 +777,30 @@ static void readString(Parser* parser)
break;
}
if (c == '%')
{
if (parser->numParens < MAX_INTERPOLATION_NESTING)
{
// TODO: Allow format string.
if (nextChar(parser) != '(') lexError(parser, "Expect '(' after '%'.");
parser->parens[parser->numParens] = 1;
parser->numParens++;
type = TOKEN_INTERPOLATION;
break;
}
lexError(parser, "Interpolation may only nest %d levels deep.",
MAX_INTERPOLATION_NESTING);
}
if (c == '\\')
{
switch (nextChar(parser))
{
case '"': wrenByteBufferWrite(parser->vm, &string, '"'); break;
case '\\': wrenByteBufferWrite(parser->vm, &string, '\\'); break;
case '%': wrenByteBufferWrite(parser->vm, &string, '%'); break;
case '0': wrenByteBufferWrite(parser->vm, &string, '\0'); break;
case 'a': wrenByteBufferWrite(parser->vm, &string, '\a'); break;
case 'b': wrenByteBufferWrite(parser->vm, &string, '\b'); break;
@@ -770,9 +828,11 @@ static void readString(Parser* parser)
}
}
parser->value = wrenNewString(parser->vm, (char*)string.data, string.count);
parser->current.value = wrenNewString(parser->vm,
(char*)string.data, string.count);
wrenByteBufferClear(parser->vm, &string);
makeToken(parser, TOKEN_STRING);
makeToken(parser, type);
}
// Lex the next token and store it in [parser.current].
@@ -784,7 +844,7 @@ static void nextToken(Parser* parser)
// copy the TOKEN_EOF to previous so that code that expects it to be consumed
// will still work.
if (parser->current.type == TOKEN_EOF) return;
while (peekChar(parser) != '\0')
{
parser->tokenStart = parser->currentChar;
@@ -792,8 +852,27 @@ static void nextToken(Parser* parser)
char c = nextChar(parser);
switch (c)
{
case '(': makeToken(parser, TOKEN_LEFT_PAREN); return;
case ')': makeToken(parser, TOKEN_RIGHT_PAREN); return;
case '(':
// If we are inside an interpolated expression, count the unmatched "(".
if (parser->numParens > 0) parser->parens[parser->numParens - 1]++;
makeToken(parser, TOKEN_LEFT_PAREN);
return;
case ')':
// If we are inside an interpolated expression, count the ")".
if (parser->numParens > 0 &&
--parser->parens[parser->numParens - 1] == 0)
{
// This is the final ")", so the interpolation expression has ended.
// This ")" now begins the next section of the template string.
parser->numParens--;
readString(parser);
return;
}
makeToken(parser, TOKEN_RIGHT_PAREN);
return;
case '[': makeToken(parser, TOKEN_LEFT_BRACKET); return;
case ']': makeToken(parser, TOKEN_RIGHT_BRACKET); return;
case '{': makeToken(parser, TOKEN_LEFT_BRACE); return;
@@ -1835,15 +1914,9 @@ static void list(Compiler* compiler, bool allowAssignment)
// Stop if we hit the end of the list.
if (peek(compiler) == TOKEN_RIGHT_BRACKET) break;
// Push a copy of the list since the add() call will consume it.
emit(compiler, CODE_DUP);
// The element.
expression(compiler);
callMethod(compiler, 1, "add(_)", 6);
// Discard the result of the add() call.
emit(compiler, CODE_POP);
callMethod(compiler, 1, "addCore_(_)", 11);
} while (match(compiler, TOKEN_COMMA));
// Allow newlines before the closing ']'.
@@ -1867,9 +1940,6 @@ static void map(Compiler* compiler, bool allowAssignment)
// Stop if we hit the end of the map.
if (peek(compiler) == TOKEN_RIGHT_BRACE) break;
// Push a copy of the map since the subscript call will consume it.
emit(compiler, CODE_DUP);
// The key.
parsePrecedence(compiler, false, PREC_PRIMARY);
consume(compiler, TOKEN_COLON, "Expect ':' after map key.");
@@ -1877,10 +1947,7 @@ static void map(Compiler* compiler, bool allowAssignment)
// The value.
expression(compiler);
callMethod(compiler, 2, "[_]=(_)", 7);
// Discard the result of the setter call.
emit(compiler, CODE_POP);
callMethod(compiler, 2, "addCore_(_,_)", 13);
} while (match(compiler, TOKEN_COMMA));
// Allow newlines before the closing '}'.
@@ -2138,7 +2205,46 @@ static void null(Compiler* compiler, bool allowAssignment)
// A number or string literal.
static void literal(Compiler* compiler, bool allowAssignment)
{
emitConstant(compiler, compiler->parser->value);
emitConstant(compiler, compiler->parser->previous.value);
}
static void stringInterpolation(Compiler* compiler, bool allowAssignment)
{
// TODO: Allow other expressions here so that user-defined classes can control
// interpolation processing like "tagged template strings" in ES6.
loadCoreVariable(compiler, "String");
// Instantiate a new list.
loadCoreVariable(compiler, "List");
callMethod(compiler, 0, "new()", 5);
do
{
// The opening string part.
literal(compiler, false);
callMethod(compiler, 1, "addCore_(_)", 11);
ignoreNewlines(compiler);
// Compile the interpolated expression part to a function.
Compiler fnCompiler;
initCompiler(&fnCompiler, compiler->parser, compiler, true);
expression(&fnCompiler);
emit(&fnCompiler, CODE_RETURN);
endCompiler(&fnCompiler, "interpolation", 9);
callMethod(compiler, 1, "addCore_(_)", 11);
ignoreNewlines(compiler);
} while (match(compiler, TOKEN_INTERPOLATION));
// The trailing string part.
consume(compiler, TOKEN_STRING, "Expect end of string interpolation.");
literal(compiler, false);
callMethod(compiler, 1, "addCore_(_)", 11);
// Call .interpolate() with the list of interpolation parts.
callMethod(compiler, 1, "interpolate_(_)", 15);
}
static void super_(Compiler* compiler, bool allowAssignment)
@@ -2482,6 +2588,7 @@ GrammarRule rules[] =
/* TOKEN_NAME */ { name, NULL, namedSignature, PREC_NONE, NULL },
/* TOKEN_NUMBER */ PREFIX(literal),
/* TOKEN_STRING */ PREFIX(literal),
/* TOKEN_INTERPOLATION */ PREFIX(stringInterpolation),
/* TOKEN_LINE */ UNUSED,
/* TOKEN_ERROR */ UNUSED,
/* TOKEN_EOF */ UNUSED
@@ -3131,7 +3238,7 @@ static void classDefinition(Compiler* compiler, bool isForeign)
static void import(Compiler* compiler)
{
consume(compiler, TOKEN_STRING, "Expect a string after 'import'.");
int moduleConstant = addConstant(compiler, compiler->parser->value);
int moduleConstant = addConstant(compiler, compiler->parser->previous.value);
// Load the module.
emitShortArg(compiler, CODE_LOAD_MODULE, moduleConstant);
@@ -3221,11 +3328,11 @@ ObjFn* wrenCompile(WrenVM* vm, ObjModule* module, const char* source,
parser.vm = vm;
parser.module = module;
parser.source = source;
parser.value = UNDEFINED_VAL;
parser.tokenStart = source;
parser.currentChar = source;
parser.currentLine = 1;
parser.numParens = 0;
// Zero-init the current token. This will get copied to previous when
// advance() is called below.
@@ -3233,6 +3340,7 @@ ObjFn* wrenCompile(WrenVM* vm, ObjModule* module, const char* source,
parser.current.start = source;
parser.current.length = 0;
parser.current.line = 0;
parser.current.value = UNDEFINED_VAL;
// Ignore leading newlines.
parser.skipNewlines = true;
@@ -3272,7 +3380,7 @@ ObjFn* wrenCompile(WrenVM* vm, ObjModule* module, const char* source,
parser.module->variableNames.data[i].buffer);
}
}
return endCompiler(&compiler, "(script)", 8);
}
@@ -3344,7 +3452,8 @@ void wrenBindMethodCode(ObjClass* classObj, ObjFn* fn)
void wrenMarkCompiler(WrenVM* vm, Compiler* compiler)
{
wrenGrayValue(vm, compiler->parser->value);
wrenGrayValue(vm, compiler->parser->current.value);
wrenGrayValue(vm, compiler->parser->previous.value);
// Walk up the parent chain to mark the outer compilers too. The VM only
// tracks the innermost one.
src/vm/wren_core.c
+26
View File
@@ -264,6 +264,17 @@ DEF_PRIMITIVE(list_add)
RETURN_VAL(args[1]);
}
// Adds an element to the list and then returns the list itself. This is called
// by the compiler when compiling list literals instead of using add() to
// minimize stack churn.
DEF_PRIMITIVE(list_addCore)
{
wrenValueBufferWrite(vm, &AS_LIST(args[0])->elements, args[1]);
// Return the list.
RETURN_VAL(args[0]);
}
DEF_PRIMITIVE(list_clear)
{
wrenValueBufferClear(vm, &AS_LIST(args[0])->elements);
@@ -394,6 +405,19 @@ DEF_PRIMITIVE(map_subscriptSetter)
RETURN_VAL(args[2]);
}
// Adds an entry to the map and then returns the map itself. This is called by
// the compiler when compiling map literals instead of using [_]=(_) to
// minimize stack churn.
DEF_PRIMITIVE(map_addCore)
{
if (!validateKey(vm, args[1])) return false;
wrenMapSet(vm, AS_MAP(args[0]), args[1], args[2]);
// Return the map itself.
RETURN_VAL(args[0]);
}
DEF_PRIMITIVE(map_clear)
{
wrenMapClear(vm, AS_MAP(args[0]));
@@ -1229,6 +1253,7 @@ void wrenInitializeCore(WrenVM* vm)
PRIMITIVE(vm->listClass, "[_]", list_subscript);
PRIMITIVE(vm->listClass, "[_]=(_)", list_subscriptSetter);
PRIMITIVE(vm->listClass, "add(_)", list_add);
PRIMITIVE(vm->listClass, "addCore_(_)", list_addCore);
PRIMITIVE(vm->listClass, "clear()", list_clear);
PRIMITIVE(vm->listClass, "count", list_count);
PRIMITIVE(vm->listClass, "insert(_,_)", list_insert);
@@ -1240,6 +1265,7 @@ void wrenInitializeCore(WrenVM* vm)
PRIMITIVE(vm->mapClass->obj.classObj, "new()", map_new);
PRIMITIVE(vm->mapClass, "[_]", map_subscript);
PRIMITIVE(vm->mapClass, "[_]=(_)", map_subscriptSetter);
PRIMITIVE(vm->mapClass, "addCore_(_,_)", map_addCore);
PRIMITIVE(vm->mapClass, "clear()", map_clear);
PRIMITIVE(vm->mapClass, "containsKey(_)", map_containsKey);
PRIMITIVE(vm->mapClass, "count", map_count);
src/vm/wren_core.wren
+15 -2
View File
@@ -131,6 +131,19 @@ class WhereSequence is Sequence {
class String is Sequence {
bytes { StringByteSequence.new(this) }
codePoints { StringCodePointSequence.new(this) }
static interpolate_(parts) {
var result = ""
for (part in parts) {
if (part is String) {
result = result + part
} else {
result = result + part.call().toString
}
}
return result
}
}
class StringByteSequence is Sequence {
@@ -165,7 +178,7 @@ class List is Sequence {
return other
}
toString { "[" + join(", ") + "]" }
toString { "[%(join(", "))]" }
+(other) {
var result = this[0..-1]
@@ -187,7 +200,7 @@ class Map {
for (key in keys) {
if (!first) result = result + ", "
first = false
result = result + key.toString + ": " + this[key].toString
result = result + "%(key): %(this[key])"
}
return result + "}"
src/vm/wren_core.wren.inc
+15 -2
View File
@@ -133,6 +133,19 @@ static const char* coreModuleSource =
"class String is Sequence {\n"
" bytes { StringByteSequence.new(this) }\n"
" codePoints { StringCodePointSequence.new(this) }\n"
"\n"
" static interpolate_(parts) {\n"
" var result = \"\"\n"
" for (part in parts) {\n"
" if (part is String) {\n"
" result = result + part\n"
" } else {\n"
" result = result + part.call().toString\n"
" }\n"
" }\n"
"\n"
" return result\n"
" }\n"
"}\n"
"\n"
"class StringByteSequence is Sequence {\n"
@@ -167,7 +180,7 @@ static const char* coreModuleSource =
" return other\n"
" }\n"
"\n"
" toString { \"[\" + join(\", \") + \"]\" }\n"
" toString { \"[%(join(\", \"))]\" }\n"
"\n"
" +(other) {\n"
" var result = this[0..-1]\n"
@@ -189,7 +202,7 @@ static const char* coreModuleSource =
" for (key in keys) {\n"
" if (!first) result = result + \", \"\n"
" first = false\n"
" result = result + key.toString + \": \" + this[key].toString\n"
" result = result + \"%(key): %(this[key])\"\n"
" }\n"
"\n"
" return result + \"}\"\n"