feat: add StringCodePointSequence class and expose codePoints property on String

Extract codePointAt() logic into a dedicated StringCodePointSequence class that wraps a string and implements the Sequence interface, enabling iteration over code points. Add a `codePoints` getter to String that returns a new instance of this sequence. Refactor the underlying C implementation of `codePointAt_` to decode UTF-8 into actual code point integers (returning raw bytes for invalid sequences) and rename the primitive to use trailing underscore convention. Update all related tests to reflect the new behavior where mid-sequence byte indices return the raw byte string instead of an empty string, and add new test files for the code point sequence iteration.

test/core/string/code_point_at.wren

@@ -1,38 +0,0 @@
-// Bytes:           11111
-//        012345678901234
-// Chars: sø mé ஃ  thî ng
-var s = "søméஃthîng"
-
-IO.print(s.codePointAt(0)) // expect: 115
-IO.print(s.codePointAt(1)) // expect: 248
-IO.print(s.codePointAt(2)) // expect: -1
-IO.print(s.codePointAt(3)) // expect: 109
-IO.print(s.codePointAt(4)) // expect: 233
-IO.print(s.codePointAt(5)) // expect: -1
-IO.print(s.codePointAt(6)) // expect: 2947
-IO.print(s.codePointAt(7)) // expect: -1
-IO.print(s.codePointAt(8)) // expect: -1
-IO.print(s.codePointAt(9)) // expect: 116
-IO.print(s.codePointAt(10)) // expect: 104
-IO.print(s.codePointAt(11)) // expect: 238
-IO.print(s.codePointAt(12)) // expect: -1
-IO.print(s.codePointAt(13)) // expect: 110
-IO.print(s.codePointAt(14)) // expect: 103
-
-IO.print(s.codePointAt(-15)) // expect: 115
-IO.print(s.codePointAt(-14)) // expect: 248
-IO.print(s.codePointAt(-13)) // expect: -1
-IO.print(s.codePointAt(-12)) // expect: 109
-IO.print(s.codePointAt(-11)) // expect: 233
-IO.print(s.codePointAt(-10)) // expect: -1
-IO.print(s.codePointAt(-9)) // expect: 2947
-IO.print(s.codePointAt(-8)) // expect: -1
-IO.print(s.codePointAt(-7)) // expect: -1
-IO.print(s.codePointAt(-6)) // expect: 116
-IO.print(s.codePointAt(-5)) // expect: 104
-IO.print(s.codePointAt(-4)) // expect: 238
-IO.print(s.codePointAt(-3)) // expect: -1
-IO.print(s.codePointAt(-2)) // expect: 110
-IO.print(s.codePointAt(-1)) // expect: 103
-
-IO.print("\0".codePointAt(0)) // expect: 0

test/core/string/code_point_at_incomplete.wren

@@ -1,8 +0,0 @@
-// The first byte of a two-octet sequence.
-IO.print("\xc0".codePointAt(0)) // expect: -1
-
-// The first byte of a three-octet sequence.
-IO.print("\xe0".codePointAt(0)) // expect: -1
-
-// The first two bytes of a three-octet sequence.
-IO.print("\xe0\xae".codePointAt(0)) // expect: -1

test/core/string/code_point_at_not_int.wren

@@ -1 +0,0 @@
-IO.print("string".codePointAt(12.34)) // expect runtime error: Index must be an integer.

test/core/string/code_point_at_not_num.wren

@@ -1 +0,0 @@
-IO.print("string".codePointAt("not num")) // expect runtime error: Index must be a number.

test/core/string/code_point_at_too_large.wren

@@ -1 +0,0 @@
-IO.print("string".codePointAt(6)) // expect runtime error: Index out of bounds.

test/core/string/code_point_at_too_small.wren

@@ -1 +0,0 @@
-IO.print("string".codePointAt(-7)) // expect runtime error: Index out of bounds.

test/core/string/iterate.wren

@@ -22,3 +22,8 @@ IO.print("a\0b\0c".iterate(1)) // expect: 2
 IO.print("a\0b\0c".iterate(2)) // expect: 3
 IO.print("a\0b\0c".iterate(3)) // expect: 4
 IO.print("a\0b\0c".iterate(4)) // expect: false
+
+// Iterates over invalid UTF-8 one byte at a time.
+IO.print("\xef\xf7".iterate(null)) // expect: 0
+IO.print("\xef\xf7".iterate(0)) // expect: 1
+IO.print("\xef\xf7".iterate(1)) // expect: false

test/core/string/iterator_value.wren

@@ -2,8 +2,8 @@ var s = "abçd"
 IO.print(s.iteratorValue(0)) // expect: a
 IO.print(s.iteratorValue(1)) // expect: b
 IO.print(s.iteratorValue(2)) // expect: ç
-// Iterator value in middle of UTF sequence is an empty string.
-IO.print(s.iteratorValue(3) == "") // expect: true
+// Iterator value in middle of UTF sequence is the unencoded byte.
+IO.print(s.iteratorValue(3) == "\xa7") // expect: true
 IO.print(s.iteratorValue(4)) // expect: d
 
 // 8-bit clean.
@@ -13,3 +13,7 @@ IO.print(t.iteratorValue(1) == "\0") // expect: true
 IO.print(t.iteratorValue(2) == "b") // expect: true
 IO.print(t.iteratorValue(3) == "\0") // expect: true
 IO.print(t.iteratorValue(4) == "c") // expect: true
+
+// Returns single byte strings for invalid UTF-8 sequences.
+IO.print("\xef\xf7".iteratorValue(0) == "\xef") // expect: true
+IO.print("\xef\xf7".iteratorValue(1) == "\xf7") // expect: true

test/core/string/subscript.wren

@@ -27,12 +27,12 @@ IO.print("søméஃthîng"[-1]) // expect: g
 IO.print("søméஃthîng"[-2]) // expect: n
 IO.print("søméஃthîng"[-4]) // expect: î
 
-// If the subscript is in the middle of a UTF-8 sequence, yield an empty string.
-IO.print("søméஃthîng"[2] == "") // expect: true
-IO.print("søméஃthîng"[7] == "") // expect: true
-IO.print("søméஃthîng"[8] == "") // expect: true
-IO.print("søméஃ"[-1] == "") // expect: true
-IO.print("søméஃ"[-2] == "") // expect: true
+// If the subscript is in the middle of a UTF-8 sequence, return the raw byte.
+IO.print("søméஃthîng"[2] == "\xb8") // expect: true
+IO.print("søméஃthîng"[7] == "\xae") // expect: true
+IO.print("søméஃthîng"[8] == "\x83") // expect: true
+IO.print("søméஃ"[-1] == "\x83") // expect: true
+IO.print("søméஃ"[-2] == "\xae") // expect: true
 
 // 8-bit clean.
 IO.print("a\0b\0c"[0] == "a") // expect: true
@@ -40,3 +40,7 @@ IO.print("a\0b\0c"[1] == "\0") // expect: true
 IO.print("a\0b\0c"[2] == "b") // expect: true
 IO.print("a\0b\0c"[3] == "\0") // expect: true
 IO.print("a\0b\0c"[4] == "c") // expect: true
+
+// Returns single byte strings for invalid UTF-8 sequences.
+IO.print("\xef\xf7"[0] == "\xef") // expect: true
+IO.print("\xef\xf7"[1] == "\xf7") // expect: true

test/core/string/subscript_range.wren

@@ -45,3 +45,5 @@ IO.print("søméஃthîng"[3...10]) // expect: méஃt
 IO.print("søméஃthîng"[2..6]) // expect: méஃ
 IO.print("søméஃthîng"[2...6]) // expect: mé
 IO.print("søméஃthîng"[2...7]) // expect: méஃ
+
+// TODO: Strings including invalid UTF-8.