镜像站点 · 本页由第三方 GitHub 只读镜像提供,非 GitHub 官方站点,不接受任何登录或凭据输入。前往 github.com
Skip to content

Commit 4fdff17

Browse files
committed
perf(frontend): normalize numeric separators once (#544)
Replace incremental ArrayList growth with one exact arena allocation after counting validated separators. Preserve borrowed source storage for plain literals and raw token offsets for all normalized forms. Add decimal, fraction, exponent, radix, small/large BigInt, invalid-placement, exact-allocation, zero-allocation, and OOM witnesses. Validation: 4/4 focused TSan; 12/12 lexer; 12/12 frontend; 1608 passed, 1 skipped, 0 failed, 0 leaked in test-parallel. test262 numeric literals remain 157 -> 157 passing and BigInt literals remain 59 -> 59 passing; flips 0 test262 cases. Issue: #544
1 parent 0a212b5 commit 4fdff17

1 file changed

Lines changed: 85 additions & 5 deletions

File tree

‎src/lexer.zig‎

Lines changed: 85 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1215,12 +1215,18 @@ fn separatorsValid(s: []const u8, radix: u8) bool {
12151215
/// slice safe to hand to `parseInt`/`parseFloat`.
12161216
fn stripSeparators(arena: std.mem.Allocator, s: []const u8) LexError![]const u8 {
12171217
if (std.mem.indexOfScalar(u8, s, '_') == null and (s.len == 0 or s[s.len - 1] != 'n')) return s;
1218-
var buf: std.ArrayListUnmanaged(u8) = .empty;
1219-
for (s) |c| {
1220-
if (c == '_' or c == 'n') continue;
1221-
try buf.append(arena, c);
1218+
const end = if (s.len != 0 and s[s.len - 1] == 'n') s.len - 1 else s.len;
1219+
var separator_count: usize = 0;
1220+
for (s[0..end]) |c| separator_count += @intFromBool(c == '_');
1221+
const normalized = try arena.alloc(u8, end - separator_count);
1222+
var out: usize = 0;
1223+
for (s[0..end]) |c| {
1224+
if (c == '_') continue;
1225+
normalized[out] = c;
1226+
out += 1;
12221227
}
1223-
return buf.toOwnedSlice(arena);
1228+
std.debug.assert(out == normalized.len);
1229+
return normalized;
12241230
}
12251231

12261232
fn canonicalDecimalDigits(arena: std.mem.Allocator, digits: []const u8) LexError![]const u8 {
@@ -1397,6 +1403,80 @@ test "lexer tokenizes arithmetic" {
13971403
try std.testing.expectEqual(TokenKind.eof, (try lx.next()).kind);
13981404
}
13991405

1406+
test "lexer normalizes numeric separators in one exact allocation" {
1407+
const separated = "100_000_000_123";
1408+
var measured = std.testing.FailingAllocator.init(std.testing.allocator, .{});
1409+
const normalized = try stripSeparators(measured.allocator(), separated);
1410+
defer measured.allocator().free(normalized);
1411+
try std.testing.expectEqualStrings("100000000123", normalized);
1412+
try std.testing.expectEqual(@as(usize, 1), measured.alloc_index);
1413+
try std.testing.expectEqual(normalized.len, measured.allocated_bytes);
1414+
1415+
const plain = "100000000123";
1416+
var no_memory: [0]u8 = .{};
1417+
var fixed = std.heap.FixedBufferAllocator.init(&no_memory);
1418+
const borrowed = try stripSeparators(fixed.allocator(), plain);
1419+
try std.testing.expectEqual(@intFromPtr(plain.ptr), @intFromPtr(borrowed.ptr));
1420+
1421+
var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = 0 });
1422+
try std.testing.expectError(error.OutOfMemory, stripSeparators(failing.allocator(), separated));
1423+
}
1424+
1425+
test "lexer preserves numeric separator values forms and source offsets" {
1426+
const numbers = [_]struct { source: []const u8, expected: f64 }{
1427+
.{ .source = "1_000_000", .expected = 1_000_000 },
1428+
.{ .source = "1_2.3_4", .expected = 12.34 },
1429+
.{ .source = "1_2.5e1_0", .expected = 125_000_000_000 },
1430+
.{ .source = "0b1010_0101", .expected = 165 },
1431+
.{ .source = "0o7_7_7", .expected = 511 },
1432+
.{ .source = "0xFF_FF", .expected = 65_535 },
1433+
};
1434+
for (numbers) |case| {
1435+
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
1436+
defer arena.deinit();
1437+
var lexer = Lexer.init(arena.allocator(), case.source);
1438+
const token = try lexer.next();
1439+
try std.testing.expectEqual(TokenKind.number, token.kind);
1440+
try std.testing.expectEqual(case.expected, token.number);
1441+
try std.testing.expectEqualStrings(case.source, token.text);
1442+
try std.testing.expectEqual(@as(usize, 0), token.pos);
1443+
try std.testing.expectEqual(case.source.len, token.end);
1444+
}
1445+
1446+
var bigint_arena = std.heap.ArenaAllocator.init(std.testing.allocator);
1447+
defer bigint_arena.deinit();
1448+
var small_bigint = Lexer.init(bigint_arena.allocator(), "123_456n");
1449+
const small = try small_bigint.next();
1450+
try std.testing.expect(small.is_bigint);
1451+
try std.testing.expectEqual(@as(i128, 123_456), small.bigint);
1452+
1453+
const large_source = "340_282_366_920_938_463_463_374_607_431_768_211_456n";
1454+
var large_bigint = Lexer.init(bigint_arena.allocator(), large_source);
1455+
const large = try large_bigint.next();
1456+
try std.testing.expect(large.is_bigint);
1457+
try std.testing.expectEqualStrings("340282366920938463463374607431768211456", large.bigint_text.?);
1458+
try std.testing.expectEqualStrings(large_source, large.text);
1459+
try std.testing.expectEqual(large_source.len, large.end);
1460+
1461+
const invalid = [_][]const u8{
1462+
"1__0",
1463+
"1_",
1464+
"0x_1",
1465+
"0b1_",
1466+
"1_.0",
1467+
"1._0",
1468+
"1e_1",
1469+
"1e1_",
1470+
"1_n",
1471+
};
1472+
for (invalid) |source| {
1473+
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
1474+
defer arena.deinit();
1475+
var lexer = Lexer.init(arena.allocator(), source);
1476+
try std.testing.expectError(LexError.InvalidNumber, lexer.next());
1477+
}
1478+
}
1479+
14001480
test "lexer borrows ordinary identifiers and owns escaped decoding" {
14011481
const ordinary_source = "ordinary_π";
14021482
var no_memory: [0]u8 = .{};

0 commit comments

Comments
 (0)