@@ -1215,12 +1215,18 @@ fn separatorsValid(s: []const u8, radix: u8) bool {
12151215/// slice safe to hand to `parseInt`/`parseFloat`.
12161216fn stripSeparators (arena : std.mem.Allocator , s : []const u8 ) LexError ! []const u8 {
12171217 if (std .mem .indexOfScalar (u8 , s , '_' ) == null and (s .len == 0 or s [s .len - 1 ] != 'n' )) return s ;
1218- var buf : std .ArrayListUnmanaged (u8 ) = .empty ;
1219- for (s ) | c | {
1220- if (c == '_' or c == 'n' ) continue ;
1221- try buf .append (arena , c );
1218+ const end = if (s .len != 0 and s [s .len - 1 ] == 'n' ) s .len - 1 else s .len ;
1219+ var separator_count : usize = 0 ;
1220+ for (s [0.. end ]) | c | separator_count += @intFromBool (c == '_' );
1221+ const normalized = try arena .alloc (u8 , end - separator_count );
1222+ var out : usize = 0 ;
1223+ for (s [0.. end ]) | c | {
1224+ if (c == '_' ) continue ;
1225+ normalized [out ] = c ;
1226+ out += 1 ;
12221227 }
1223- return buf .toOwnedSlice (arena );
1228+ std .debug .assert (out == normalized .len );
1229+ return normalized ;
12241230}
12251231
12261232fn canonicalDecimalDigits (arena : std.mem.Allocator , digits : []const u8 ) LexError ! []const u8 {
@@ -1397,6 +1403,80 @@ test "lexer tokenizes arithmetic" {
13971403 try std .testing .expectEqual (TokenKind .eof , (try lx .next ()).kind );
13981404}
13991405
1406+ test "lexer normalizes numeric separators in one exact allocation" {
1407+ const separated = "100_000_000_123" ;
1408+ var measured = std .testing .FailingAllocator .init (std .testing .allocator , .{});
1409+ const normalized = try stripSeparators (measured .allocator (), separated );
1410+ defer measured .allocator ().free (normalized );
1411+ try std .testing .expectEqualStrings ("100000000123" , normalized );
1412+ try std .testing .expectEqual (@as (usize , 1 ), measured .alloc_index );
1413+ try std .testing .expectEqual (normalized .len , measured .allocated_bytes );
1414+
1415+ const plain = "100000000123" ;
1416+ var no_memory : [0 ]u8 = .{};
1417+ var fixed = std .heap .FixedBufferAllocator .init (& no_memory );
1418+ const borrowed = try stripSeparators (fixed .allocator (), plain );
1419+ try std .testing .expectEqual (@intFromPtr (plain .ptr ), @intFromPtr (borrowed .ptr ));
1420+
1421+ var failing = std .testing .FailingAllocator .init (std .testing .allocator , .{ .fail_index = 0 });
1422+ try std .testing .expectError (error .OutOfMemory , stripSeparators (failing .allocator (), separated ));
1423+ }
1424+
1425+ test "lexer preserves numeric separator values forms and source offsets" {
1426+ const numbers = [_ ]struct { source : []const u8 , expected : f64 }{
1427+ .{ .source = "1_000_000" , .expected = 1_000_000 },
1428+ .{ .source = "1_2.3_4" , .expected = 12.34 },
1429+ .{ .source = "1_2.5e1_0" , .expected = 125_000_000_000 },
1430+ .{ .source = "0b1010_0101" , .expected = 165 },
1431+ .{ .source = "0o7_7_7" , .expected = 511 },
1432+ .{ .source = "0xFF_FF" , .expected = 65_535 },
1433+ };
1434+ for (numbers ) | case | {
1435+ var arena = std .heap .ArenaAllocator .init (std .testing .allocator );
1436+ defer arena .deinit ();
1437+ var lexer = Lexer .init (arena .allocator (), case .source );
1438+ const token = try lexer .next ();
1439+ try std .testing .expectEqual (TokenKind .number , token .kind );
1440+ try std .testing .expectEqual (case .expected , token .number );
1441+ try std .testing .expectEqualStrings (case .source , token .text );
1442+ try std .testing .expectEqual (@as (usize , 0 ), token .pos );
1443+ try std .testing .expectEqual (case .source .len , token .end );
1444+ }
1445+
1446+ var bigint_arena = std .heap .ArenaAllocator .init (std .testing .allocator );
1447+ defer bigint_arena .deinit ();
1448+ var small_bigint = Lexer .init (bigint_arena .allocator (), "123_456n" );
1449+ const small = try small_bigint .next ();
1450+ try std .testing .expect (small .is_bigint );
1451+ try std .testing .expectEqual (@as (i128 , 123_456 ), small .bigint );
1452+
1453+ const large_source = "340_282_366_920_938_463_463_374_607_431_768_211_456n" ;
1454+ var large_bigint = Lexer .init (bigint_arena .allocator (), large_source );
1455+ const large = try large_bigint .next ();
1456+ try std .testing .expect (large .is_bigint );
1457+ try std .testing .expectEqualStrings ("340282366920938463463374607431768211456" , large .bigint_text .? );
1458+ try std .testing .expectEqualStrings (large_source , large .text );
1459+ try std .testing .expectEqual (large_source .len , large .end );
1460+
1461+ const invalid = [_ ][]const u8 {
1462+ "1__0" ,
1463+ "1_" ,
1464+ "0x_1" ,
1465+ "0b1_" ,
1466+ "1_.0" ,
1467+ "1._0" ,
1468+ "1e_1" ,
1469+ "1e1_" ,
1470+ "1_n" ,
1471+ };
1472+ for (invalid ) | source | {
1473+ var arena = std .heap .ArenaAllocator .init (std .testing .allocator );
1474+ defer arena .deinit ();
1475+ var lexer = Lexer .init (arena .allocator (), source );
1476+ try std .testing .expectError (LexError .InvalidNumber , lexer .next ());
1477+ }
1478+ }
1479+
14001480test "lexer borrows ordinary identifiers and owns escaped decoding" {
14011481 const ordinary_source = "ordinary_π" ;
14021482 var no_memory : [0 ]u8 = .{};
0 commit comments