30 lines
507 B
Zig
30 lines
507 B
Zig
|
const std = @import("std");
|
||
|
|
|
||
|
|
pub const Token = struct {
|
||
|
|
text: []const u8,
|
||
|
|
};
|
||
|
|
|
||
|
|
pub fn lex(
|
||
|
|
allocator: std.mem.Allocator,
|
||
|
|
source: []const u8,
|
||
|
|
) !std.ArrayList(Token) {
|
||
|
|
var tokens: std.ArrayList(Token) = .empty;
|
||
|
|
|
||
|
|
var iterator = std.mem.tokenizeAny(
|
||
|
|
u8,
|
||
|
|
source,
|
||
|
|
" \t\r\n",
|
||
|
|
);
|
||
|
|
|
||
|
|
while (iterator.next()) |word| {
|
||
|
|
try tokens.append(
|
||
|
|
allocator,
|
||
|
|
.{
|
||
|
|
.text = word,
|
||
|
|
},
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
return tokens;
|
||
|
|
}
|