const std = @import("std");

pub const Token = struct {
    text: []const u8,
};

pub fn lex(
    allocator: std.mem.Allocator,
    source: []const u8,
) !std.ArrayList(Token) {
    var tokens: std.ArrayList(Token) = .empty;

    var iterator = std.mem.tokenizeAny(
        u8,
        source,
        " \t\r\n",
    );

    while (iterator.next()) |word| {
        try tokens.append(
            allocator,
            .{
                .text = word,
            },
        );
    }

    return tokens;
}