Initial commit

src/shell/lexer.zig

@@ -0,0 +1,29 @@
+const std = @import("std");
+
+pub const Token = struct {
+    text: []const u8,
+};
+
+pub fn lex(
+    allocator: std.mem.Allocator,
+    source: []const u8,
+) !std.ArrayList(Token) {
+    var tokens: std.ArrayList(Token) = .empty;
+
+    var iterator = std.mem.tokenizeAny(
+        u8,
+        source,
+        " \t\r\n",
+    );
+
+    while (iterator.next()) |word| {
+        try tokens.append(
+            allocator,
+            .{
+                .text = word,
+            },
+        );
+    }
+
+    return tokens;
+}