way better lex function

token.go

@@ -0,0 +1,92 @@
+package shsh;
+
+import (
+    "strings"
+    "unicode"
+)
+
+type token_t int
+const (
+    LIST      token_t = iota
+    STRING    token_t = iota
+    NUMBER    token_t = iota
+    SYMBOL    token_t = iota
+)
+
+type Token struct {
+    next *Token
+    tag token_t
+    position int
+    _inner interface{}
+}
+
+func Lex(input string) *Token {
+    if len(input) == 0 {
+        return nil
+    }
+
+    var ret *Token
+    iter := &ret
+    delim := ' '
+    var tok strings.Builder
+    is_list := false
+    is_str  := false
+    is_num  := true
+
+    tokenBuilder := func (pos int, tok string) {
+        *iter = new(Token)
+        (*iter).position = pos
+
+        if is_list {
+            (*iter)._inner = Lex(tok)
+            (*iter).tag = LIST
+            is_list = false
+
+        } else {
+            (*iter)._inner = tok
+            if is_str {
+                (*iter).tag = STRING
+                is_str = false
+
+            } else if is_num {
+                (*iter).tag = NUMBER
+
+            } else {
+                (*iter).tag = SYMBOL
+            }
+        }
+
+        iter = &(*iter).next
+    }
+
+    for pos, char := range input {
+        switch char {
+        // TODO: User configurable delimiters
+        case '\'', '"', '`':
+            is_str = true
+            delim = char
+        case '(':
+            is_list = true
+            delim = ')'
+
+        case delim:
+            delim = ' '
+            if tok.Len() == 0 {
+                continue
+            }
+
+            tokenBuilder(pos, tok.String())
+            tok.Reset()
+
+        default:
+            is_num = is_num && unicode.IsDigit(char)
+            tok.WriteRune(char)
+        }
+    }
+
+    if tok.Len() > 0 {
+        tokenBuilder(len(input), tok.String())
+    }
+
+    return ret
+}