way better lex function

pkg/shsh/token.go

@@ -1,83 +0,0 @@
-package shsh;
-
-import (
-    "strings"
-    "unicode"
-)
-
-type token_t int
-const (
-    LIST      token_t = iota
-    STRING    token_t = iota
-    NUMBER    token_t = iota
-    SYMBOL    token_t = iota
-)
-
-type Token struct {
-    next *Token
-    tag token_t
-    position int
-    _inner interface{}
-}
-
-func Lex(input string) *Token {
-    ret  := new(Token)
-    iter := &ret
-    delim := ' '
-    var tok strings.Builder
-    iter_alloced := false
-    is_list := false
-    is_str  := false
-    is_num  := true
-
-    for pos, char := range input {
-        switch char {
-        // TODO: User configurable delimiters
-        case '\'', '"', '`':
-            is_str = true
-            delim = char
-        case '(':
-            is_list = true
-            delim = ')'
-
-        case delim:
-            *iter = new(Token)
-            (*iter).position = pos
-
-            if is_list {
-                (*iter)._inner = Lex(tok.String())
-                is_list = false
-                (*iter)._inner = LIST
-
-            } else {
-                (*iter)._inner = tok.String()
-                if is_str {
-                    (*iter).tag = STRING
-                    is_str = false
-
-                } else if is_num {
-                    (*iter).tag = NUMBER
-
-                } else {
-                    (*iter).tag = SYMBOL
-                }
-            }
-
-            iter_alloced = true
-            delim = ' '
-
-        default:
-            is_num = is_num && unicode.IsDigit(char)
-            tok.WriteRune(char)
-        }
-
-        if iter_alloced {
-            iter = &((*iter).next)
-            iter_alloced = false
-            tok.Reset()
-        }
-    }
-
-    // TODO: Throw parsing error here if there is leftover in tok
-    return ret
-}