Lexer is significantly better tested and more advanced

token.go

@@ -31,9 +31,12 @@ func Lex(input string) *Token {
     var tok strings.Builder
     is_list := false
     is_str  := false
-    is_num  := true
 
     tokenBuilder := func (pos int, tok string) {
+        if len(tok) == 0 && !is_list && !is_str {
+            return
+        }
+
         *iter = new(Token)
         (*iter).position = pos
 
@@ -48,7 +51,7 @@ func Lex(input string) *Token {
                 (*iter).tag = STRING
                 is_str = false
 
-            } else if is_num {
+            } else if tokenIsNumber(tok) {
                 (*iter).tag = NUMBER
 
             } else {
@@ -60,33 +63,61 @@ func Lex(input string) *Token {
     }
 
     for pos, char := range input {
-        switch char {
+        if char == delim {
-        // TODO: User configurable delimiters
+            if is_str && is_list {
-        case '\'', '"', '`':
+                // String just ended inside list
-            is_str = true
+                is_str = false
-            delim = char
+                delim = ')'
-        case '(':
+                tok.WriteRune(char)
-            is_list = true
-            delim = ')'
-
-        case delim:
-            delim = ' '
-            if tok.Len() == 0 {
                 continue
             }
 
+            delim = ' '
             tokenBuilder(pos, tok.String())
             tok.Reset()
 
-        default:
+        } else {
-            is_num = is_num && unicode.IsDigit(char)
+            if strings.ContainsRune("\"'`", char) {
+                is_str = true
+                delim = char
+                if !is_list {
+                    continue
+                }
+
+            } else if char == '(' && !is_str {
+                is_list = true
+                delim = ')'
+                continue
+            }
+
             tok.WriteRune(char)
         }
     }
 
     if tok.Len() > 0 {
+        if is_list || is_str {
+            // TODO: Throw hella lex error here
+            return ret
+        }
+
         tokenBuilder(len(input), tok.String())
     }
 
     return ret
 }
+
+func tokenIsNumber(arg string) bool {
+    dotCount := 0
+
+    for _, char := range arg {
+        if !unicode.IsDigit(char) {
+            if char == '.' && dotCount == 0 {
+                dotCount++
+            } else {
+                return false
+            }
+        }
+    }
+
+    return true
+}