Redid lex process

token.go

@@ -18,7 +18,6 @@
 package shs;
 
 import (
-    "strings"
     "unicode"
 )
 
@@ -37,17 +36,17 @@ type Token struct {
     _inner interface{}
 }
 
+const string_delims string = "\"'`"
+
 func Lex(input string) *Token {
     if len(input) == 0 {
         return nil
     }
 
     var ret *Token
-    var tok strings.Builder
     iter := &ret
-    delim := ' '
-    is_list := false
     is_str  := false
+    is_list := false
 
     tokenBuilder := func (pos int, tok string) {
         if len(tok) == 0 && !is_list && !is_str {
@@ -79,48 +78,100 @@ func Lex(input string) *Token {
         iter = &(*iter).next
     }
 
-    for pos, char := range input {
-        if char == delim {
-            if is_str && is_list {
-                // String just ended inside list
-                is_str = false
-                delim = ')'
-                tok.WriteRune(char)
-                continue
+    // returns -1 on unmatched string delim
+    matchStrEnd := func(start int, delim byte) int {
+        for i := start; i < len(input); i++ {
+            if input[i] == delim {
+                return i
             }
+        }
 
-            delim = ' '
-            tokenBuilder(pos, tok.String())
-            tok.Reset()
+        return -1
+    }
 
-        } else {
-            if strings.ContainsRune("\"'`", char) {
-                is_str = true
-                delim = char
-                if !is_list {
-                    continue
+    // returns -1 on unmatched string delim
+    // returns -2 on unmatched list delim
+    matchListEnd := func(start int) int {
+        depth := 0
+
+        for i := start; i < len(input); i++ {
+            switch input[i] {
+            case '"','\'','`':
+                i = matchStrEnd(i + 1, input[i])
+                if i == -1 {
+                    return -1
                 }
 
-            } else if char == '(' && !is_str {
-                is_list = true
-                delim = ')'
+            case '(':
+                depth++
+
+            case ')':
+                if depth == 0 {
+                    return i
+                } else {
+                    depth -= 1
+                }
+            }
+        }
+
+        return -2
+    }
+
+    needs_alloc := false
+    start_pos := 0
+    for i := 0; i < len(input); i++ {
+        switch input[i] {
+        case '(':
+            start_pos = i + 1
+            i = matchListEnd(start_pos)
+            is_list = true
+            needs_alloc = true
+
+        case '"','\'','`':
+            start_pos = i + 1
+            i = matchStrEnd(start_pos, input[i])
+            is_str = true
+            needs_alloc = true
+
+        case ' ':
+            if i == start_pos {
+                start_pos += 1
                 continue
             }
 
-            tok.WriteRune(char)
+            needs_alloc = true
+        }
+
+        if needs_alloc {
+            needs_alloc = false
+            if (i < 0) {
+                // TODO: Maybe not overload this.
+                start_pos = i
+                goto error
+            }
+
+            tokenBuilder(start_pos, input[start_pos:i])
+            start_pos = i+1
         }
     }
 
-    if tok.Len() > 0 {
-        if is_list || is_str {
-            // TODO: Throw hella lex error here
-            return ret
-        }
-
-        tokenBuilder(len(input), tok.String())
+    if start_pos < len(input) {
+        tokenBuilder(start_pos, input[start_pos:])
     }
-
     return ret
+
+error:
+    // TODO: Hook into error module
+    // TODO: Finalize and GC alloced tokens
+    if start_pos == -1 {
+        println("[-] Unmatched string delimiter in input. discarding.")
+    } else if start_pos == -2 {
+        println("[-] Unmatched list delimiter in input. discarding.")
+    } else {
+        println("[-] Unknown error in input. discarding.")
+    }
+
+    return nil
 }
 
 func tokenIsNumber(arg string) bool {