way better lex function

cmd/shsh/shsh.go

@@ -9,6 +9,6 @@ import (
 func main() {
     scanner := bufio.NewScanner(os.Stdin)
     for scanner.Scan() {
-        PrintSExpression(Lex(scanner.Text()))
+        shsh.PrintSExpression(shsh.Lex(scanner.Text()))
     }
 }

debug.go

@@ -6,9 +6,9 @@ import (
 )
 
 func FmtToken(arg *Token) string {
-    suffix := ""
-    if arg.next != nil {
-        suffix = "->"
+    suffix := "->"
+    if arg.next == nil {
+        suffix = ""
     }
 
     switch arg.tag {
@@ -37,6 +37,10 @@ func GetTagAsStr(tag token_t) string {
 }
 
 func PrintSExpression(arg *Token) {
+    if arg == nil {
+        return //TODO: Handle error here?
+    }
+
     var lists TokenStack;
     lists.Push(arg)
 
@@ -47,10 +51,20 @@ loop:
         return
     }
 
-    for iter := i; iter != nil; iter = i.next {
+    for iter := i; iter != nil; iter = iter.next {
+        if iter.tag == LIST {
+            lists.Push(iter._inner.(*Token))
+            constructor.WriteString(FmtToken(&Token{
+                next: iter.next,
+                tag: STRING,
+                position: iter.position,
+                _inner: "[LIST]"}))
+            continue
+        }
+
         constructor.WriteString(FmtToken(iter))
     }
 
-    print(constructor.String())
+    println(constructor.String())
     goto loop
 }

token.go

@@ -21,36 +21,29 @@ type Token struct {
 }
 
 func Lex(input string) *Token {
-    ret  := new(Token)
+    if len(input) == 0 {
+        return nil
+    }
+
+    var ret *Token
     iter := &ret
     delim := ' '
     var tok strings.Builder
-    iter_alloced := false
     is_list := false
     is_str  := false
     is_num  := true
 
-    for pos, char := range input {
-        switch char {
-        // TODO: User configurable delimiters
-        case '\'', '"', '`':
-            is_str = true
-            delim = char
-        case '(':
-            is_list = true
-            delim = ')'
-
-        case delim:
+    tokenBuilder := func (pos int, tok string) {
         *iter = new(Token)
         (*iter).position = pos
 
         if is_list {
-                (*iter)._inner = Lex(tok.String())
+            (*iter)._inner = Lex(tok)
+            (*iter).tag = LIST
             is_list = false
-                (*iter)._inner = LIST
 
         } else {
-                (*iter)._inner = tok.String()
+            (*iter)._inner = tok
             if is_str {
                 (*iter).tag = STRING
                 is_str = false
@@ -63,21 +56,37 @@ func Lex(input string) *Token {
             }
         }
 
-            iter_alloced = true
+        iter = &(*iter).next
+    }
+
+    for pos, char := range input {
+        switch char {
+        // TODO: User configurable delimiters
+        case '\'', '"', '`':
+            is_str = true
+            delim = char
+        case '(':
+            is_list = true
+            delim = ')'
+
+        case delim:
             delim = ' '
+            if tok.Len() == 0 {
+                continue
+            }
+
+            tokenBuilder(pos, tok.String())
+            tok.Reset()
 
         default:
             is_num = is_num && unicode.IsDigit(char)
             tok.WriteRune(char)
         }
-
-        if iter_alloced {
-            iter = &((*iter).next)
-            iter_alloced = false
-            tok.Reset()
-        }
     }
 
-    // TODO: Throw parsing error here if there is leftover in tok
+    if tok.Len() > 0 {
+        tokenBuilder(len(input), tok.String())
+    }
+
     return ret
 }