way better lex function

cmd/shsh/shsh.go

@@ -9,6 +9,6 @@ import (
 func main() {
     scanner := bufio.NewScanner(os.Stdin)
     for scanner.Scan() {
-        PrintSExpression(Lex(scanner.Text()))
+        shsh.PrintSExpression(shsh.Lex(scanner.Text()))
     }
 }

debug.go

@@ -6,9 +6,9 @@ import (
 )
 
 func FmtToken(arg *Token) string {
-    suffix := ""
+    suffix := "->"
-    if arg.next != nil {
+    if arg.next == nil {
-        suffix = "->"
+        suffix = ""
     }
 
     switch arg.tag {
@@ -37,6 +37,10 @@ func GetTagAsStr(tag token_t) string {
 }
 
 func PrintSExpression(arg *Token) {
+    if arg == nil {
+        return //TODO: Handle error here?
+    }
+
     var lists TokenStack;
     lists.Push(arg)
 
@@ -47,10 +51,20 @@ loop:
         return
     }
 
-    for iter := i; iter != nil; iter = i.next {
+    for iter := i; iter != nil; iter = iter.next {
+        if iter.tag == LIST {
+            lists.Push(iter._inner.(*Token))
+            constructor.WriteString(FmtToken(&Token{
+                next: iter.next,
+                tag: STRING,
+                position: iter.position,
+                _inner: "[LIST]"}))
+            continue
+        }
+
         constructor.WriteString(FmtToken(iter))
     }
 
-    print(constructor.String())
+    println(constructor.String())
     goto loop
 }

token.go

@@ -21,15 +21,44 @@ type Token struct {
 }
 
 func Lex(input string) *Token {
-    ret  := new(Token)
+    if len(input) == 0 {
+        return nil
+    }
+
+    var ret *Token
     iter := &ret
     delim := ' '
     var tok strings.Builder
-    iter_alloced := false
     is_list := false
     is_str  := false
     is_num  := true
 
+    tokenBuilder := func (pos int, tok string) {
+        *iter = new(Token)
+        (*iter).position = pos
+
+        if is_list {
+            (*iter)._inner = Lex(tok)
+            (*iter).tag = LIST
+            is_list = false
+
+        } else {
+            (*iter)._inner = tok
+            if is_str {
+                (*iter).tag = STRING
+                is_str = false
+
+            } else if is_num {
+                (*iter).tag = NUMBER
+
+            } else {
+                (*iter).tag = SYMBOL
+            }
+        }
+
+        iter = &(*iter).next
+    }
+
     for pos, char := range input {
         switch char {
         // TODO: User configurable delimiters
@@ -41,43 +70,23 @@ func Lex(input string) *Token {
             delim = ')'
 
         case delim:
-            *iter = new(Token)
+            delim = ' '
-            (*iter).position = pos
+            if tok.Len() == 0 {
-
+                continue
-            if is_list {
-                (*iter)._inner = Lex(tok.String())
-                is_list = false
-                (*iter)._inner = LIST
-
-            } else {
-                (*iter)._inner = tok.String()
-                if is_str {
-                    (*iter).tag = STRING
-                    is_str = false
-
-                } else if is_num {
-                    (*iter).tag = NUMBER
-
-                } else {
-                    (*iter).tag = SYMBOL
-                }
             }
 
-            iter_alloced = true
+            tokenBuilder(pos, tok.String())
-            delim = ' '
+            tok.Reset()
 
         default:
             is_num = is_num && unicode.IsDigit(char)
             tok.WriteRune(char)
         }
-
-        if iter_alloced {
-            iter = &((*iter).next)
-            iter_alloced = false
-            tok.Reset()
-        }
     }
 
-    // TODO: Throw parsing error here if there is leftover in tok
+    if tok.Len() > 0 {
+        tokenBuilder(len(input), tok.String())
+    }
+
     return ret
 }