From 65cecb36474753140a636208dc2e74d362df73a8 Mon Sep 17 00:00:00 2001
From: Aidan Hahn <aidan@aidanis.online>
Date: Thu, 28 Nov 2019 22:43:30 -0800
Subject: [PATCH] way better lex function

---
 pkg/shsh/.parsers.go.swp => .token.go.swp | Bin 12288 -> 12288 bytes
 cmd/shsh/shsh.go                          |   2 +-
 pkg/shsh/debug.go => debug.go             |  24 ++++++--
 pkg/shsh/func_table.go => func_table.go   |   0
 pkg/shsh/stack.go => stack.go             |   0
 pkg/shsh/token.go => token.go             |  71 ++++++++++++----------
 pkg/shsh/var_table.go => var_table.go     |   0
 7 files changed, 60 insertions(+), 37 deletions(-)
 rename pkg/shsh/.parsers.go.swp => .token.go.swp (79%)
 rename pkg/shsh/debug.go => debug.go (62%)
 rename pkg/shsh/func_table.go => func_table.go (100%)
 rename pkg/shsh/stack.go => stack.go (100%)
 rename pkg/shsh/token.go => token.go (55%)
 rename pkg/shsh/var_table.go => var_table.go (100%)

diff --git a/pkg/shsh/.parsers.go.swp b/.token.go.swp
similarity index 79%
rename from pkg/shsh/.parsers.go.swp
rename to .token.go.swp
index 38eb4f46a8c2ea95a0a347f79ea7b2bac144dd4f..a658897437ea733d945fe5d869b3398c70254357 100644
GIT binary patch
literal 12288
zcmeI2UuYaf9LFaW6}7e6Kkw5Mb9-&POBAF%O{t+p0!dTyry8il&EDK)=-%wHJDXNY
z)t9QEh~TS&BKjhN_*4X6g_ah4t_UIsz9=Yyef0aA+1X?-Y4cp2h0pEo{ARv0^PS%e
zgzW10+38t+v^l|WJ;s>&?&nwX$J--&k1^IyQZ4%;rO@53e%KRn9ICt3q1fwAu+>Xi
zSsJx^Qa7V8jt6oeno-hkg|ZW>R<@R{wRCb`sAezONz})^1Kxq19msX}siS+?Gfz*9
zQ{^XWhxq<|OFOsoHM|4f0q=l!z&qd_@D6wfyaV2W|BM5AFv8xzY<3khI#=9BhVI3y
zzj+6|1Kt7efOo(<;2rP|cn7=#-U07`cfdPv4>}Nb82jcy#*ROP!Q=n`?(hG<_A>S-
z_zm0wH^6mp4O|6ZgAc(vSO(97HaG^J1P8(W;O|En`w9FAZh>#Xr{I0?4tN`A5P^B{
zEI0&qgWG!;`v-gvz69@qcfmS1117*hZ~)u~e%XyNg3rNQ;0ox17r+T{1Uvw4Kf>4_
z;CJu?xDKv?FTe+26`TS`z&^mhuMabJ6Wjn-!6)EN&;!fhWiSQ~fc;<>xCu|c0-u48
z!AIZ<I1A2zIdB3P@9)9f@k90wcn7=#-U08x|I>lXjN?j$&QnD<X5MWnMa)NxQ@M;g
zA`N1p0;vYMZj4c&xu8?2dO@RM8p*CHoI#}qKhDSbrD3rR*+RPcQYy8W&y@(GwJ^0h
z6r)RKD=P2vN%t^1%5x>7q$8Ts*-42{FNYd+L^sT1-L~CY5i^74yvT$Os`8bY4U%kl
zO3Vq+y*hF#i4vuy$_1krcU3HyoHQP|7~MM}mNcU}uQOL3g_*E{p@El`8yl6W5qybd
zrXi!#P5m34dQfkd%2e5jkiC@vR5G1zFLJJ4ALI2J{k*zKuhrs<b0_E8d?^zt8PJuz
zJPlXlV*FUjG%GvFE=V1%aq1YEfT8eil5%6TDQi#iG*msoO-5}^7E;6TMuWrj)MzSk
zvB*x+sxC<NjpDAA4to$;Se~7ln_(1i*JUnZ7+>V5E2Vq2>FK4}sZ;Z8bA_TQ7Q0B%
zZA40YyCJ%%G$*NQ9}A1~)2CmoRI=i<Le@KKM3-y0B9$`!AdQ)c;Ai_eZRE?=u{b$v
z)6De3Vkt(VE?o9DBQ6H$abqXhahiPG<brF5)s|W&3QK^;QrfI2%9OYeEYerfR&Vo;
z<VcTht|A_wAyPz}?a5&XPMn5Yw!(j^T<(R!c_UDI?>r#YvTz-li9QR_tXUj$W|(&0
zpt{?tCLQrUN{1w@;nW(>{IHq9@KINss7l7BGHlv7Jb7_4nE8>Tb_ck*SwWhjP3yvZ
zcp%bl7>P@^xbzkQ(+91gv2ew{(}r=Q)2|H#Hv^^F=8-0O>$=?=G^3(17DgsIG@3U1
z?+~<Udxs$95{0(N;@rUD6)-a-*CoxN7*DM@<ZFx?*Q_g9jmiE1U-P3O8-&q$e6Y{y
IEBoizzm(P&cK`qY

literal 12288
zcmeI2PiqrF7>6gt-u#FC0<RUZt=Mf9r4)(?^-$1kY14`a36o^98L~UG?9R4Uson*}
zyWmaQFW}9yXF<_ZkA4KfLl6F@*|fFR+H2(<c-T$mH#6@%w~)6nu{?L5rp6})#(5zI
zKYn_2WpKH&JtRb8Gp7@k`KhBuA|n;Y(j1lXbI<x4237UgHDg<}f|^RwAn)XzAniuw
zi_CJB<>Qh4*M>aL2H3#y3>2<?ZR*U@jq8&We!->Ti!^j$>3C~7#Rk{_8(;%$fDNz#
zHoykhz<*)Dr4{iUHyJ2zbGdw04m`_`{9prYfDNz#Hoykh02^QfY=8~00XDz}PM`tV
z65_{cA>N%q@%aD0|NDRIoDlE9CfERP!E5jeyaX?R0U@{ww$BRj6MP3--~)IAo`8qo
z0aykzU<%v<H^Cq{1qMI`d_x^y!58ouY=RBo>+8RtK+Rk-8(;%$fDNz#Hoykh02^Qf
zY~XMM>j#Gr?ZPxEcbPU(UFUV1s!TeYjmRu2zaFKAwJ|*vV!e#VI(K_vZOu-4tY*8)
z{FyE{Z>d-(+Mxlvz9M7Y`pXG~(JN}+w`>2lGfEg0J`m2|UAR-53#;`Sr9M`7GNKvM
z*2#T=Z~^m21gf)s{_w!Pxy9Pf>|ueXHMye^;nq^MZ@D&3&{zQWDx4-s-WOy_biJF1
zR<-MpG}KqU7Y`SZHl&Qp)<_$y%-0ep8?oBWRA>E$J*8wbGDXrrBRx*^1@9ha+oWQZ
zV5eLJV|{JIy><Dp&`F9OeB3~!ve`wOJ^5C75d^f@u|?dXRYe`SqR1LWZJZ<Hm~4?z
e)3(%YpR0wl;WW99HeHepTR3>=?PO%4a`6j@<ODwe

diff --git a/cmd/shsh/shsh.go b/cmd/shsh/shsh.go
index ef87a6c..ddc4e0f 100644
--- a/cmd/shsh/shsh.go
+++ b/cmd/shsh/shsh.go
@@ -9,6 +9,6 @@ import (
 func main() {
     scanner := bufio.NewScanner(os.Stdin)
     for scanner.Scan() {
-        PrintSExpression(Lex(scanner.Text()))
+        shsh.PrintSExpression(shsh.Lex(scanner.Text()))
     }
 }
diff --git a/pkg/shsh/debug.go b/debug.go
similarity index 62%
rename from pkg/shsh/debug.go
rename to debug.go
index 290720e..d6bc4b8 100644
--- a/pkg/shsh/debug.go
+++ b/debug.go
@@ -6,9 +6,9 @@ import (
 )
 
 func FmtToken(arg *Token) string {
-    suffix := ""
-    if arg.next != nil {
-        suffix = "->"
+    suffix := "->"
+    if arg.next == nil {
+        suffix = ""
     }
 
     switch arg.tag {
@@ -37,6 +37,10 @@ func GetTagAsStr(tag token_t) string {
 }
 
 func PrintSExpression(arg *Token) {
+    if arg == nil {
+        return //TODO: Handle error here?
+    }
+
     var lists TokenStack;
     lists.Push(arg)
 
@@ -47,10 +51,20 @@ loop:
         return
     }
 
-    for iter := i; iter != nil; iter = i.next {
+    for iter := i; iter != nil; iter = iter.next {
+        if iter.tag == LIST {
+            lists.Push(iter._inner.(*Token))
+            constructor.WriteString(FmtToken(&Token{
+                next: iter.next,
+                tag: STRING,
+                position: iter.position,
+                _inner: "[LIST]"}))
+            continue
+        }
+
         constructor.WriteString(FmtToken(iter))
     }
 
-    print(constructor.String())
+    println(constructor.String())
     goto loop
 }
diff --git a/pkg/shsh/func_table.go b/func_table.go
similarity index 100%
rename from pkg/shsh/func_table.go
rename to func_table.go
diff --git a/pkg/shsh/stack.go b/stack.go
similarity index 100%
rename from pkg/shsh/stack.go
rename to stack.go
diff --git a/pkg/shsh/token.go b/token.go
similarity index 55%
rename from pkg/shsh/token.go
rename to token.go
index 24805e1..4a954a2 100644
--- a/pkg/shsh/token.go
+++ b/token.go
@@ -21,15 +21,44 @@ type Token struct {
 }
 
 func Lex(input string) *Token {
-    ret  := new(Token)
+    if len(input) == 0 {
+        return nil
+    }
+
+    var ret *Token
     iter := &ret
     delim := ' '
     var tok strings.Builder
-    iter_alloced := false
     is_list := false
     is_str  := false
     is_num  := true
 
+    tokenBuilder := func (pos int, tok string) {
+        *iter = new(Token)
+        (*iter).position = pos
+
+        if is_list {
+            (*iter)._inner = Lex(tok)
+            (*iter).tag = LIST
+            is_list = false
+
+        } else {
+            (*iter)._inner = tok
+            if is_str {
+                (*iter).tag = STRING
+                is_str = false
+
+            } else if is_num {
+                (*iter).tag = NUMBER
+
+            } else {
+                (*iter).tag = SYMBOL
+            }
+        }
+
+        iter = &(*iter).next
+    }
+
     for pos, char := range input {
         switch char {
         // TODO: User configurable delimiters
@@ -41,43 +70,23 @@ func Lex(input string) *Token {
             delim = ')'
 
         case delim:
-            *iter = new(Token)
-            (*iter).position = pos
-
-            if is_list {
-                (*iter)._inner = Lex(tok.String())
-                is_list = false
-                (*iter)._inner = LIST
-
-            } else {
-                (*iter)._inner = tok.String()
-                if is_str {
-                    (*iter).tag = STRING
-                    is_str = false
-
-                } else if is_num {
-                    (*iter).tag = NUMBER
-
-                } else {
-                    (*iter).tag = SYMBOL
-                }
+            delim = ' '
+            if tok.Len() == 0 {
+                continue
             }
 
-            iter_alloced = true
-            delim = ' '
+            tokenBuilder(pos, tok.String())
+            tok.Reset()
 
         default:
             is_num = is_num && unicode.IsDigit(char)
             tok.WriteRune(char)
         }
-
-        if iter_alloced {
-            iter = &((*iter).next)
-            iter_alloced = false
-            tok.Reset()
-        }
     }
 
-    // TODO: Throw parsing error here if there is leftover in tok
+    if tok.Len() > 0 {
+        tokenBuilder(len(input), tok.String())
+    }
+
     return ret
 }
diff --git a/pkg/shsh/var_table.go b/var_table.go
similarity index 100%
rename from pkg/shsh/var_table.go
rename to var_table.go