Merge branch 'better-eval' into 'dev'

Better eval

See merge request whom/shs!3

ast/eval.go

@@ -19,86 +19,103 @@ package ast
 
 import "gitlab.com/whom/shs/log"
 
-var CallExecutablesFromUndefFuncCalls = false
-var CallExecutableToken = "l"
+/* determines whether or not to execute a system call
+ * when a function cannot be found in the functable
+ * (use case: shell)
+ * ExecFunc determines the name of the system call function to fetch
+ */
+var ExecWhenFuncUndef = false
+var ExecFunc = "l"
 
-func (t *Token) Eval(funcs FuncTable, vars VarTable) (*Token, bool) {
-    if t == nil {
-        return nil, false
+/* Runs through an AST of tokens
+ * Evaluates the Tokens to determine simplest form
+ * Returns simplest form
+ *
+ * canFunc determines whether a symbol could be a function to call
+ * (true when first elem of a list)
+ */
+func (in *Token) Eval(funcs FuncTable, vars VarTable, cnvtUndefVars bool) *Token {
+    if in == nil {
+        return nil
     }
 
-    var reduce func(*Token) *Token
-    reduce = func(t_ *Token) *Token {
-        var unwrap bool
+    var res *Token
 
-        if t_.Next != nil {
-            t_.Next = reduce(t_.Next)
-        }
+    switch in.Tag {
+    case BOOL, NUMBER, STRING:
+        res = in
 
-        switch (t_.Tag) {
-        case SYMBOL:
-            maybeToken := GetVar(t_.Inner.(string), vars)
-            if maybeToken != nil {
-                tok, _ := maybeToken.Eval(funcs, vars)
-                tok.Next = t_.Next
-                return tok
-            }
+    case SYMBOL:
+        res = GetVar(in.Value(), vars)
+        if res == nil {
+            res = in
 
-        case LIST:
-            t_.Inner, unwrap = t_.Inner.(*Token).Eval(funcs, vars)
-            if unwrap {
-                next := t_.Next
-                t_ = t_.Inner.(*Token)
-                if t_ == nil {
-                    log.Log(log.DEBUG, "nil Inner on list unwrap", "eval")
-                    return nil
+            if GetFunction(in.Value(), funcs) == nil {
+                if cnvtUndefVars {
+                    in.Tag = STRING
+                    res = in
+                    break
                 }
 
-                i := &t_
-                for (*i).Next != nil {
-                    i = &((*i).Next)
-                }
-
-                (*i).Next = next
-            }
-        }
-
-        return t_
-    }
-
-    ret := reduce(t)
-    if ret == nil {
-        log.Log(log.INFO, "reduce returned nil", "eval")
-        return nil, false
-    }
-
-    //if symbol in front of a list, could be a function call
-    if ret.Tag == SYMBOL {
-        f := GetFunction(ret.Inner.(string), funcs)
-        if f == nil {
-            if CallExecutablesFromUndefFuncCalls {
-                f = GetFunction(CallExecutableToken, funcs)
-                if f == nil {
-                    log.Log(log.DEBUG, "Symbol " + ret.Inner.(string) + 
-                        " has no definition in function table. Additionally " +
-                        "the configured LoadExecutableToken is also not defined",
-                        "eval")
-                    return ret, false
-                }
-
-                // see the use of CallFunction below
-                ret = &Token{Next: ret}
-
-            } else {
-               log.Log(log.DEBUG,
-                    "could not find definition for symbol " + ret.Inner.(string),
+                log.Log(log.ERR, 
+                    "undefined symbol: "+in.Value(),
                     "eval")
-                return ret, false
+                return nil
+            }
+        }
+        res.Next = in.Next
+
+
+    case LIST:
+        inner := in.Expand()
+        if inner == nil {
+            res = in
+            break
+        }
+
+        if inner.Tag != SYMBOL {
+            in.Direct(inner.Eval(funcs, vars, cnvtUndefVars))
+            res = in
+            break
+        }
+
+        makeHead := false
+        funct := GetFunction(inner.Value(), funcs)
+        if funct == nil {
+            if ExecWhenFuncUndef {
+                funct = GetFunction(ExecFunc, funcs)
+                makeHead = true
             }
         }
 
-        return (*f).CallFunction(ret.Next, vars, funcs), true
+        if funct != nil {
+            if makeHead {
+                inner = &Token{Next: inner}
+            }
+            res = funct.CallFunction(inner.Next, vars, funcs).Eval(funcs, vars, false)
+            if res == nil {
+                // function failed. logging is its responsibility
+                return nil
+            }
+
+            res.Append(in.Next)
+
+        } else {
+            log.Log(log.ERR,
+                "undefined function "+inner.Value()+" called",
+                "eval")
+            return nil
+        }
+
+    default:
+        log.Log(log.ERR,
+            "Eval hit unknown token type!",
+            "eval")
+        return nil
     }
 
-    return ret, false
+    if res.Next != nil {
+        res.Next = res.Next.Eval(funcs, vars, cnvtUndefVars)
+    }
+    return res
 }

ast/lex.go

@@ -0,0 +1,185 @@
+/*   SHS: Syntactically Homogeneous Shell
+ *   Copyright (C) 2019  Aidan Hahn
+ *
+ *   This program is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package ast
+
+import (
+    "gitlab.com/whom/shs/log"
+    "unicode"
+)
+
+const string_delims string = "\"'`"
+
+func Lex(input string) *Token {
+    if len(input) == 0 {
+        return nil
+    }
+
+    var ret *Token
+    iter := &ret
+    is_str  := false
+    is_list := false
+
+    tokenBuilder := func (pos int, tok string) {
+        if len(tok) == 0 && !is_list && !is_str {
+            return
+        }
+
+        *iter = new(Token)
+        (*iter).Position = pos
+
+        if is_list {
+            (*iter).inner = Lex(tok)
+            (*iter).Tag = LIST
+            is_list = false
+
+        } else {
+            (*iter).inner = tok
+            if is_str {
+                (*iter).Tag = STRING
+                is_str = false
+
+            } else if StrIsNumber(tok) {
+                (*iter).Tag = NUMBER
+
+            } else {
+                (*iter).Tag = SYMBOL
+            }
+        }
+
+        iter = &(*iter).Next
+    }
+
+    // returns -1 on unmatched string delim
+    matchStrEnd := func(start int, delim byte) int {
+        for i := start; i < len(input); i++ {
+            if input[i] == delim {
+                return i
+            }
+        }
+
+        return -1
+    }
+
+    // returns -1 on unmatched string delim
+    // returns -2 on unmatched list delim
+    matchListEnd := func(start int) int {
+        depth := 0
+
+        for i := start; i < len(input); i++ {
+            switch input[i] {
+            case '"','\'','`':
+                i = matchStrEnd(i + 1, input[i])
+                if i == -1 {
+                    return -1
+                }
+
+            case '(':
+                depth++
+
+            case ')':
+                if depth == 0 {
+                    return i
+                } else {
+                    depth -= 1
+                }
+            }
+        }
+
+        return -2
+    }
+
+    needs_alloc := false
+    start_pos := 0
+    for i := 0; i < len(input); i++ {
+        switch input[i] {
+        case '(':
+            start_pos = i + 1
+            i = matchListEnd(start_pos)
+            is_list = true
+            needs_alloc = true
+
+        case '"','\'','`':
+            start_pos = i + 1
+            i = matchStrEnd(start_pos, input[i])
+            is_str = true
+            needs_alloc = true
+
+        case ' ':
+            if i == start_pos {
+                start_pos += 1
+                continue
+            }
+
+            needs_alloc = true
+        }
+
+        if needs_alloc {
+            needs_alloc = false
+            if (i < 0) {
+                // TODO: Maybe not overload this.
+                start_pos = i
+                goto error
+            }
+
+            tokenBuilder(start_pos, input[start_pos:i])
+            start_pos = i+1
+        }
+    }
+
+    if start_pos < len(input) {
+        tokenBuilder(start_pos, input[start_pos:])
+    }
+
+    return ret
+
+error:
+    // TODO: Hook into error module
+    // TODO: Finalize and GC alloced tokens
+    if start_pos == -1 {
+        log.Log(log.ERR,
+                "Unmatched string delimiter in input. discarding.",
+                "lex")
+    } else if start_pos == -2 {
+        log.Log(log.ERR,
+                "Unmatched list delimiter in input. discarding.",
+                "lex")
+    } else {
+        log.Log(log.ERR,
+                "Unknown error in input. discarding.",
+                "lex")
+    }
+
+    return nil
+}
+
+func StrIsNumber(arg string) bool {
+    dotCount := 0
+
+    for _, char := range arg {
+        if !unicode.IsDigit(char) {
+            if char == '.' && dotCount == 0 {
+                dotCount++
+            } else {
+                return false
+            }
+        }
+    }
+
+    return true
+}
+

ast/print.go

@@ -19,39 +19,8 @@ package ast
 
 import (
     "strings"
-    "fmt"
 )
 
-/* Print function which is better suited for repl.
- * This one prints the SEXPRs as one would write them.
- */
-func (t *Token) String() string {
-    switch t.Tag {
-    case STRING:
-        return "\"" + t.Inner.(string) + "\""
-
-    case NUMBER, BOOL:
-        return t.Inner.(string)
-
-    case LIST:
-        repr := "("
-        if t.Inner.(*Token) == nil {
-            return repr + ")"
-        }
-
-        for i := t.Inner.(*Token); i != nil; i = i.Next {
-            repr = repr + i.String() + " "
-        }
-        // remove trailing space
-        return repr[:len(repr)-1] + ")"
-
-    case SYMBOL:
-        return "<" + t.Inner.(string) + ">"
-    }
-
-    return "[UNKNOWN CELL TYPE]"
-}
-
 /* Print function which breaks each embedded list out on individual lines.
  * Used in the print_ast debug tool. not too useful for repl applications.
  * Very useful for debugging syntax though.
@@ -74,42 +43,13 @@ loop:
 
     for iter := i; iter != nil; iter = iter.Next {
         if iter.Tag == LIST {
-            lists.Push(iter.Inner.(*Token))
+            lists.Push(iter.Expand())
         }
 
-        constructor.WriteString(FmtToken(iter))
+        constructor.WriteString(iter.FmtToken())
     }
 
     println(constructor.String())
     goto loop
 }
 
-func FmtToken(arg *Token) string {
-    suffix := "->"
-    if arg.Next == nil {
-        suffix = ""
-    }
-
-    switch arg.Tag {
-    case LIST:
-        return fmt.Sprintf("(%s, [List])%s", "LIST", suffix)
-
-    default:
-        return fmt.Sprintf("(%s, %s)%s", GetTagAsStr(arg.Tag), arg.Inner, suffix)
-    }
-}
-
-func GetTagAsStr(tag Token_t) string {
-    switch tag {
-    case LIST:
-        return "LIST"
-    case STRING:
-        return "STRING"
-    case NUMBER:
-        return "NUMBER"
-    case SYMBOL:
-        return "SYMBOL"
-    }
-    return "UNKNOWN"
-}
-

ast/token.go

@@ -17,10 +17,7 @@
 
 package ast
 
-import (
-    "gitlab.com/whom/shs/log"
-    "unicode"
-)
+import "fmt"
 
 type Token_t int
 const (
@@ -35,166 +32,127 @@ type Token struct {
     Next *Token
     Tag Token_t
     Position int
-    Inner interface{}
+    inner interface{}
 }
 
-const string_delims string = "\"'`"
+/* Appends another token to the end of this token list
+ */
+func (t *Token) Append(arg *Token) {
+    if t.Next != nil {
+        t.Next.Append(arg)
+    } else {
+        t.Next = arg
+    }
+}
 
-func Lex(input string) *Token {
-    if len(input) == 0 {
+/* Print function which is better suited for repl.
+ * This one prints the SEXPRs as one would write them.
+ * Does not evaluate tokens.
+ */
+func (t *Token) String() string {
+    switch t.Tag {
+    case STRING:
+        return "\"" + t.inner.(string) + "\""
+
+    case NUMBER, BOOL:
+        return t.inner.(string)
+
+    case LIST:
+        repr := "("
+        if t.inner.(*Token) == nil {
+            return repr + ")"
+        }
+
+        for i := t.inner.(*Token); i != nil; i = i.Next {
+            repr = repr + i.String() + " "
+        }
+        // remove trailing space
+        return repr[:len(repr)-1] + ")"
+
+    case SYMBOL:
+        return "<" + t.inner.(string) + ">"
+    }
+
+    return "[UNKNOWN CELL TYPE]"
+}
+
+/* Returns a list held by a token
+ * returns nil if token holds no list
+ */
+func (t *Token) Expand() *Token {
+    if t.Tag != LIST {
         return nil
     }
 
-    var ret *Token
-    iter := &ret
-    is_str  := false
-    is_list := false
-
-    tokenBuilder := func (pos int, tok string) {
-        if len(tok) == 0 && !is_list && !is_str {
-            return
-        }
-
-        *iter = new(Token)
-        (*iter).Position = pos
-
-        if is_list {
-            (*iter).Inner = Lex(tok)
-            (*iter).Tag = LIST
-            is_list = false
-
-        } else {
-            (*iter).Inner = tok
-            if is_str {
-                (*iter).Tag = STRING
-                is_str = false
-
-            } else if StrIsNumber(tok) {
-                (*iter).Tag = NUMBER
-
-            } else {
-                (*iter).Tag = SYMBOL
-            }
-        }
-
-        iter = &(*iter).Next
-    }
-
-    // returns -1 on unmatched string delim
-    matchStrEnd := func(start int, delim byte) int {
-        for i := start; i < len(input); i++ {
-            if input[i] == delim {
-                return i
-            }
-        }
-
-        return -1
-    }
-
-    // returns -1 on unmatched string delim
-    // returns -2 on unmatched list delim
-    matchListEnd := func(start int) int {
-        depth := 0
-
-        for i := start; i < len(input); i++ {
-            switch input[i] {
-            case '"','\'','`':
-                i = matchStrEnd(i + 1, input[i])
-                if i == -1 {
-                    return -1
-                }
-
-            case '(':
-                depth++
-
-            case ')':
-                if depth == 0 {
-                    return i
-                } else {
-                    depth -= 1
-                }
-            }
-        }
-
-        return -2
-    }
-
-    needs_alloc := false
-    start_pos := 0
-    for i := 0; i < len(input); i++ {
-        switch input[i] {
-        case '(':
-            start_pos = i + 1
-            i = matchListEnd(start_pos)
-            is_list = true
-            needs_alloc = true
-
-        case '"','\'','`':
-            start_pos = i + 1
-            i = matchStrEnd(start_pos, input[i])
-            is_str = true
-            needs_alloc = true
-
-        case ' ':
-            if i == start_pos {
-                start_pos += 1
-                continue
-            }
-
-            needs_alloc = true
-        }
-
-        if needs_alloc {
-            needs_alloc = false
-            if (i < 0) {
-                // TODO: Maybe not overload this.
-                start_pos = i
-                goto error
-            }
-
-            tokenBuilder(start_pos, input[start_pos:i])
-            start_pos = i+1
-        }
-    }
-
-    if start_pos < len(input) {
-        tokenBuilder(start_pos, input[start_pos:])
-    }
-    return ret
-
-error:
-    // TODO: Hook into error module
-    // TODO: Finalize and GC alloced tokens
-    if start_pos == -1 {
-        log.Log(log.ERR,
-                "Unmatched string delimiter in input. discarding.",
-                "lex")
-    } else if start_pos == -2 {
-        log.Log(log.ERR,
-                "Unmatched list delimiter in input. discarding.",
-                "lex")
-    } else {
-        log.Log(log.ERR,
-                "Unknown error in input. discarding.",
-                "lex")
-    }
-
-    return nil
+    return t.inner.(*Token)
 }
 
-func StrIsNumber(arg string) bool {
-    dotCount := 0
-
-    for _, char := range arg {
-        if !unicode.IsDigit(char) {
-            if char == '.' && dotCount == 0 {
-                dotCount++
-            } else {
-                return false
-            }
-        }
+/* Sets inner to a Token value
+ * returns false if parent token is not a list
+ * otherwise returns true
+ */
+func (t *Token) Direct(head *Token) bool {
+    if t.Tag != LIST {
+        return false
     }
 
+    t.inner = head
     return true
 }
 
+/* If token holds an atomic value
+ * (not a symbol or list)
+ * will return its value as a string
+ * else returns ""
+ */
+func (t *Token) Value() string {
+    if t.Tag == LIST {
+        return ""
+    }
+
+    return t.inner.(string)
+}
+
+/* returns an ascii representation of a token
+ */
+func (t *Token) FmtToken() string {
+    suffix := "->"
+    if t.Next == nil {
+        suffix = ""
+    }
+
+    switch t.Tag {
+    case LIST:
+        return fmt.Sprintf("(%s, [List])%s", "LIST", suffix)
+
+    default:
+        return fmt.Sprintf("(%s, %s)%s", GetTagAsStr(t.Tag), t.inner.(string), suffix)
+    }
+}
+
+/* Sets the string value for a non-list token
+ */
+func (t *Token) Set(arg string) bool {
+    if t.Tag == LIST {
+        return false
+    }
+
+    t.inner = arg
+    return true
+}
+
+/* Returns a tag in text
+ */
+func GetTagAsStr(tag Token_t) string {
+    switch tag {
+    case LIST:
+        return "LIST"
+    case STRING:
+        return "STRING"
+    case NUMBER:
+        return "NUMBER"
+    case SYMBOL:
+        return "SYMBOL"
+    }
+    return "UNKNOWN"
+}

ast/var_table.go

@@ -40,7 +40,7 @@ func GetVar(arg string, vt VarTable) *Token {
 
         e := os.Getenv(arg)
         if e != "" {
-            t := &Token{Inner: e}
+            t := &Token{inner: e}
             if StrIsNumber(e) {
                 t.Tag = NUMBER
             } else {
@@ -56,11 +56,13 @@ func GetVar(arg string, vt VarTable) *Token {
 }
 
 // TODO: this could be much more optimal
+// TODO: Make sure variables are evaluated before being set
+// probably a stdlib thing
 func SetVar(variable string, value *Token, vt VarTable) {
     (*vt)[variable] = value
     if SyncTablesWithOSEnviron && 
        (value.Tag == NUMBER || value.Tag == STRING) {
-        token := value.Inner.(string)
+        token := value.Value()
         if value.Tag == NUMBER {
             // make sure its an int
             a, err := strconv.ParseFloat(token, 64)
@@ -91,7 +93,7 @@ func GetVarFromTables(arg string, library []VarTable) *Token {
 func InitVarTable(table VarTable) {
     for _, val := range os.Environ() {
         variable := strings.Split(val, "=")
-        t := &Token{Inner: variable[1]}
+        t := &Token{inner: variable[1]}
         if StrIsNumber(variable[1]) {
             t.Tag = NUMBER
         } else {

cmd/print_ast.go

@@ -20,10 +20,9 @@ package main
 import (
     "strings"
     "os"
-    "gitlab.com/whom/shs/log"
     "gitlab.com/whom/shs/ast"
 )
 
 func main() {
-    log.PrintSExprsIndividually(ast.Lex(strings.Join(os.Args[1:], " ")))
+    ast.PrintSExprsIndividually(ast.Lex(strings.Join(os.Args[1:], " ")))
 }

cmd/shs_repl.go

@@ -46,8 +46,6 @@ func setLogLvl() {
 }
 
 func main() {
-    ast.CallExecutablesFromUndefFuncCalls = true
-
     debug  := os.Getenv("SH_DEBUG_MODE")
     hist   := os.Getenv("SH_HIST_FILE")
     prompt := os.Getenv("SHS_SH_PROMPT")
@@ -57,8 +55,10 @@ func main() {
 
     funcs = stdlib.GenFuncTable()
     vars = &map[string]*ast.Token{}
+
     ast.InitVarTable(vars)
     ast.SyncTablesWithOSEnviron = true
+    ast.ExecWhenFuncUndef = true
 
     var err error
 
@@ -96,12 +96,8 @@ func main() {
             ast.PrintSExprsIndividually(userInput)
         }
 
-        result, unwrap := userInput.Eval(funcs, vars)
+        result := userInput.Eval(funcs, vars, false)
         if result != nil {
-            if result.Tag == ast.LIST && unwrap {
-                result = result.Inner.(*ast.Token)
-            }
-
             for i := result; i != nil; i = i.Next {
                 fmt.Printf(i.String() + " ")
             }

stdlib/arith.go

@@ -32,13 +32,15 @@ import (
 func add(in *ast.Token, a ast.VarTable, f ast.FuncTable) *ast.Token {
     var res float64
 
+    in = in.Eval(f, a, false)
+
     for i := in; i != nil; i = i.Next {
         if i.Tag != ast.NUMBER {
             log.Log(log.ERR, "Non-number given to ADD", "add")
             return nil
         }
 
-        token := i.Inner.(string)
+        token := i.Value()
         isFloat := false
         for _, char := range token {
             if char == '.' {
@@ -64,20 +66,24 @@ func add(in *ast.Token, a ast.VarTable, f ast.FuncTable) *ast.Token {
         }
     }
 
-    return &ast.Token{Tag: ast.NUMBER, Inner: fmt.Sprintf("%f", res)}
+    t := &ast.Token{Tag: ast.NUMBER}
+    t.Set(fmt.Sprintf("%f", res))
+    return t
 }
 
 func sub(in *ast.Token, a ast.VarTable, f ast.FuncTable) *ast.Token {
     var res float64
     var sub float64
 
+    in = in.Eval(f, a, false)
+
     for i := in; i != nil; i = i.Next {
         if i.Tag != ast.NUMBER {
             log.Log(log.ERR, "Non-number given to SUB", "sub")
             return nil
         }
 
-        token := i.Inner.(string)
+        token := i.Value()
         isFloat := false
         var inner float64
         for _, char := range token {
@@ -110,19 +116,23 @@ func sub(in *ast.Token, a ast.VarTable, f ast.FuncTable) *ast.Token {
         }
     }
 
-    return &ast.Token{Tag: ast.NUMBER, Inner: fmt.Sprintf("%f", res - sub)}
+    t := &ast.Token{Tag: ast.NUMBER}
+    t.Set(fmt.Sprintf("%f", res - sub))
+    return t
 }
 
 func mult(in *ast.Token, a ast.VarTable, f ast.FuncTable) *ast.Token {
     res := 1.0
 
+    in = in.Eval(f, a, false)
+
     for i := in; i != nil; i = i.Next {
         if i.Tag != ast.NUMBER {
             log.Log(log.ERR, "Non-number given to MULT", "mult")
             return nil
         }
 
-        token := i.Inner.(string)
+        token := i.Value()
         isFloat := false
         for _, char := range token {
             if char == '.' {
@@ -148,12 +158,16 @@ func mult(in *ast.Token, a ast.VarTable, f ast.FuncTable) *ast.Token {
         }
     }
 
-    return &ast.Token{Tag: ast.NUMBER, Inner: fmt.Sprintf("%f", res)}
+    t := &ast.Token{Tag: ast.NUMBER}
+    t.Set(fmt.Sprintf("%f", res))
+    return t
 }
 
 func div(in *ast.Token, a ast.VarTable, f ast.FuncTable) *ast.Token {
     var res float64
 
+    in = in.Eval(f, a, false)
+
     for i := in; i != nil; i = i.Next {
         inner := 0.0
 
@@ -162,7 +176,7 @@ func div(in *ast.Token, a ast.VarTable, f ast.FuncTable) *ast.Token {
             return nil
         }
 
-        token := i.Inner.(string)
+        token := i.Value()
         isFloat := false
         for _, char := range token {
             if char == '.' {
@@ -194,5 +208,7 @@ func div(in *ast.Token, a ast.VarTable, f ast.FuncTable) *ast.Token {
         }
     }
 
-    return &ast.Token{Tag: ast.NUMBER, Inner: fmt.Sprintf("%f", res)}
+    t := &ast.Token{Tag: ast.NUMBER}
+    t.Set(fmt.Sprintf("%f", res))
+    return t
 }

stdlib/bool.go

@@ -23,26 +23,30 @@ import (
     "gitlab.com/whom/shs/ast"
 )
 
-// lt, gt
-
 func not(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
+    in = in.Eval(ft, vt, false)
+
     if in.Tag != ast.BOOL {
         log.Log(log.ERR, "non-bool argument to 'not'", "not")
         return nil
     }
 
     out := "T"
-    if in.Inner.(string) == "T" {
+    if in.Value() == "T" {
         out = "F"
     }
 
-    return &ast.Token{Tag: ast.BOOL, Inner: out}
+    t := &ast.Token{Tag: ast.BOOL}
+    t.Set(out)
+    return t
 }
 
 func eq(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
     out := "T"
     second := in.Next
 
+    in = in.Eval(ft, vt, false)
+
     if in.Tag != second.Tag {
         out = "F"
     } else {
@@ -67,14 +71,13 @@ func eq(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
                     }
 
                     if l_iter.Tag == ast.LIST {
-                        diff := consume_list(l_iter.Inner.(*ast.Token),
-                                     r_iter.Inner.(*ast.Token))
+                        diff := consume_list(l_iter.Expand(), r_iter.Expand())
                         if diff {
                             return true
                         } 
 
                     } else {
-                        if l_iter.Inner.(string) != r_iter.Inner.(string) {
+                        if l_iter.Value() != r_iter.Value() {
                             return true
                         }
                     }
@@ -90,56 +93,80 @@ func eq(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
                 }
             }
 
-            if consume_list(in.Inner.(*ast.Token), second.Inner.(*ast.Token)) {
+            if consume_list(in.Expand(), second.Expand()) {
                 out = "F"
             }
 
-        case ast.NUMBER, ast.STRING, ast.BOOL:
-            if in.Inner.(string) != second.Inner.(string) {
+        case ast.STRING, ast.BOOL:
+            if in.Value() != second.Value() {
+                out = "F"
+            }
+
+        case ast.NUMBER:
+            l_val, parse_err := strconv.ParseFloat(in.Value(), 64)
+            r_val, parse_err := strconv.ParseFloat(second.Value(), 64)
+            if parse_err != nil {
+                log.Log(log.ERR,
+                    "error parsing number: "+parse_err.Error(),
+                    "eq")
+                return nil
+            }
+
+            if l_val != r_val {
                 out = "F"
             }
         }
     }
 
-    return &ast.Token{Tag: ast.BOOL, Inner: out}
+    t := &ast.Token{Tag: ast.BOOL}
+    t.Set(out)
+    return t
 }
 
 func lt(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
     out := "T"
-
     second := in.Next
+
+    in = in.Eval(ft, vt, false)
+
     if in.Tag != ast.NUMBER || second.Tag != ast.NUMBER {
         log.Log(log.ERR, "non-number argument to numeric boolean operator", ">/<=")
         return nil
     }
 
-    l, _ := strconv.ParseInt(in.Inner.(string), 10, 64)
-    r, _ := strconv.ParseInt(second.Inner.(string), 10, 64)
+    l, _ := strconv.ParseInt(in.Value(), 10, 64)
+    r, _ := strconv.ParseInt(second.Value(), 10, 64)
 
     if l >= r {
         out = "F"
     }
 
-    return &ast.Token{Tag: ast.BOOL, Inner: out}
+    t := &ast.Token{Tag: ast.BOOL}
+    t.Set(out)
+    return t
 }
 
 func gt(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
     out := "T"
-
     second := in.Next
+
+    in = in.Eval(ft, vt, false)
+
     if in.Tag != ast.NUMBER || second.Tag != ast.NUMBER {
         log.Log(log.ERR, "non-number argument to numeric boolean operator", ">/<=")
         return nil
     }
 
-    l, _ := strconv.ParseInt(in.Inner.(string), 10, 64)
-    r, _ := strconv.ParseInt(second.Inner.(string), 10, 64)
+    l, _ := strconv.ParseInt(in.Value(), 10, 64)
+    r, _ := strconv.ParseInt(second.Value(), 10, 64)
 
     if l <= r {
         out = "F"
     }
 
-    return &ast.Token{Tag: ast.BOOL, Inner: out}
+    t := &ast.Token{Tag: ast.BOOL}
+    t.Set(out)
+    return t
 }
 
 func ne(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {

stdlib/call.go

@@ -35,13 +35,19 @@ var sigs = []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGTSTP, syscall.S
 
 
 func call(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
+    in = in.Eval(ft, vt, true)
     if in == nil {
         return nil
     }
 
-    path, err := exec.LookPath(in.Inner.(string))
+    if in.Tag == ast.LIST {
+        log.Log(log.ERR, "couldnt exec, target bin is a list", "call")
+        return nil
+    }
+
+    path, err := exec.LookPath(in.Value())
     if err != nil {
-        log.Log(log.ERR, "Couldnt exec " + in.Inner.(string) + ", file not found", "call")
+        log.Log(log.ERR, "Couldnt exec " + in.Value() + ", file not found", "call")
         return nil
     }
 
@@ -52,7 +58,7 @@ func call(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
             return nil
         }
 
-        args = append(args, i.Inner.(string))
+        args = append(args, i.Value())
     }
 
     var cmd *exec.Cmd
@@ -91,9 +97,14 @@ func bgcall(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
         return nil
     }
 
-    path, err := exec.LookPath(in.Inner.(string))
+    if in.Tag == ast.LIST {
+        log.Log(log.ERR, "couldnt exec, target bin is a list", "call")
+        return nil
+    }
+
+    path, err := exec.LookPath(in.Value())
     if err != nil {
-       log.Log(log.ERR, "Couldnt exec " + in.Inner.(string) + ", file not found", "call")
+       log.Log(log.ERR, "Couldnt exec " + in.Value() + ", file not found", "call")
         return nil
     }
 
@@ -104,7 +115,7 @@ func bgcall(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
             return nil
         }
 
-        args = append(args, i.Inner.(string))
+        args = append(args, i.Value())
     }
 
     var cmd *exec.Cmd
@@ -157,21 +168,22 @@ func fg(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
 
 func jobs(in *ast.Token, vt ast.VarTable, fg ast.FuncTable) *ast.Token {
     ret := &ast.Token{
-        Tag: ast.LIST, 
-        Inner: &ast.Token{
-            Tag: ast.STRING,
-            Inner: fmt.Sprintf("Total: %d", len(bgProcs)),
-        },
+        Tag: ast.LIST,
     }
 
-    ptr := ret.Inner.(*ast.Token)
-    iter := &ptr
+    _inner := &ast.Token{
+        Tag: ast.STRING,
+    }
+
+    ret.Direct(_inner)
+    _inner.Set(fmt.Sprintf("Total: %d", len(bgProcs)))
+
+    iter := &_inner
     for i := 0; i < len(bgProcs); i += 1 {
         (*iter).Next = &ast.Token{
             Tag: ast.STRING,
-            Inner: fmt.Sprintf("[%d]: %d", i, bgProcs[i].Process.Pid),
         }
-
+        (*iter).Next.Set(fmt.Sprintf("[%d]: %d", i, bgProcs[i].Process.Pid))
         iter = &(*iter).Next
     }
 
@@ -179,14 +191,21 @@ func jobs(in *ast.Token, vt ast.VarTable, fg ast.FuncTable) *ast.Token {
 }
 
 func read_cmd(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
+    in = in.Eval(ft, vt, true)
+
     if in == nil {
         return nil
     }
 
+    if in.Tag == ast.LIST {
+        log.Log(log.ERR, "couldnt exec, target bin is a list", "call")
+        return nil
+    }
+
     var out bytes.Buffer
-    path, err := exec.LookPath(in.Inner.(string))
+    path, err := exec.LookPath(in.Value())
     if err != nil {
-        log.Log(log.ERR, "Couldnt exec " + in.Inner.(string) + ", file not found", "call")
+        log.Log(log.ERR, "Couldnt exec " + in.Value() + ", file not found", "call")
         return nil
     }
 
@@ -197,7 +216,7 @@ func read_cmd(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
             return nil
         }
 
-        args = append(args, i.Inner.(string))
+        args = append(args, i.Value())
     }
 
     var cmd *exec.Cmd
@@ -228,21 +247,27 @@ func read_cmd(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
         }
     }
 
-    return &ast.Token{Tag: ast.STRING, Inner: out.String()}
+    ret := &ast.Token{Tag: ast.STRING}
+    ret.Set(out.String())
+    return ret
 
 }
 
 func get_exit(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
-    return &ast.Token{Tag: ast.NUMBER, Inner: fmt.Sprintf("%d", LastExitCode)}
+    ret := &ast.Token{Tag: ast.NUMBER}
+    ret.Set(fmt.Sprintf("%d", LastExitCode))
+    return ret
 }
 
 func kill(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
+    in = in.Eval(ft, vt, true)
+
     if in.Tag == ast.LIST {
         log.Log(log.ERR, "non-number argument to kill function", "kill")
         return nil
     }
 
-    pid, err := strconv.ParseInt(in.Inner.(string), 10, 64)
+    pid, err := strconv.ParseInt(in.Value(), 10, 64)
     if err != nil {
         log.Log(log.ERR, "error parsing arg to kill: " + err.Error(), "kill")
         return nil

stdlib/filesys.go

@@ -24,12 +24,13 @@ import (
 )
 
 func cd(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
+    in = in.Eval(ft, vt, false)
     if in.Tag == ast.LIST {
         log.Log(log.ERR, "Couldnt change dir to a list", "cd")
         return nil
     }
 
-    err := os.Chdir(in.Inner.(string))
+    err := os.Chdir(in.Value())
     if err != nil {
         log.Log(log.ERR, err.Error(), "cd")
     }

stdlib/list.go

@@ -33,7 +33,7 @@ func expand(input *ast.Token, vars ast.VarTable, funcs ast.FuncTable) *ast.Token
         return input
     }
 
-    return input.Inner.(*ast.Token)
+    return input.Eval(funcs, vars, false).Expand()
 }
 
 /* L_APPEND (append from repl)
@@ -45,15 +45,16 @@ func l_append(input *ast.Token, vars ast.VarTable, funcs ast.FuncTable) *ast.Tok
     src := input
 
     if input.Tag != ast.LIST {
-        // TODO: position??
-        return input
+        r := &ast.Token{Tag: ast.LIST}
+        r.Direct(input)
+        return r
     }
 
     // deref inner first
-    i := src.Inner.(*ast.Token)
+    i := src.Expand()
     iter := &i
     if *iter == nil {
-        src.Inner = input.Next
+        src.Direct(input.Next)
         src.Next = nil
 
     } else {

stdlib/stdlib.go

@@ -39,83 +39,6 @@ func GenFuncTable() ast.FuncTable {
             Args:           -1,
         },
 
-        "+": &ast.Function{
-            Function:      add,
-            Name:        "add",
-            TimesCalled:     0,
-            Args:           -1,
-        },
-
-        "-": &ast.Function{
-            Function:      sub,
-            Name:        "sub",
-            TimesCalled:     0,
-            Args:           -1,
-        },
-
-        "*": &ast.Function{
-            Function:     mult,
-            Name:       "mult",
-            TimesCalled:     0,
-            Args:           -1,
-        },
-
-        "/": &ast.Function{
-            Function:      div,
-            Name:        "div",
-            TimesCalled:     0,
-            Args:           -1,
-        },
-
-        "l": &ast.Function{
-            Function:     call,
-            Name:       "call",
-            TimesCalled:     0,
-            Args:           -1,
-        },
-
-        "bg": &ast.Function{
-            Function:        bgcall,
-            Name: "background call",
-            TimesCalled:          0,
-            Args:                -1,
-        },
-
-        "fg": &ast.Function{
-            Function:       fg,
-            Name: "foreground",
-            TimesCalled:     0,
-            Args:            0,
-        },
-
-        "cd": &ast.Function{
-            Function:       cd,
-            Name:  "changedir",
-            TimesCalled:     0,
-            Args:            1,
-        },
-
-        "$": &ast.Function{
-            Function: read_cmd,
-            Name:   "read cmd",
-            TimesCalled:     0,
-            Args:           -1,
-        },
-
-        "concat": &ast.Function{
-            Function:   concat,
-            Name:"concatenate",
-            TimesCalled:     0,
-            Args:           -1,
-        },
-
-        "print": &ast.Function{
-            Function:print_str,
-            Name:      "print",
-            TimesCalled:     0,
-            Args:            1,
-        },
-
         "exit": &ast.Function{
             Function: exit_shell,
             Name:         "exit",
@@ -123,23 +46,6 @@ func GenFuncTable() ast.FuncTable {
             Args:              0,
         },
 
-        "?": &ast.Function{
-            Function:   get_exit,
-            Name:"get exit code",
-            TimesCalled:       0,
-            Args:              0,
-        },
-
-        /*
-        USE NATIVE KILL COMMAND.
-        "kill": &ast.Function{
-            Function:       kill,
-            Name:     "kill job",
-            TimesCalled:       0,
-            Args:              1,
-        },
-        */
-
         "eq": &ast.Function{
             Function:         eq,
             Name:           "==",
@@ -189,12 +95,107 @@ func GenFuncTable() ast.FuncTable {
             Args:              1,
         },
 
+        "+": &ast.Function{
+            Function:      add,
+            Name:        "add",
+            TimesCalled:     0,
+            Args:           -1,
+        },
+
+        "-": &ast.Function{
+            Function:      sub,
+            Name:        "sub",
+            TimesCalled:     0,
+            Args:           -1,
+        },
+
+        "*": &ast.Function{
+            Function:     mult,
+            Name:       "mult",
+            TimesCalled:     0,
+            Args:           -1,
+        },
+
+        "/": &ast.Function{
+            Function:      div,
+            Name:        "div",
+            TimesCalled:     0,
+            Args:           -1,
+        },
+
+        "cd": &ast.Function{
+            Function:       cd,
+            Name:  "changedir",
+            TimesCalled:     0,
+            Args:            1,
+        },
+
+        "concat": &ast.Function{
+            Function:   concat,
+            Name:"concatenate",
+            TimesCalled:     0,
+            Args:           -1,
+        },
+
+        "print": &ast.Function{
+            Function:print_str,
+            Name:      "print",
+            TimesCalled:     0,
+            Args:            1,
+        },
+
+        "l": &ast.Function{
+            Function:     call,
+            Name:       "call",
+            TimesCalled:     0,
+            Args:           -1,
+        },
+
+        "bg": &ast.Function{
+            Function:        bgcall,
+            Name: "background call",
+            TimesCalled:          0,
+            Args:                -1,
+        },
+
+        "fg": &ast.Function{
+            Function:       fg,
+            Name: "foreground",
+            TimesCalled:     0,
+            Args:            0,
+        },
+
+        "$": &ast.Function{
+            Function: read_cmd,
+            Name:   "read cmd",
+            TimesCalled:     0,
+            Args:           -1,
+        },
+
+        "?": &ast.Function{
+            Function:   get_exit,
+            Name:"get exit code",
+            TimesCalled:       0,
+            Args:              0,
+        },
+
+        /*
+        USE NATIVE KILL COMMAND.
+        "kill": &ast.Function{
+            Function:       kill,
+            Name:     "kill job",
+            TimesCalled:       0,
+            Args:              1,
+        },
+        */
+
         "jobs": &ast.Function{
             Function:       jobs,
             Name:    "list jobs",
             TimesCalled:       0,
             Args:              0,
         },
+
     }
 
     return stdlib

stdlib/string.go

@@ -24,6 +24,8 @@ import (
 )
 
 func concat(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
+    in = in.Eval(ft, vt, false)
+
     var res string
     for i := in; i != nil; i = i.Next {
         if i.Tag == ast.LIST {
@@ -32,13 +34,15 @@ func concat(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
             continue
         }
 
-        res += i.Inner.(string)
+        res += i.Value()
     }
 
-    return &ast.Token{Tag: ast.STRING, Inner: res}
+    t := &ast.Token{Tag: ast.STRING}
+    t.Set(res)
+    return t
 }
 
 func print_str(in *ast.Token, vt ast.VarTable, ft ast.FuncTable) *ast.Token {
-    fmt.Println(in)
+    fmt.Println(in.Eval(ft, vt, false))
     return nil
 }