SHS/token.go

package shsh;

import (
    "strings"
    "unicode"
)

type token_t int
const (
    LIST      token_t = iota
    STRING    token_t = iota
    NUMBER    token_t = iota
    SYMBOL    token_t = iota
)

type Token struct {
    next *Token
    tag token_t
    position int
    _inner interface{}
}

func Lex(input string) *Token {
    if len(input) == 0 {
        return nil
    }

    var ret *Token
    iter := &ret
    delim := ' '
    var tok strings.Builder
    is_list := false
    is_str  := false

    tokenBuilder := func (pos int, tok string) {
        if len(tok) == 0 && !is_list && !is_str {
            return
        }

        *iter = new(Token)
        (*iter).position = pos

        if is_list {
            (*iter)._inner = Lex(tok)
            (*iter).tag = LIST
            is_list = false

        } else {
            (*iter)._inner = tok
            if is_str {
                (*iter).tag = STRING
                is_str = false

            } else if tokenIsNumber(tok) {
                (*iter).tag = NUMBER

            } else {
                (*iter).tag = SYMBOL
            }
        }

        iter = &(*iter).next
    }

    for pos, char := range input {
        if char == delim {
            if is_str && is_list {
                // String just ended inside list
                is_str = false
                delim = ')'
                tok.WriteRune(char)
                continue
            }

            delim = ' '
            tokenBuilder(pos, tok.String())
            tok.Reset()

        } else {
            if strings.ContainsRune("\"'`", char) {
                is_str = true
                delim = char
                if !is_list {
                    continue
                }

            } else if char == '(' && !is_str {
                is_list = true
                delim = ')'
                continue
            }

            tok.WriteRune(char)
        }
    }

    if tok.Len() > 0 {
        if is_list || is_str {
            // TODO: Throw hella lex error here
            return ret
        }

        tokenBuilder(len(input), tok.String())
    }

    return ret
}

func tokenIsNumber(arg string) bool {
    dotCount := 0

    for _, char := range arg {
        if !unicode.IsDigit(char) {
            if char == '.' && dotCount == 0 {
                dotCount++
            } else {
                return false
            }
        }
    }

    return true
}