SHS/token.go

package shsh;

import (
    "strings"
    "unicode"
)

type token_t int
const (
    LIST      token_t = iota
    STRING    token_t = iota
    NUMBER    token_t = iota
    SYMBOL    token_t = iota
)

type Token struct {
    next *Token
    tag token_t
    position int
    _inner interface{}
}

func Lex(input string) *Token {
    if len(input) == 0 {
        return nil
    }

    var ret *Token
    iter := &ret
    delim := ' '
    var tok strings.Builder
    is_list := false
    is_str  := false
    is_num  := true

    tokenBuilder := func (pos int, tok string) {
        *iter = new(Token)
        (*iter).position = pos

        if is_list {
            (*iter)._inner = Lex(tok)
            (*iter).tag = LIST
            is_list = false

        } else {
            (*iter)._inner = tok
            if is_str {
                (*iter).tag = STRING
                is_str = false

            } else if is_num {
                (*iter).tag = NUMBER

            } else {
                (*iter).tag = SYMBOL
            }
        }

        iter = &(*iter).next
    }

    for pos, char := range input {
        switch char {
        // TODO: User configurable delimiters
        case '\'', '"', '`':
            is_str = true
            delim = char
        case '(':
            is_list = true
            delim = ')'

        case delim:
            delim = ' '
            if tok.Len() == 0 {
                continue
            }

            tokenBuilder(pos, tok.String())
            tok.Reset()

        default:
            is_num = is_num && unicode.IsDigit(char)
            tok.WriteRune(char)
        }
    }

    if tok.Len() > 0 {
        tokenBuilder(len(input), tok.String())
    }

    return ret
}