package shsh; import ( "strings" "unicode" ) type token_t int const ( LIST token_t = iota STRING token_t = iota NUMBER token_t = iota SYMBOL token_t = iota ) type Token struct { next *Token tag token_t position int _inner interface{} } func Lex(input string) *Token { ret := new(Token) iter := &ret delim := ' ' var tok strings.Builder iter_alloced := false is_list := false is_str := false is_num := true for pos, char := range input { switch char { // TODO: User configurable delimiters case '\'', '"', '`': is_str = true delim = char case '(': is_list = true delim = ')' case delim: *iter = new(Token) (*iter).position = pos if is_list { (*iter)._inner = Lex(tok.String()) is_list = false (*iter)._inner = LIST } else { (*iter)._inner = tok.String() if is_str { (*iter).tag = STRING is_str = false } else if is_num { (*iter).tag = NUMBER } else { (*iter).tag = SYMBOL } } iter_alloced = true delim = ' ' default: is_num = is_num && unicode.IsDigit(char) tok.WriteRune(char) } if iter_alloced { iter = &((*iter).next) iter_alloced = false tok.Reset() } } // TODO: Throw parsing error here if there is leftover in tok return ret }