package shsh; import ( "strings" "unicode" ) type token_t int const ( LIST token_t = iota STRING token_t = iota NUMBER token_t = iota SYMBOL token_t = iota ) type Token struct { next *Token tag token_t position int _inner interface{} } func Lex(input string) *Token { if len(input) == 0 { return nil } var ret *Token iter := &ret delim := ' ' var tok strings.Builder is_list := false is_str := false tokenBuilder := func (pos int, tok string) { if len(tok) == 0 && !is_list && !is_str { return } *iter = new(Token) (*iter).position = pos if is_list { (*iter)._inner = Lex(tok) (*iter).tag = LIST is_list = false } else { (*iter)._inner = tok if is_str { (*iter).tag = STRING is_str = false } else if tokenIsNumber(tok) { (*iter).tag = NUMBER } else { (*iter).tag = SYMBOL } } iter = &(*iter).next } for pos, char := range input { if char == delim { if is_str && is_list { // String just ended inside list is_str = false delim = ')' tok.WriteRune(char) continue } delim = ' ' tokenBuilder(pos, tok.String()) tok.Reset() } else { if strings.ContainsRune("\"'`", char) { is_str = true delim = char if !is_list { continue } } else if char == '(' && !is_str { is_list = true delim = ')' continue } tok.WriteRune(char) } } if tok.Len() > 0 { if is_list || is_str { // TODO: Throw hella lex error here return ret } tokenBuilder(len(input), tok.String()) } return ret } func tokenIsNumber(arg string) bool { dotCount := 0 for _, char := range arg { if !unicode.IsDigit(char) { if char == '.' && dotCount == 0 { dotCount++ } else { return false } } } return true }