diff --git a/.token.go.swp b/.token.go.swp index a658897..280a049 100644 Binary files a/.token.go.swp and b/.token.go.swp differ diff --git a/shsh b/shsh new file mode 100755 index 0000000..f8286ff Binary files /dev/null and b/shsh differ diff --git a/token.go b/token.go index 4a954a2..9a5d77f 100644 --- a/token.go +++ b/token.go @@ -31,9 +31,12 @@ func Lex(input string) *Token { var tok strings.Builder is_list := false is_str := false - is_num := true tokenBuilder := func (pos int, tok string) { + if len(tok) == 0 && !is_list && !is_str { + return + } + *iter = new(Token) (*iter).position = pos @@ -48,7 +51,7 @@ func Lex(input string) *Token { (*iter).tag = STRING is_str = false - } else if is_num { + } else if tokenIsNumber(tok) { (*iter).tag = NUMBER } else { @@ -60,33 +63,61 @@ func Lex(input string) *Token { } for pos, char := range input { - switch char { - // TODO: User configurable delimiters - case '\'', '"', '`': - is_str = true - delim = char - case '(': - is_list = true - delim = ')' - - case delim: - delim = ' ' - if tok.Len() == 0 { + if char == delim { + if is_str && is_list { + // String just ended inside list + is_str = false + delim = ')' + tok.WriteRune(char) continue } + delim = ' ' tokenBuilder(pos, tok.String()) tok.Reset() - default: - is_num = is_num && unicode.IsDigit(char) + } else { + if strings.ContainsRune("\"'`", char) { + is_str = true + delim = char + if !is_list { + continue + } + + } else if char == '(' && !is_str { + is_list = true + delim = ')' + continue + } + tok.WriteRune(char) } } if tok.Len() > 0 { + if is_list || is_str { + // TODO: Throw hella lex error here + return ret + } + tokenBuilder(len(input), tok.String()) } return ret } + +func tokenIsNumber(arg string) bool { + dotCount := 0 + + for _, char := range arg { + if !unicode.IsDigit(char) { + if char == '.' && dotCount == 0 { + dotCount++ + } else { + return false + } + } + } + + return true +}