implement iseq, tests
Signed-off-by: Ava Hahn <ava@aidanis.online>
This commit is contained in:
parent
5ce0a8e8b2
commit
cb83fa5655
6 changed files with 134 additions and 25 deletions
|
|
@ -212,7 +212,7 @@ fn process(document: &String) -> Result<Box<Seg>, String> {
|
|||
*/
|
||||
fn tok_is_symbol(token: &str) -> Option<String> {
|
||||
for t in token.chars() {
|
||||
if !t.is_alphanumeric() && t != '-' && t != '_' {
|
||||
if !t.is_alphanumeric() && t != '-' && t != '_' && t != '?' {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue