Include numbers in tokenized output

Closes #12
2022-12-24 10:42:50 -08:00 · 2022-12-24 10:42:50 -08:00 · 295a1189de
commit 295a1189de
parent 74b2ba81b9
1 changed files with 6 additions and 1 deletions
--- a/gptc/tokenizer.py
+++ b/gptc/tokenizer.py
@ -19,7 +19,12 @@ def tokenize(text: str, max_ngram_length: int = 1) -> List[str]:
    tokens = [""]

    for char in converted_text:
-        if char.isalpha() or char == "'":
+        if (
+            char.isalpha()
+            or char.isnumeric()
+            or char == "'"
+            or (char in ",." and (" " + tokens[-1])[-1].isnumeric())
+        ):
            tokens[-1] += char
        elif emoji.is_emoji(char):
            tokens.append(char)