Make sure text is lowercase

2022-11-22 11:44:13 -08:00 · 2022-11-22 11:44:13 -08:00 · 7d1cbcaee0
commit 7d1cbcaee0
parent c2cd6f62fb
1 changed files with 1 additions and 0 deletions
--- a/gptc/tokenizer.py
+++ b/gptc/tokenizer.py
@ -17,6 +17,7 @@ def tokenize(
    converted_text: Union[str, List[str]] = text.lower()

    if has_emoji and use_emoji:
+        text = text.lower()
        parts = []
        highest_end = 0
        for emoji_part in emoji.emoji_list(text):