diff --git a/gptc/tokenizer.py b/gptc/tokenizer.py index fe09223..bf687a2 100644 --- a/gptc/tokenizer.py +++ b/gptc/tokenizer.py @@ -17,6 +17,7 @@ def tokenize( converted_text: Union[str, List[str]] = text.lower() if has_emoji and use_emoji: + text = text.lower() parts = [] highest_end = 0 for emoji_part in emoji.emoji_list(text):