Normalize characters

Closes #3
2022-12-24 10:46:40 -08:00 · 2022-12-24 10:46:40 -08:00 · 10668691ea
commit 10668691ea
parent 295a1189de
1 changed files with 2 additions and 1 deletions
--- a/gptc/tokenizer.py
+++ b/gptc/tokenizer.py
@ -3,10 +3,11 @@
 from typing import List, Union
 import hashlib
 import emoji
+import unicodedata


 def tokenize(text: str, max_ngram_length: int = 1) -> List[str]:
-    text = text.lower()
+    text = unicodedata.normalize("NFKD", text).lower()
    parts = []
    highest_end = 0
    for emoji_part in emoji.emoji_list(text):