Comparing 4cb8b71407..82524345f3 - gptc - KJ7RRV Gitea

kj7rrv/gptc

1 changed files with 0 additions and 1 deletions

									
										1

gptc/tokenizer.py
									
										View File
										
					@ -17,7 +17,6 @@ def tokenize(

					    converted_text: Union[str, List[str]] = text.lower()

					    converted_text: Union[str, List[str]] = text.lower()

					    if has_emoji and use_emoji:

					    if has_emoji and use_emoji:

					        text = text.lower()

					        parts = []

					        parts = []

					        highest_end = 0

					        highest_end = 0

					        for emoji_part in emoji.emoji_list(text):

					        for emoji_part in emoji.emoji_list(text):