kj7rrv/micronlp
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Fix type bug

Browse Source
This commit is contained in:
Samuel Sloniker 2023-06-15 12:15:39 -07:00
parent a1a200dd58
commit 79d8c73183
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
micronlp/tokenizer.py
View File

@ -1,7 +1,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import unicodedata import unicodedata
from typing import List, Iterator, Iterable from typing import Iterator, Iterable
import emoji import emoji
@ -46,7 +46,7 @@ def tokenize(
yield last_token yield last_token
def ngrams(tokens: Iterable[str], max_ngram_length: int) -> List[str]: def ngrams(tokens: Iterable[str], max_ngram_length: int) -> Iterable[str]:
if max_ngram_length == 1: if max_ngram_length == 1:
return tokens return tokens