diff --git a/micronlp/tokenizer.py b/micronlp/tokenizer.py index e373763..c33a294 100644 --- a/micronlp/tokenizer.py +++ b/micronlp/tokenizer.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import unicodedata -from typing import List, Iterator, Iterable +from typing import Iterator, Iterable import emoji @@ -46,7 +46,7 @@ def tokenize( yield last_token -def ngrams(tokens: Iterable[str], max_ngram_length: int) -> List[str]: +def ngrams(tokens: Iterable[str], max_ngram_length: int) -> Iterable[str]: if max_ngram_length == 1: return tokens