From 79d8c73183b89f7017ca96c8b3bf77b4fbcb8343 Mon Sep 17 00:00:00 2001 From: Samuel Sloniker Date: Thu, 15 Jun 2023 12:15:39 -0700 Subject: [PATCH] Fix type bug --- micronlp/tokenizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/micronlp/tokenizer.py b/micronlp/tokenizer.py index e373763..c33a294 100644 --- a/micronlp/tokenizer.py +++ b/micronlp/tokenizer.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import unicodedata -from typing import List, Iterator, Iterable +from typing import Iterator, Iterable import emoji @@ -46,7 +46,7 @@ def tokenize( yield last_token -def ngrams(tokens: Iterable[str], max_ngram_length: int) -> List[str]: +def ngrams(tokens: Iterable[str], max_ngram_length: int) -> Iterable[str]: if max_ngram_length == 1: return tokens