Add normalize()
This commit is contained in:
parent
fc4665bb9e
commit
08437a2696
|
@ -6,6 +6,7 @@ from gptc.compiler import compile as compile
|
|||
from gptc.classifier import Classifier as Classifier
|
||||
from gptc.pack import pack as pack
|
||||
from gptc.model import Model as Model, deserialize as deserialize
|
||||
from gptc.tokenizer import normalize as normalize
|
||||
from gptc.exceptions import (
|
||||
GPTCError as GPTCError,
|
||||
ModelError as ModelError,
|
||||
|
|
|
@ -46,3 +46,7 @@ def hash(tokens: List[str]) -> List[int]:
|
|||
)
|
||||
for token in tokens
|
||||
]
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
return " ".join(tokenize(text, 1))
|
||||
|
|
Loading…
Reference in New Issue
Block a user