kj7rrv/gptc
1
0
Fork 0
Code Issues 2 Pull Requests Projects Releases 12 Wiki Activity

Split hash function

Browse Source
This commit is contained in:
Samuel Sloniker 2022-11-26 17:42:42 -08:00
parent 08437a2696
commit b3a43150d8
Signed by: kj7rrv
GPG Key ID: 1BB4029E66285A62
1 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
gptc/tokenizer.py
View File

@ -39,13 +39,14 @@ def tokenize(text: str, max_ngram_length: int = 1) -> List[str]:
return ngrams return ngrams
def hash(tokens: List[str]) -> List[int]: def hash_single(token: str) -> int:
return [ return int.from_bytes(
int.from_bytes(
hashlib.sha256(token.encode("utf-8")).digest()[:6], "big" hashlib.sha256(token.encode("utf-8")).digest()[:6], "big"
) )
for token in tokens
]
def hash(tokens: List[str]) -> List[int]:
return [hash_single(token) for token in tokens]
def normalize(text: str) -> str: def normalize(text: str) -> str: