Compare commits
No commits in common. "97c4eef086564655f22716318a2f5d704afd89f9" and "2c3fc77ba6465d42568a0f5788a9931d7045f955" have entirely different histories.
97c4eef086
...
2c3fc77ba6
|
@ -48,7 +48,7 @@ example of the format. Any exceptions will be printed to stderr.
|
||||||
|
|
||||||
Write binary data representing the model to `file`.
|
Write binary data representing the model to `file`.
|
||||||
|
|
||||||
### `Model.deserialize(encoded_model)`
|
### `gptc.deserialize(encoded_model)`
|
||||||
|
|
||||||
Deserialize a `Model` from a file containing data from `Model.serialize()`.
|
Deserialize a `Model` from a file containing data from `Model.serialize()`.
|
||||||
|
|
||||||
|
@ -70,7 +70,7 @@ Return a confidence dict for the given token or ngram. This function is very
|
||||||
similar to `Model.confidence()`, except it treats the input as a single token
|
similar to `Model.confidence()`, except it treats the input as a single token
|
||||||
or ngram.
|
or ngram.
|
||||||
|
|
||||||
### `Model.compile(raw_model, max_ngram_length=1, min_count=1, hash_algorithm="sha256")`
|
### `gptc.compile(raw_model, max_ngram_length=1, min_count=1, hash_algorithm="sha256")`
|
||||||
|
|
||||||
Compile a raw model (as a list, not JSON) and return the compiled model (as a
|
Compile a raw model (as a list, not JSON) and return the compiled model (as a
|
||||||
`gptc.Model` object).
|
`gptc.Model` object).
|
||||||
|
|
|
@ -25,7 +25,7 @@ print(
|
||||||
round(
|
round(
|
||||||
1000000
|
1000000
|
||||||
* timeit.timeit(
|
* timeit.timeit(
|
||||||
"gptc.Model.compile(raw_model, max_ngram_length)",
|
"gptc.compile(raw_model, max_ngram_length)",
|
||||||
number=compile_iterations,
|
number=compile_iterations,
|
||||||
globals=globals(),
|
globals=globals(),
|
||||||
)
|
)
|
||||||
|
|
|
@ -2,11 +2,12 @@
|
||||||
|
|
||||||
"""General-Purpose Text Classifier"""
|
"""General-Purpose Text Classifier"""
|
||||||
|
|
||||||
from gptc.pack import pack
|
from gptc.compiler import compile as compile
|
||||||
from gptc.model import Model
|
from gptc.pack import pack as pack
|
||||||
from gptc.tokenizer import normalize
|
from gptc.model import Model as Model, deserialize as deserialize
|
||||||
|
from gptc.tokenizer import normalize as normalize
|
||||||
from gptc.exceptions import (
|
from gptc.exceptions import (
|
||||||
GPTCError,
|
GPTCError as GPTCError,
|
||||||
ModelError,
|
ModelError as ModelError,
|
||||||
InvalidModelError,
|
InvalidModelError as InvalidModelError,
|
||||||
)
|
)
|
||||||
|
|
|
@ -59,16 +59,16 @@ def main() -> None:
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if args.subparser_name == "compile":
|
if args.subparser_name == "compile":
|
||||||
with open(args.model, "r", encoding="utf-8") as input_file:
|
with open(args.model, "r") as f:
|
||||||
model = json.load(input_file)
|
model = json.load(f)
|
||||||
|
|
||||||
with open(args.out, "wb+") as output_file:
|
with open(args.out, "wb+") as f:
|
||||||
gptc.Model.compile(
|
gptc.compile(
|
||||||
model, args.max_ngram_length, args.min_count
|
model, args.max_ngram_length, args.min_count
|
||||||
).serialize(output_file)
|
).serialize(f)
|
||||||
elif args.subparser_name == "classify":
|
elif args.subparser_name == "classify":
|
||||||
with open(args.model, "rb") as model_file:
|
with open(args.model, "rb") as f:
|
||||||
model = gptc.Model.deserialize(model_file)
|
model = gptc.deserialize(f)
|
||||||
|
|
||||||
if sys.stdin.isatty():
|
if sys.stdin.isatty():
|
||||||
text = input("Text to analyse: ")
|
text = input("Text to analyse: ")
|
||||||
|
@ -77,8 +77,8 @@ def main() -> None:
|
||||||
|
|
||||||
print(json.dumps(model.confidence(text, args.max_ngram_length)))
|
print(json.dumps(model.confidence(text, args.max_ngram_length)))
|
||||||
elif args.subparser_name == "check":
|
elif args.subparser_name == "check":
|
||||||
with open(args.model, "rb") as model_file:
|
with open(args.model, "rb") as f:
|
||||||
model = gptc.Model.deserialize(model_file)
|
model = gptc.deserialize(f)
|
||||||
print(json.dumps(model.get(args.token)))
|
print(json.dumps(model.get(args.token)))
|
||||||
else:
|
else:
|
||||||
print(json.dumps(gptc.pack(args.model, True)[0]))
|
print(json.dumps(gptc.pack(args.model, True)[0]))
|
||||||
|
|
92
gptc/compiler.py
Executable file
92
gptc/compiler.py
Executable file
|
@ -0,0 +1,92 @@
|
||||||
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
|
import gptc.tokenizer
|
||||||
|
import gptc.model
|
||||||
|
from typing import Iterable, Mapping, List, Dict, Union, Tuple
|
||||||
|
|
||||||
|
|
||||||
|
def _count_words(
|
||||||
|
raw_model: Iterable[Mapping[str, str]],
|
||||||
|
max_ngram_length: int,
|
||||||
|
hash_algorithm: str,
|
||||||
|
) -> Tuple[Dict[int, Dict[str, int]], Dict[str, int], List[str]]:
|
||||||
|
word_counts: Dict[int, Dict[str, int]] = {}
|
||||||
|
category_lengths: Dict[str, int] = {}
|
||||||
|
names: List[str] = []
|
||||||
|
|
||||||
|
for portion in raw_model:
|
||||||
|
text = gptc.tokenizer.hash(
|
||||||
|
gptc.tokenizer.tokenize(portion["text"], max_ngram_length),
|
||||||
|
hash_algorithm,
|
||||||
|
)
|
||||||
|
category = portion["category"]
|
||||||
|
|
||||||
|
if not category in names:
|
||||||
|
names.append(category)
|
||||||
|
|
||||||
|
category_lengths[category] = category_lengths.get(category, 0) + len(
|
||||||
|
text
|
||||||
|
)
|
||||||
|
|
||||||
|
for word in text:
|
||||||
|
if word in word_counts:
|
||||||
|
try:
|
||||||
|
word_counts[word][category] += 1
|
||||||
|
except KeyError:
|
||||||
|
word_counts[word][category] = 1
|
||||||
|
else:
|
||||||
|
word_counts[word] = {category: 1}
|
||||||
|
|
||||||
|
return word_counts, category_lengths, names
|
||||||
|
|
||||||
|
|
||||||
|
def _get_weights(
|
||||||
|
min_count: int,
|
||||||
|
word_counts: Dict[int, Dict[str, int]],
|
||||||
|
category_lengths: Dict[str, int],
|
||||||
|
names: List[str],
|
||||||
|
) -> Dict[int, List[int]]:
|
||||||
|
model: Dict[int, List[int]] = {}
|
||||||
|
for word, counts in word_counts.items():
|
||||||
|
if sum(counts.values()) >= min_count:
|
||||||
|
weights = {
|
||||||
|
category: value / category_lengths[category]
|
||||||
|
for category, value in counts.items()
|
||||||
|
}
|
||||||
|
total = sum(weights.values())
|
||||||
|
new_weights: List[int] = []
|
||||||
|
for category in names:
|
||||||
|
new_weights.append(
|
||||||
|
round((weights.get(category, 0) / total) * 65535)
|
||||||
|
)
|
||||||
|
model[word] = new_weights
|
||||||
|
return model
|
||||||
|
|
||||||
|
|
||||||
|
def compile(
|
||||||
|
raw_model: Iterable[Mapping[str, str]],
|
||||||
|
max_ngram_length: int = 1,
|
||||||
|
min_count: int = 1,
|
||||||
|
hash_algorithm: str = "sha256",
|
||||||
|
) -> gptc.model.Model:
|
||||||
|
"""Compile a raw model.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
raw_model : list of dict
|
||||||
|
A raw GPTC model.
|
||||||
|
|
||||||
|
max_ngram_length : int
|
||||||
|
Maximum ngram lenght to compile with.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
dict
|
||||||
|
A compiled GPTC model.
|
||||||
|
|
||||||
|
"""
|
||||||
|
word_counts, category_lengths, names = _count_words(
|
||||||
|
raw_model, max_ngram_length, hash_algorithm
|
||||||
|
)
|
||||||
|
model = _get_weights(min_count, word_counts, category_lengths, names)
|
||||||
|
return gptc.model.Model(model, names, max_ngram_length, hash_algorithm)
|
268
gptc/model.py
268
gptc/model.py
|
@ -1,120 +1,11 @@
|
||||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
from typing import (
|
|
||||||
Iterable,
|
|
||||||
Mapping,
|
|
||||||
List,
|
|
||||||
Dict,
|
|
||||||
cast,
|
|
||||||
BinaryIO,
|
|
||||||
Tuple,
|
|
||||||
TypedDict,
|
|
||||||
)
|
|
||||||
import json
|
|
||||||
import gptc.tokenizer
|
import gptc.tokenizer
|
||||||
from gptc.exceptions import InvalidModelError
|
from gptc.exceptions import InvalidModelError
|
||||||
import gptc.weighting
|
import gptc.weighting
|
||||||
|
from typing import Iterable, Mapping, List, Dict, Union, cast, BinaryIO
|
||||||
def _count_words(
|
import json
|
||||||
raw_model: Iterable[Mapping[str, str]],
|
import collections
|
||||||
max_ngram_length: int,
|
|
||||||
hash_algorithm: str,
|
|
||||||
) -> Tuple[Dict[int, Dict[str, int]], Dict[str, int], List[str]]:
|
|
||||||
word_counts: Dict[int, Dict[str, int]] = {}
|
|
||||||
category_lengths: Dict[str, int] = {}
|
|
||||||
names: List[str] = []
|
|
||||||
|
|
||||||
for portion in raw_model:
|
|
||||||
text = gptc.tokenizer.hash_list(
|
|
||||||
gptc.tokenizer.tokenize(portion["text"], max_ngram_length),
|
|
||||||
hash_algorithm,
|
|
||||||
)
|
|
||||||
category = portion["category"]
|
|
||||||
|
|
||||||
if not category in names:
|
|
||||||
names.append(category)
|
|
||||||
|
|
||||||
category_lengths[category] = category_lengths.get(category, 0) + len(
|
|
||||||
text
|
|
||||||
)
|
|
||||||
|
|
||||||
for word in text:
|
|
||||||
if word in word_counts:
|
|
||||||
try:
|
|
||||||
word_counts[word][category] += 1
|
|
||||||
except KeyError:
|
|
||||||
word_counts[word][category] = 1
|
|
||||||
else:
|
|
||||||
word_counts[word] = {category: 1}
|
|
||||||
|
|
||||||
return word_counts, category_lengths, names
|
|
||||||
|
|
||||||
|
|
||||||
def _get_weights(
|
|
||||||
min_count: int,
|
|
||||||
word_counts: Dict[int, Dict[str, int]],
|
|
||||||
category_lengths: Dict[str, int],
|
|
||||||
names: List[str],
|
|
||||||
) -> Dict[int, List[int]]:
|
|
||||||
model: Dict[int, List[int]] = {}
|
|
||||||
for word, counts in word_counts.items():
|
|
||||||
if sum(counts.values()) >= min_count:
|
|
||||||
weights = {
|
|
||||||
category: value / category_lengths[category]
|
|
||||||
for category, value in counts.items()
|
|
||||||
}
|
|
||||||
total = sum(weights.values())
|
|
||||||
new_weights: List[int] = []
|
|
||||||
for category in names:
|
|
||||||
new_weights.append(
|
|
||||||
round((weights.get(category, 0) / total) * 65535)
|
|
||||||
)
|
|
||||||
model[word] = new_weights
|
|
||||||
return model
|
|
||||||
|
|
||||||
class ExplanationEntry(TypedDict):
|
|
||||||
weight: float
|
|
||||||
probabilities: Dict[str, float]
|
|
||||||
count: int
|
|
||||||
|
|
||||||
|
|
||||||
Explanation = Dict[
|
|
||||||
str,
|
|
||||||
ExplanationEntry,
|
|
||||||
]
|
|
||||||
|
|
||||||
Log = List[Tuple[str, float, List[float]]]
|
|
||||||
|
|
||||||
|
|
||||||
class Confidences(dict[str, float]):
|
|
||||||
def __init__(self, probs: Dict[str, float]):
|
|
||||||
dict.__init__(self, probs)
|
|
||||||
|
|
||||||
|
|
||||||
class TransparentConfidences(Confidences):
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
probs: Dict[str, float],
|
|
||||||
explanation: Explanation,
|
|
||||||
):
|
|
||||||
self.explanation = explanation
|
|
||||||
Confidences.__init__(self, probs)
|
|
||||||
|
|
||||||
|
|
||||||
def convert_log(log: Log, names: List[str]) -> Explanation:
|
|
||||||
explanation: Explanation = {}
|
|
||||||
for word2, weight, word_probs in log:
|
|
||||||
if word2 in explanation:
|
|
||||||
explanation[word2]["count"] += 1
|
|
||||||
else:
|
|
||||||
explanation[word2] = {
|
|
||||||
"weight": weight,
|
|
||||||
"probabilities": {
|
|
||||||
name: word_probs[index] for index, name in enumerate(names)
|
|
||||||
},
|
|
||||||
"count": 1,
|
|
||||||
}
|
|
||||||
return explanation
|
|
||||||
|
|
||||||
|
|
||||||
class Model:
|
class Model:
|
||||||
|
@ -132,7 +23,7 @@ class Model:
|
||||||
|
|
||||||
def confidence(
|
def confidence(
|
||||||
self, text: str, max_ngram_length: int, transparent: bool = False
|
self, text: str, max_ngram_length: int, transparent: bool = False
|
||||||
) -> Confidences:
|
) -> Dict[str, float]:
|
||||||
"""Classify text with confidence.
|
"""Classify text with confidence.
|
||||||
|
|
||||||
Parameters
|
Parameters
|
||||||
|
@ -158,14 +49,14 @@ class Model:
|
||||||
text, min(max_ngram_length, self.max_ngram_length)
|
text, min(max_ngram_length, self.max_ngram_length)
|
||||||
)
|
)
|
||||||
|
|
||||||
tokens = gptc.tokenizer.hash_list(
|
tokens = gptc.tokenizer.hash(
|
||||||
raw_tokens,
|
raw_tokens,
|
||||||
self.hash_algorithm,
|
self.hash_algorithm,
|
||||||
)
|
)
|
||||||
|
|
||||||
if transparent:
|
if transparent:
|
||||||
token_map = {tokens[i]: raw_tokens[i] for i in range(len(tokens))}
|
token_map = {tokens[i]: raw_tokens[i] for i in range(len(tokens))}
|
||||||
log: Log = []
|
log = []
|
||||||
|
|
||||||
numbered_probs: Dict[int, float] = {}
|
numbered_probs: Dict[int, float] = {}
|
||||||
|
|
||||||
|
@ -180,13 +71,7 @@ class Model:
|
||||||
)
|
)
|
||||||
|
|
||||||
if transparent:
|
if transparent:
|
||||||
log.append(
|
log.append([token_map[word], weight, unweighted_numbers])
|
||||||
(
|
|
||||||
token_map[word],
|
|
||||||
weight,
|
|
||||||
unweighted_numbers,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
for category, value in enumerate(weighted_numbers):
|
for category, value in enumerate(weighted_numbers):
|
||||||
try:
|
try:
|
||||||
|
@ -203,10 +88,25 @@ class Model:
|
||||||
}
|
}
|
||||||
|
|
||||||
if transparent:
|
if transparent:
|
||||||
explanation = convert_log(log, self.names)
|
explanation = {}
|
||||||
return TransparentConfidences(probs, explanation)
|
for word, weight, word_probs in log:
|
||||||
|
if word in explanation:
|
||||||
|
explanation[word]["count"] += 1
|
||||||
|
else:
|
||||||
|
explanation[word] = {
|
||||||
|
"weight": weight,
|
||||||
|
"probabilities": {
|
||||||
|
name: word_probs[index]
|
||||||
|
for index, name in enumerate(self.names)
|
||||||
|
},
|
||||||
|
"count": 1,
|
||||||
|
}
|
||||||
|
|
||||||
return Confidences(probs)
|
return TransparentConfidences(
|
||||||
|
probs, explanation, self, text, max_ngram_length
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return Confidences(probs, self, text, max_ngram_length)
|
||||||
|
|
||||||
def get(self, token: str) -> Dict[str, float]:
|
def get(self, token: str) -> Dict[str, float]:
|
||||||
try:
|
try:
|
||||||
|
@ -240,83 +140,67 @@ class Model:
|
||||||
+ b"".join([weight.to_bytes(2, "big") for weight in weights])
|
+ b"".join([weight.to_bytes(2, "big") for weight in weights])
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def compile(
|
|
||||||
raw_model: Iterable[Mapping[str, str]],
|
|
||||||
max_ngram_length: int = 1,
|
|
||||||
min_count: int = 1,
|
|
||||||
hash_algorithm: str = "sha256",
|
|
||||||
) -> 'Model':
|
|
||||||
"""Compile a raw model.
|
|
||||||
|
|
||||||
Parameters
|
class Confidences(collections.UserDict):
|
||||||
----------
|
def __init__(self, probs, model, text, max_ngram_length):
|
||||||
raw_model : list of dict
|
collections.UserDict.__init__(self, probs)
|
||||||
A raw GPTC model.
|
self.model = model
|
||||||
|
self.text = text
|
||||||
|
self.max_ngram_length = max_ngram_length
|
||||||
|
|
||||||
max_ngram_length : int
|
|
||||||
Maximum ngram lenght to compile with.
|
|
||||||
|
|
||||||
Returns
|
class TransparentConfidences(Confidences):
|
||||||
-------
|
def __init__(self, probs, explanation, model, text, max_ngram_length):
|
||||||
dict
|
Confidences.__init__(self, probs, model, text, max_ngram_length)
|
||||||
A compiled GPTC model.
|
self.explanation = explanation
|
||||||
|
|
||||||
"""
|
|
||||||
word_counts, category_lengths, names = _count_words(
|
|
||||||
raw_model, max_ngram_length, hash_algorithm
|
|
||||||
)
|
|
||||||
model = _get_weights(min_count, word_counts, category_lengths, names)
|
|
||||||
return Model(model, names, max_ngram_length, hash_algorithm)
|
|
||||||
|
|
||||||
@staticmethod
|
def deserialize(encoded_model: BinaryIO) -> Model:
|
||||||
def deserialize(encoded_model: BinaryIO) -> "Model":
|
prefix = encoded_model.read(14)
|
||||||
prefix = encoded_model.read(14)
|
if prefix != b"GPTC model v6\n":
|
||||||
if prefix != b"GPTC model v6\n":
|
raise InvalidModelError()
|
||||||
|
|
||||||
|
config_json = b""
|
||||||
|
while True:
|
||||||
|
byte = encoded_model.read(1)
|
||||||
|
if byte == b"\n":
|
||||||
|
break
|
||||||
|
elif byte == b"":
|
||||||
raise InvalidModelError()
|
raise InvalidModelError()
|
||||||
|
else:
|
||||||
config_json = b""
|
|
||||||
while True:
|
|
||||||
byte = encoded_model.read(1)
|
|
||||||
if byte == b"\n":
|
|
||||||
break
|
|
||||||
|
|
||||||
if byte == b"":
|
|
||||||
raise InvalidModelError()
|
|
||||||
|
|
||||||
config_json += byte
|
config_json += byte
|
||||||
|
|
||||||
try:
|
try:
|
||||||
config = json.loads(config_json.decode("utf-8"))
|
config = json.loads(config_json.decode("utf-8"))
|
||||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||||
raise InvalidModelError() from exc
|
raise InvalidModelError()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
names = config["names"]
|
names = config["names"]
|
||||||
max_ngram_length = config["max_ngram_length"]
|
max_ngram_length = config["max_ngram_length"]
|
||||||
hash_algorithm = config["hash_algorithm"]
|
hash_algorithm = config["hash_algorithm"]
|
||||||
except KeyError as exc:
|
except KeyError:
|
||||||
raise InvalidModelError() from exc
|
raise InvalidModelError()
|
||||||
|
|
||||||
if not (
|
if not (
|
||||||
isinstance(names, list) and isinstance(max_ngram_length, int)
|
isinstance(names, list) and isinstance(max_ngram_length, int)
|
||||||
) or not all(isinstance(name, str) for name in names):
|
) or not all([isinstance(name, str) for name in names]):
|
||||||
|
raise InvalidModelError()
|
||||||
|
|
||||||
|
weight_code_length = 6 + 2 * len(names)
|
||||||
|
|
||||||
|
weights: Dict[int, List[int]] = {}
|
||||||
|
|
||||||
|
while True:
|
||||||
|
code = encoded_model.read(weight_code_length)
|
||||||
|
if not code:
|
||||||
|
break
|
||||||
|
elif len(code) != weight_code_length:
|
||||||
raise InvalidModelError()
|
raise InvalidModelError()
|
||||||
|
|
||||||
weight_code_length = 6 + 2 * len(names)
|
weights[int.from_bytes(code[:6], "big")] = [
|
||||||
|
int.from_bytes(value, "big")
|
||||||
|
for value in [code[x : x + 2] for x in range(6, len(code), 2)]
|
||||||
|
]
|
||||||
|
|
||||||
weights: Dict[int, List[int]] = {}
|
return Model(weights, names, max_ngram_length, hash_algorithm)
|
||||||
|
|
||||||
while True:
|
|
||||||
code = encoded_model.read(weight_code_length)
|
|
||||||
if not code:
|
|
||||||
break
|
|
||||||
if len(code) != weight_code_length:
|
|
||||||
raise InvalidModelError()
|
|
||||||
|
|
||||||
weights[int.from_bytes(code[:6], "big")] = [
|
|
||||||
int.from_bytes(value, "big")
|
|
||||||
for value in [code[x : x + 2] for x in range(6, len(code), 2)]
|
|
||||||
]
|
|
||||||
|
|
||||||
return Model(weights, names, max_ngram_length, hash_algorithm)
|
|
||||||
|
|
20
gptc/pack.py
20
gptc/pack.py
|
@ -7,7 +7,7 @@ from typing import List, Dict, Tuple
|
||||||
|
|
||||||
def pack(
|
def pack(
|
||||||
directory: str, print_exceptions: bool = False
|
directory: str, print_exceptions: bool = False
|
||||||
) -> Tuple[List[Dict[str, str]], List[Tuple[OSError]]]:
|
) -> Tuple[List[Dict[str, str]], List[Tuple[Exception]]]:
|
||||||
paths = os.listdir(directory)
|
paths = os.listdir(directory)
|
||||||
texts: Dict[str, List[str]] = {}
|
texts: Dict[str, List[str]] = {}
|
||||||
exceptions = []
|
exceptions = []
|
||||||
|
@ -17,18 +17,16 @@ def pack(
|
||||||
try:
|
try:
|
||||||
for file in os.listdir(os.path.join(directory, path)):
|
for file in os.listdir(os.path.join(directory, path)):
|
||||||
try:
|
try:
|
||||||
with open(
|
with open(os.path.join(directory, path, file)) as f:
|
||||||
os.path.join(directory, path, file), encoding="utf-8"
|
texts[path].append(f.read())
|
||||||
) as input_file:
|
except Exception as e:
|
||||||
texts[path].append(input_file.read())
|
exceptions.append((e,))
|
||||||
except OSError as error:
|
|
||||||
exceptions.append((error,))
|
|
||||||
if print_exceptions:
|
if print_exceptions:
|
||||||
print(error, file=sys.stderr)
|
print(e, file=sys.stderr)
|
||||||
except OSError as error:
|
except Exception as e:
|
||||||
exceptions.append((error,))
|
exceptions.append((e,))
|
||||||
if print_exceptions:
|
if print_exceptions:
|
||||||
print(error, file=sys.stderr)
|
print(e, file=sys.stderr)
|
||||||
|
|
||||||
raw_model = []
|
raw_model = []
|
||||||
|
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
import unicodedata
|
from typing import List, Union, Callable, Any, cast
|
||||||
from typing import List, cast
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import emoji
|
import emoji
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
|
||||||
def tokenize(text: str, max_ngram_length: int = 1) -> List[str]:
|
def tokenize(text: str, max_ngram_length: int = 1) -> List[str]:
|
||||||
|
@ -37,12 +37,12 @@ def tokenize(text: str, max_ngram_length: int = 1) -> List[str]:
|
||||||
|
|
||||||
if max_ngram_length == 1:
|
if max_ngram_length == 1:
|
||||||
return tokens
|
return tokens
|
||||||
|
else:
|
||||||
ngrams = []
|
ngrams = []
|
||||||
for ngram_length in range(1, max_ngram_length + 1):
|
for ngram_length in range(1, max_ngram_length + 1):
|
||||||
for index in range(len(tokens) + 1 - ngram_length):
|
for index in range(len(tokens) + 1 - ngram_length):
|
||||||
ngrams.append(" ".join(tokens[index : index + ngram_length]))
|
ngrams.append(" ".join(tokens[index : index + ngram_length]))
|
||||||
return ngrams
|
return ngrams
|
||||||
|
|
||||||
|
|
||||||
def _hash_single(token: str, hash_function: type) -> int:
|
def _hash_single(token: str, hash_function: type) -> int:
|
||||||
|
@ -69,15 +69,15 @@ def _get_hash_function(hash_algorithm: str) -> type:
|
||||||
"sha3_384",
|
"sha3_384",
|
||||||
}:
|
}:
|
||||||
return cast(type, getattr(hashlib, hash_algorithm))
|
return cast(type, getattr(hashlib, hash_algorithm))
|
||||||
|
else:
|
||||||
raise ValueError("not a valid hash function: " + hash_algorithm)
|
raise ValueError("not a valid hash function: " + hash_algorithm)
|
||||||
|
|
||||||
|
|
||||||
def hash_single(token: str, hash_algorithm: str) -> int:
|
def hash_single(token: str, hash_algorithm: str) -> int:
|
||||||
return _hash_single(token, _get_hash_function(hash_algorithm))
|
return _hash_single(token, _get_hash_function(hash_algorithm))
|
||||||
|
|
||||||
|
|
||||||
def hash_list(tokens: List[str], hash_algorithm: str) -> List[int]:
|
def hash(tokens: List[str], hash_algorithm: str) -> List[int]:
|
||||||
hash_function = _get_hash_function(hash_algorithm)
|
hash_function = _get_hash_function(hash_algorithm)
|
||||||
return [_hash_single(token, hash_function) for token in tokens]
|
return [_hash_single(token, hash_function) for token in tokens]
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
import math
|
import math
|
||||||
from typing import Sequence, Tuple, List
|
from typing import Sequence, Union, Tuple, List
|
||||||
|
|
||||||
|
|
||||||
def _mean(numbers: Sequence[float]) -> float:
|
def _mean(numbers: Sequence[float]) -> float:
|
||||||
|
@ -39,8 +39,8 @@ def _standard_deviation(numbers: Sequence[float]) -> float:
|
||||||
return math.sqrt(_mean(squared_deviations))
|
return math.sqrt(_mean(squared_deviations))
|
||||||
|
|
||||||
|
|
||||||
def weight(numbers: Sequence[float]) -> Tuple[float, List[float]]:
|
def weight(numbers: Sequence[float]) -> List[float]:
|
||||||
standard_deviation = _standard_deviation(numbers)
|
standard_deviation = _standard_deviation(numbers)
|
||||||
weight_assigned = standard_deviation * 2
|
weight = standard_deviation * 2
|
||||||
weighted_numbers = [i * weight_assigned for i in numbers]
|
weighted_numbers = [i * weight for i in numbers]
|
||||||
return weight_assigned, weighted_numbers
|
return weight, weighted_numbers
|
||||||
|
|
|
@ -13,4 +13,4 @@ with open("models/raw.json") as f:
|
||||||
with open("models/benchmark_text.txt") as f:
|
with open("models/benchmark_text.txt") as f:
|
||||||
text = f.read()
|
text = f.read()
|
||||||
|
|
||||||
cProfile.run("gptc.Model.compile(raw_model, max_ngram_length)")
|
cProfile.run("gptc.compile(raw_model, max_ngram_length)")
|
||||||
|
|
Loading…
Reference in New Issue
Block a user