README.md

@@ -48,7 +48,7 @@ example of the format. Any exceptions will be printed to stderr.
 
 Write binary data representing the model to `file`.
 
-### `Model.deserialize(encoded_model)`
+### `gptc.deserialize(encoded_model)`
 
 Deserialize a `Model` from a file containing data from `Model.serialize()`.
 
@@ -70,7 +70,7 @@ Return a confidence dict for the given token or ngram. This function is very
 similar to `Model.confidence()`, except it treats the input as a single token
 or ngram.
 
-### `Model.compile(raw_model, max_ngram_length=1, min_count=1, hash_algorithm="sha256")`
+### `gptc.compile(raw_model, max_ngram_length=1, min_count=1, hash_algorithm="sha256")`
 
 Compile a raw model (as a list, not JSON) and return the compiled model (as a
 `gptc.Model` object).

benchmark.py

@@ -25,7 +25,7 @@ print(
     round(
         1000000
         * timeit.timeit(
-            "gptc.Model.compile(raw_model, max_ngram_length)",
+            "gptc.compile(raw_model, max_ngram_length)",
             number=compile_iterations,
             globals=globals(),
         )

gptc/__init__.py

@@ -2,11 +2,12 @@
 
 """General-Purpose Text Classifier"""
 
-from gptc.pack import pack
-from gptc.model import Model
-from gptc.tokenizer import normalize
+from gptc.compiler import compile as compile
+from gptc.pack import pack as pack
+from gptc.model import Model as Model, deserialize as deserialize
+from gptc.tokenizer import normalize as normalize
 from gptc.exceptions import (
-    GPTCError,
-    ModelError,
-    InvalidModelError,
+    GPTCError as GPTCError,
+    ModelError as ModelError,
+    InvalidModelError as InvalidModelError,
 )

gptc/__main__.py

@@ -59,16 +59,16 @@ def main() -> None:
     args = parser.parse_args()
 
     if args.subparser_name == "compile":
-        with open(args.model, "r", encoding="utf-8") as input_file:
-            model = json.load(input_file)
+        with open(args.model, "r") as f:
+            model = json.load(f)
 
-        with open(args.out, "wb+") as output_file:
-            gptc.Model.compile(
+        with open(args.out, "wb+") as f:
+            gptc.compile(
                 model, args.max_ngram_length, args.min_count
-            ).serialize(output_file)
+            ).serialize(f)
     elif args.subparser_name == "classify":
-        with open(args.model, "rb") as model_file:
-            model = gptc.Model.deserialize(model_file)
+        with open(args.model, "rb") as f:
+            model = gptc.deserialize(f)
 
         if sys.stdin.isatty():
             text = input("Text to analyse: ")
@@ -77,8 +77,8 @@ def main() -> None:
 
         print(json.dumps(model.confidence(text, args.max_ngram_length)))
     elif args.subparser_name == "check":
-        with open(args.model, "rb") as model_file:
-            model = gptc.Model.deserialize(model_file)
+        with open(args.model, "rb") as f:
+            model = gptc.deserialize(f)
         print(json.dumps(model.get(args.token)))
     else:
         print(json.dumps(gptc.pack(args.model, True)[0]))

gptc/compiler.py

@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import gptc.tokenizer
+import gptc.model
+from typing import Iterable, Mapping, List, Dict, Union, Tuple
+
+
+def _count_words(
+    raw_model: Iterable[Mapping[str, str]],
+    max_ngram_length: int,
+    hash_algorithm: str,
+) -> Tuple[Dict[int, Dict[str, int]], Dict[str, int], List[str]]:
+    word_counts: Dict[int, Dict[str, int]] = {}
+    category_lengths: Dict[str, int] = {}
+    names: List[str] = []
+
+    for portion in raw_model:
+        text = gptc.tokenizer.hash(
+            gptc.tokenizer.tokenize(portion["text"], max_ngram_length),
+            hash_algorithm,
+        )
+        category = portion["category"]
+
+        if not category in names:
+            names.append(category)
+
+        category_lengths[category] = category_lengths.get(category, 0) + len(
+            text
+        )
+
+        for word in text:
+            if word in word_counts:
+                try:
+                    word_counts[word][category] += 1
+                except KeyError:
+                    word_counts[word][category] = 1
+            else:
+                word_counts[word] = {category: 1}
+
+    return word_counts, category_lengths, names
+
+
+def _get_weights(
+    min_count: int,
+    word_counts: Dict[int, Dict[str, int]],
+    category_lengths: Dict[str, int],
+    names: List[str],
+) -> Dict[int, List[int]]:
+    model: Dict[int, List[int]] = {}
+    for word, counts in word_counts.items():
+        if sum(counts.values()) >= min_count:
+            weights = {
+                category: value / category_lengths[category]
+                for category, value in counts.items()
+            }
+            total = sum(weights.values())
+            new_weights: List[int] = []
+            for category in names:
+                new_weights.append(
+                    round((weights.get(category, 0) / total) * 65535)
+                )
+            model[word] = new_weights
+    return model
+
+
+def compile(
+    raw_model: Iterable[Mapping[str, str]],
+    max_ngram_length: int = 1,
+    min_count: int = 1,
+    hash_algorithm: str = "sha256",
+) -> gptc.model.Model:
+    """Compile a raw model.
+
+    Parameters
+    ----------
+    raw_model : list of dict
+        A raw GPTC model.
+
+    max_ngram_length : int
+        Maximum ngram lenght to compile with.
+
+    Returns
+    -------
+    dict
+        A compiled GPTC model.
+
+    """
+    word_counts, category_lengths, names = _count_words(
+        raw_model, max_ngram_length, hash_algorithm
+    )
+    model = _get_weights(min_count, word_counts, category_lengths, names)
+    return gptc.model.Model(model, names, max_ngram_length, hash_algorithm)

gptc/model.py

@@ -1,120 +1,11 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-from typing import (
-        Iterable,
-        Mapping,
-    List,
-    Dict,
-    cast,
-    BinaryIO,
-    Tuple,
-    TypedDict,
-)
-import json
 import gptc.tokenizer
 from gptc.exceptions import InvalidModelError
 import gptc.weighting
-
-def _count_words(
-    raw_model: Iterable[Mapping[str, str]],
-    max_ngram_length: int,
-    hash_algorithm: str,
-) -> Tuple[Dict[int, Dict[str, int]], Dict[str, int], List[str]]:
-    word_counts: Dict[int, Dict[str, int]] = {}
-    category_lengths: Dict[str, int] = {}
-    names: List[str] = []
-
-    for portion in raw_model:
-        text = gptc.tokenizer.hash_list(
-            gptc.tokenizer.tokenize(portion["text"], max_ngram_length),
-            hash_algorithm,
-        )
-        category = portion["category"]
-
-        if not category in names:
-            names.append(category)
-
-        category_lengths[category] = category_lengths.get(category, 0) + len(
-            text
-        )
-
-        for word in text:
-            if word in word_counts:
-                try:
-                    word_counts[word][category] += 1
-                except KeyError:
-                    word_counts[word][category] = 1
-            else:
-                word_counts[word] = {category: 1}
-
-    return word_counts, category_lengths, names
-
-
-def _get_weights(
-    min_count: int,
-    word_counts: Dict[int, Dict[str, int]],
-    category_lengths: Dict[str, int],
-    names: List[str],
-) -> Dict[int, List[int]]:
-    model: Dict[int, List[int]] = {}
-    for word, counts in word_counts.items():
-        if sum(counts.values()) >= min_count:
-            weights = {
-                category: value / category_lengths[category]
-                for category, value in counts.items()
-            }
-            total = sum(weights.values())
-            new_weights: List[int] = []
-            for category in names:
-                new_weights.append(
-                    round((weights.get(category, 0) / total) * 65535)
-                )
-            model[word] = new_weights
-    return model
-
-class ExplanationEntry(TypedDict):
-    weight: float
-    probabilities: Dict[str, float]
-    count: int
-
-
-Explanation = Dict[
-    str,
-    ExplanationEntry,
-]
-
-Log = List[Tuple[str, float, List[float]]]
-
-
-class Confidences(dict[str, float]):
-    def __init__(self, probs: Dict[str, float]):
-        dict.__init__(self, probs)
-
-
-class TransparentConfidences(Confidences):
-    def __init__(
-        self,
-        probs: Dict[str, float],
-        explanation: Explanation,
-    ):
-        self.explanation = explanation
-        Confidences.__init__(self, probs)
-
-
-def convert_log(log: Log, names: List[str]) -> Explanation:
-    explanation: Explanation = {}
-    for word2, weight, word_probs in log:
-        if word2 in explanation:
-            explanation[word2]["count"] += 1
-        else:
-            explanation[word2] = {
-                "weight": weight,
-                "probabilities": {
-                    name: word_probs[index] for index, name in enumerate(names)
-                },
-                "count": 1,
-            }
-    return explanation
+from typing import Iterable, Mapping, List, Dict, Union, cast, BinaryIO
+import json
+import collections
 
 
 class Model:
@@ -132,7 +23,7 @@ class Model:
 
     def confidence(
         self, text: str, max_ngram_length: int, transparent: bool = False
-    ) -> Confidences:
+    ) -> Dict[str, float]:
         """Classify text with confidence.
 
         Parameters
@@ -158,14 +49,14 @@ class Model:
             text, min(max_ngram_length, self.max_ngram_length)
         )
 
-        tokens = gptc.tokenizer.hash_list(
+        tokens = gptc.tokenizer.hash(
             raw_tokens,
             self.hash_algorithm,
         )
 
         if transparent:
             token_map = {tokens[i]: raw_tokens[i] for i in range(len(tokens))}
-            log: Log = []
+            log = []
 
         numbered_probs: Dict[int, float] = {}
 
@@ -180,13 +71,7 @@ class Model:
                 )
 
                 if transparent:
-                    log.append(
-                        (
-                            token_map[word],
-                            weight,
-                            unweighted_numbers,
-                        )
-                    )
+                    log.append([token_map[word], weight, unweighted_numbers])
 
                 for category, value in enumerate(weighted_numbers):
                     try:
@@ -203,10 +88,25 @@ class Model:
         }
 
         if transparent:
-            explanation = convert_log(log, self.names)
-            return TransparentConfidences(probs, explanation)
+            explanation = {}
+            for word, weight, word_probs in log:
+                if word in explanation:
+                    explanation[word]["count"] += 1
+                else:
+                    explanation[word] = {
+                        "weight": weight,
+                        "probabilities": {
+                            name: word_probs[index]
+                            for index, name in enumerate(self.names)
+                        },
+                        "count": 1,
+                    }
 
-        return Confidences(probs)
+            return TransparentConfidences(
+                probs, explanation, self, text, max_ngram_length
+            )
+        else:
+            return Confidences(probs, self, text, max_ngram_length)
 
     def get(self, token: str) -> Dict[str, float]:
         try:
@@ -240,83 +140,67 @@ class Model:
                 + b"".join([weight.to_bytes(2, "big") for weight in weights])
             )
 
-    @staticmethod
-    def compile(
-        raw_model: Iterable[Mapping[str, str]],
-        max_ngram_length: int = 1,
-        min_count: int = 1,
-        hash_algorithm: str = "sha256",
-        ) -> 'Model':
-        """Compile a raw model.
 
-        Parameters
-        ----------
-        raw_model : list of dict
-            A raw GPTC model.
+class Confidences(collections.UserDict):
+    def __init__(self, probs, model, text, max_ngram_length):
+        collections.UserDict.__init__(self, probs)
+        self.model = model
+        self.text = text
+        self.max_ngram_length = max_ngram_length
 
-        max_ngram_length : int
-            Maximum ngram lenght to compile with.
 
-        Returns
-        -------
-        dict
-            A compiled GPTC model.
+class TransparentConfidences(Confidences):
+    def __init__(self, probs, explanation, model, text, max_ngram_length):
+        Confidences.__init__(self, probs, model, text, max_ngram_length)
+        self.explanation = explanation
 
-        """
-        word_counts, category_lengths, names = _count_words(
-            raw_model, max_ngram_length, hash_algorithm
-        )
-        model = _get_weights(min_count, word_counts, category_lengths, names)
-        return Model(model, names, max_ngram_length, hash_algorithm)
 
-    @staticmethod
-    def deserialize(encoded_model: BinaryIO) -> "Model":
-        prefix = encoded_model.read(14)
-        if prefix != b"GPTC model v6\n":
+def deserialize(encoded_model: BinaryIO) -> Model:
+    prefix = encoded_model.read(14)
+    if prefix != b"GPTC model v6\n":
+        raise InvalidModelError()
+
+    config_json = b""
+    while True:
+        byte = encoded_model.read(1)
+        if byte == b"\n":
+            break
+        elif byte == b"":
             raise InvalidModelError()
-
-        config_json = b""
-        while True:
-            byte = encoded_model.read(1)
-            if byte == b"\n":
-                break
-
-            if byte == b"":
-                raise InvalidModelError()
-
+        else:
             config_json += byte
 
-        try:
-            config = json.loads(config_json.decode("utf-8"))
-        except (UnicodeDecodeError, json.JSONDecodeError) as exc:
-            raise InvalidModelError() from exc
+    try:
+        config = json.loads(config_json.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError):
+        raise InvalidModelError()
 
-        try:
-            names = config["names"]
-            max_ngram_length = config["max_ngram_length"]
-            hash_algorithm = config["hash_algorithm"]
-        except KeyError as exc:
-            raise InvalidModelError() from exc
+    try:
+        names = config["names"]
+        max_ngram_length = config["max_ngram_length"]
+        hash_algorithm = config["hash_algorithm"]
+    except KeyError:
+        raise InvalidModelError()
 
-        if not (
-            isinstance(names, list) and isinstance(max_ngram_length, int)
-        ) or not all(isinstance(name, str) for name in names):
+    if not (
+        isinstance(names, list) and isinstance(max_ngram_length, int)
+    ) or not all([isinstance(name, str) for name in names]):
+        raise InvalidModelError()
+
+    weight_code_length = 6 + 2 * len(names)
+
+    weights: Dict[int, List[int]] = {}
+
+    while True:
+        code = encoded_model.read(weight_code_length)
+        if not code:
+            break
+        elif len(code) != weight_code_length:
             raise InvalidModelError()
 
-        weight_code_length = 6 + 2 * len(names)
+        weights[int.from_bytes(code[:6], "big")] = [
+            int.from_bytes(value, "big")
+            for value in [code[x : x + 2] for x in range(6, len(code), 2)]
+        ]
 
-        weights: Dict[int, List[int]] = {}
-
-        while True:
-            code = encoded_model.read(weight_code_length)
-            if not code:
-                break
-            if len(code) != weight_code_length:
-                raise InvalidModelError()
-
-            weights[int.from_bytes(code[:6], "big")] = [
-                int.from_bytes(value, "big")
-                for value in [code[x : x + 2] for x in range(6, len(code), 2)]
-            ]
-
-        return Model(weights, names, max_ngram_length, hash_algorithm)
+    return Model(weights, names, max_ngram_length, hash_algorithm)

gptc/pack.py

@@ -7,7 +7,7 @@ from typing import List, Dict, Tuple
 
 def pack(
     directory: str, print_exceptions: bool = False
-) -> Tuple[List[Dict[str, str]], List[Tuple[OSError]]]:
+) -> Tuple[List[Dict[str, str]], List[Tuple[Exception]]]:
     paths = os.listdir(directory)
     texts: Dict[str, List[str]] = {}
     exceptions = []
@@ -17,18 +17,16 @@ def pack(
         try:
             for file in os.listdir(os.path.join(directory, path)):
                 try:
-                    with open(
-                        os.path.join(directory, path, file), encoding="utf-8"
-                    ) as input_file:
-                        texts[path].append(input_file.read())
-                except OSError as error:
-                    exceptions.append((error,))
+                    with open(os.path.join(directory, path, file)) as f:
+                        texts[path].append(f.read())
+                except Exception as e:
+                    exceptions.append((e,))
                     if print_exceptions:
-                        print(error, file=sys.stderr)
-        except OSError as error:
-            exceptions.append((error,))
+                        print(e, file=sys.stderr)
+        except Exception as e:
+            exceptions.append((e,))
             if print_exceptions:
-                print(error, file=sys.stderr)
+                print(e, file=sys.stderr)
 
     raw_model = []
 

gptc/tokenizer.py

@@ -1,9 +1,9 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-import unicodedata
-from typing import List, cast
+from typing import List, Union, Callable, Any, cast
 import hashlib
 import emoji
+import unicodedata
 
 
 def tokenize(text: str, max_ngram_length: int = 1) -> List[str]:
@@ -37,12 +37,12 @@ def tokenize(text: str, max_ngram_length: int = 1) -> List[str]:
 
     if max_ngram_length == 1:
         return tokens
-
-    ngrams = []
-    for ngram_length in range(1, max_ngram_length + 1):
-        for index in range(len(tokens) + 1 - ngram_length):
-            ngrams.append(" ".join(tokens[index : index + ngram_length]))
-    return ngrams
+    else:
+        ngrams = []
+        for ngram_length in range(1, max_ngram_length + 1):
+            for index in range(len(tokens) + 1 - ngram_length):
+                ngrams.append(" ".join(tokens[index : index + ngram_length]))
+        return ngrams
 
 
 def _hash_single(token: str, hash_function: type) -> int:
@@ -69,15 +69,15 @@ def _get_hash_function(hash_algorithm: str) -> type:
         "sha3_384",
     }:
         return cast(type, getattr(hashlib, hash_algorithm))
-
-    raise ValueError("not a valid hash function: " + hash_algorithm)
+    else:
+        raise ValueError("not a valid hash function: " + hash_algorithm)
 
 
 def hash_single(token: str, hash_algorithm: str) -> int:
     return _hash_single(token, _get_hash_function(hash_algorithm))
 
 
-def hash_list(tokens: List[str], hash_algorithm: str) -> List[int]:
+def hash(tokens: List[str], hash_algorithm: str) -> List[int]:
     hash_function = _get_hash_function(hash_algorithm)
     return [_hash_single(token, hash_function) for token in tokens]
 

gptc/weighting.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import math
-from typing import Sequence, Tuple, List
+from typing import Sequence, Union, Tuple, List
 
 
 def _mean(numbers: Sequence[float]) -> float:
@@ -39,8 +39,8 @@ def _standard_deviation(numbers: Sequence[float]) -> float:
     return math.sqrt(_mean(squared_deviations))
 
 
-def weight(numbers: Sequence[float]) -> Tuple[float, List[float]]:
+def weight(numbers: Sequence[float]) -> List[float]:
     standard_deviation = _standard_deviation(numbers)
-    weight_assigned = standard_deviation * 2
-    weighted_numbers = [i * weight_assigned for i in numbers]
-    return weight_assigned, weighted_numbers
+    weight = standard_deviation * 2
+    weighted_numbers = [i * weight for i in numbers]
+    return weight, weighted_numbers

profiler.py

@@ -13,4 +13,4 @@ with open("models/raw.json") as f:
 with open("models/benchmark_text.txt") as f:
     text = f.read()
 
-cProfile.run("gptc.Model.compile(raw_model, max_ngram_length)")
+cProfile.run("gptc.compile(raw_model, max_ngram_length)")