Add min_count

This makes future changes to the algorithm much simpler.

LGPL-3.0

@@ -1,165 +0,0 @@
-                   GNU LESSER GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-
-  This version of the GNU Lesser General Public License incorporates
-the terms and conditions of version 3 of the GNU General Public
-License, supplemented by the additional permissions listed below.
-
-  0. Additional Definitions.
-
-  As used herein, "this License" refers to version 3 of the GNU Lesser
-General Public License, and the "GNU GPL" refers to version 3 of the GNU
-General Public License.
-
-  "The Library" refers to a covered work governed by this License,
-other than an Application or a Combined Work as defined below.
-
-  An "Application" is any work that makes use of an interface provided
-by the Library, but which is not otherwise based on the Library.
-Defining a subclass of a class defined by the Library is deemed a mode
-of using an interface provided by the Library.
-
-  A "Combined Work" is a work produced by combining or linking an
-Application with the Library.  The particular version of the Library
-with which the Combined Work was made is also called the "Linked
-Version".
-
-  The "Minimal Corresponding Source" for a Combined Work means the
-Corresponding Source for the Combined Work, excluding any source code
-for portions of the Combined Work that, considered in isolation, are
-based on the Application, and not on the Linked Version.
-
-  The "Corresponding Application Code" for a Combined Work means the
-object code and/or source code for the Application, including any data
-and utility programs needed for reproducing the Combined Work from the
-Application, but excluding the System Libraries of the Combined Work.
-
-  1. Exception to Section 3 of the GNU GPL.
-
-  You may convey a covered work under sections 3 and 4 of this License
-without being bound by section 3 of the GNU GPL.
-
-  2. Conveying Modified Versions.
-
-  If you modify a copy of the Library, and, in your modifications, a
-facility refers to a function or data to be supplied by an Application
-that uses the facility (other than as an argument passed when the
-facility is invoked), then you may convey a copy of the modified
-version:
-
-   a) under this License, provided that you make a good faith effort to
-   ensure that, in the event an Application does not supply the
-   function or data, the facility still operates, and performs
-   whatever part of its purpose remains meaningful, or
-
-   b) under the GNU GPL, with none of the additional permissions of
-   this License applicable to that copy.
-
-  3. Object Code Incorporating Material from Library Header Files.
-
-  The object code form of an Application may incorporate material from
-a header file that is part of the Library.  You may convey such object
-code under terms of your choice, provided that, if the incorporated
-material is not limited to numerical parameters, data structure
-layouts and accessors, or small macros, inline functions and templates
-(ten or fewer lines in length), you do both of the following:
-
-   a) Give prominent notice with each copy of the object code that the
-   Library is used in it and that the Library and its use are
-   covered by this License.
-
-   b) Accompany the object code with a copy of the GNU GPL and this license
-   document.
-
-  4. Combined Works.
-
-  You may convey a Combined Work under terms of your choice that,
-taken together, effectively do not restrict modification of the
-portions of the Library contained in the Combined Work and reverse
-engineering for debugging such modifications, if you also do each of
-the following:
-
-   a) Give prominent notice with each copy of the Combined Work that
-   the Library is used in it and that the Library and its use are
-   covered by this License.
-
-   b) Accompany the Combined Work with a copy of the GNU GPL and this license
-   document.
-
-   c) For a Combined Work that displays copyright notices during
-   execution, include the copyright notice for the Library among
-   these notices, as well as a reference directing the user to the
-   copies of the GNU GPL and this license document.
-
-   d) Do one of the following:
-
-       0) Convey the Minimal Corresponding Source under the terms of this
-       License, and the Corresponding Application Code in a form
-       suitable for, and under terms that permit, the user to
-       recombine or relink the Application with a modified version of
-       the Linked Version to produce a modified Combined Work, in the
-       manner specified by section 6 of the GNU GPL for conveying
-       Corresponding Source.
-
-       1) Use a suitable shared library mechanism for linking with the
-       Library.  A suitable mechanism is one that (a) uses at run time
-       a copy of the Library already present on the user's computer
-       system, and (b) will operate properly with a modified version
-       of the Library that is interface-compatible with the Linked
-       Version.
-
-   e) Provide Installation Information, but only if you would otherwise
-   be required to provide such information under section 6 of the
-   GNU GPL, and only to the extent that such information is
-   necessary to install and execute a modified version of the
-   Combined Work produced by recombining or relinking the
-   Application with a modified version of the Linked Version. (If
-   you use option 4d0, the Installation Information must accompany
-   the Minimal Corresponding Source and Corresponding Application
-   Code. If you use option 4d1, you must provide the Installation
-   Information in the manner specified by section 6 of the GNU GPL
-   for conveying Corresponding Source.)
-
-  5. Combined Libraries.
-
-  You may place library facilities that are a work based on the
-Library side by side in a single library together with other library
-facilities that are not Applications and are not covered by this
-License, and convey such a combined library under terms of your
-choice, if you do both of the following:
-
-   a) Accompany the combined library with a copy of the same work based
-   on the Library, uncombined with any other library facilities,
-   conveyed under the terms of this License.
-
-   b) Give prominent notice with the combined library that part of it
-   is a work based on the Library, and explaining where to find the
-   accompanying uncombined form of the same work.
-
-  6. Revised Versions of the GNU Lesser General Public License.
-
-  The Free Software Foundation may publish revised and/or new versions
-of the GNU Lesser General Public License from time to time. Such new
-versions will be similar in spirit to the present version, but may
-differ in detail to address new problems or concerns.
-
-  Each version is given a distinguishing version number. If the
-Library as you received it specifies that a certain numbered version
-of the GNU Lesser General Public License "or any later version"
-applies to it, you have the option of following the terms and
-conditions either of that published version or of any later version
-published by the Free Software Foundation. If the Library as you
-received it does not specify a version number of the GNU Lesser
-General Public License, you may choose any version of the GNU Lesser
-General Public License ever published by the Free Software Foundation.
-
-  If the Library as you received it specifies that a proxy can decide
-whether future versions of the GNU Lesser General Public License shall
-apply, that proxy's public statement of acceptance of any version is
-permanent authorization for you to choose that version for the
-Library.

LICENSE

@@ -1,14 +1,13 @@
 Copyright (c) 2020-2022 Samuel L Sloniker
 
 This program is free software: you can redistribute it and/or modify it under
-the terms of the GNU Lesser General Public License as published by the Free
-Software Foundation, either version 3 of the License, or (at your option) any
-later version.
+the terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later
+version.
 
 This program is distributed in the hope that it will be useful, but WITHOUT ANY
 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 PARTICULAR PURPOSE. See the GNU General Public License for more details.
 
-You should have received copies of the GNU General Public License and the GNU
-Lesser General Public License along with this program. If not, see
-<https://www.gnu.org/licenses/>. 
+You should have received a copy of the GNU General Public License along with
+this program. If not, see <https://www.gnu.org/licenses/>. 

README.md

@@ -29,10 +29,13 @@ stdout (or "None" if it cannot determine anything).
 
 ### Compiling models
 
-    gptc compile [-n <max_ngram_length>] <raw model file>
+    gptc compile [-n <max_ngram_length>] [-c <min_count>] <raw model file>
 
 This will print the compiled model in JSON to stdout.
 
+If `-c` is specified, words and ngrams used less than `min_count` times will be
+excluded from the compiled model.
+
 ### Packing models
 
     gptc pack <dir>
@@ -68,13 +71,16 @@ The classifier's model.
 Check whether emojis are supported by the `Classifier`. (See section "Emoji.")
 Equivalent to `gptc.has_emoji and gptc.model_has_emoji(model)`.
 
-### `gptc.compile(raw_model, max_ngram_length=1)`
+### `gptc.compile(raw_model, max_ngram_length=1, min_count=1)`
 
 Compile a raw model (as a list, not JSON) and return the compiled model (as a
 dict).
 
 For information about `max_ngram_length`, see section "Ngrams."
 
+Words or ngrams used less than `min_count` times throughout the input text are
+excluded from the model.
+
 ### `gptc.pack(directory, print_exceptions=False)
 
 Pack the model in `directory` and return a tuple of the format:
@@ -164,11 +170,3 @@ It uses the default Twain/Shakespeare model for both, and for classification it
 uses [Mark Antony's "Friends, Romans, countrymen"
 speech](https://en.wikipedia.org/wiki/Friends,_Romans,_countrymen,_lend_me_your_ears)
 from Shakespeare's *Julius Caesar*.
-
-## Please only use GPTC in copylefted (e.g. GPL'ed) software
-
-GPTC is licensed under the Lesser General Public License; this does allow use 
-in non-GPL'ed software. However, this license is only used for compatibility
-with [CedarSentinel](https://github.com/fire219/CedarSentinel), released under
-the MIT license. While this request is not legally binding, I would appreciate
-it if GPTC was not used in non-copyleft software.

benchmark.py

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
 import timeit
 import gptc
 import json

gptc/__init__.py

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: LGPL-3.0-or-later
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 """General-Purpose Text Classifier"""
 

gptc/__main__.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# SPDX-License-Identifier: LGPL-3.0-or-later
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import argparse
 import json
@@ -24,6 +24,13 @@ def main() -> None:
         type=int,
         default=1,
     )
+    compile_parser.add_argument(
+        "--min-count",
+        "-c",
+        help="minimum use count for word/ngram to be included in model",
+        type=int,
+        default=1,
+    )
 
     classify_parser = subparsers.add_parser("classify", help="classify text")
     classify_parser.add_argument("model", help="compiled model to use")
@@ -59,7 +66,7 @@ def main() -> None:
         with open(args.model, "r") as f:
             model = json.load(f)
 
-        print(json.dumps(gptc.compile(model, args.max_ngram_length)))
+        print(json.dumps(gptc.compile(model, args.max_ngram_length, args.min_count)))
     elif args.subparser_name == "classify":
         with open(args.model, "r") as f:
             model = json.load(f)

gptc/classifier.py

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: LGPL-3.0-or-later
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import gptc.tokenizer, gptc.compiler, gptc.exceptions, gptc.weighting, gptc.model_info
 import warnings

gptc/compiler.py

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: LGPL-3.0-or-later
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import gptc.tokenizer
 from typing import Iterable, Mapping, List, Dict, Union
@@ -9,7 +9,9 @@ MODEL = Dict[str, Union[WEIGHTS_T, CONFIG_T]]
 
 
 def compile(
-    raw_model: Iterable[Mapping[str, str]], max_ngram_length: int = 1
+    raw_model: Iterable[Mapping[str, str]],
+    max_ngram_length: int = 1,
+    min_count: int = 1,
 ) -> MODEL:
     """Compile a raw model.
 
@@ -38,7 +40,7 @@ def compile(
         except KeyError:
             categories[category] = text
 
-    categories_by_count: Dict[str, Dict[str, float]] = {}
+    word_counts: Dict[str, Dict[str, float]] = {}
 
     names = []
 
@@ -46,23 +48,33 @@ def compile(
         if not category in names:
             names.append(category)
 
-        categories_by_count[category] = {}
         for word in text:
             try:
-                categories_by_count[category][word] += 1 / len(
-                    categories[category]
-                )
+                counts_for_word = word_counts[word]
             except KeyError:
-                categories_by_count[category][word] = 1 / len(
-                    categories[category]
-                )
-    word_weights: Dict[str, Dict[str, float]] = {}
-    for category, words in categories_by_count.items():
-        for word, value in words.items():
+                counts_for_word = {}
+                word_counts[word] = counts_for_word
+
             try:
-                word_weights[word][category] = value
+                word_counts[word][category] += 1
             except KeyError:
-                word_weights[word] = {category: value}
+                word_counts[word][category] = 1
+
+    word_counts = {
+        word: counts
+        for word, counts in word_counts.items()
+        if sum(counts.values()) >= min_count
+    }
+
+    word_weights: Dict[str, Dict[str, float]] = {}
+    for word, values in word_counts.items():
+        for category, value in values.items():
+            try:
+                word_weights[word][category] = value / len(categories[category])
+            except KeyError:
+                word_weights[word] = {
+                    category: value / len(categories[category])
+                }
 
     model: MODEL = {}
     for word, weights in word_weights.items():

gptc/exceptions.py

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: LGPL-3.0-or-later
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 
 class GPTCError(BaseException):

gptc/model_info.py

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: LGPL-3.0-or-later
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import gptc.compiler
 from typing import Dict, Union, cast, List

gptc/pack.py

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: LGPL-3.0-or-later
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import sys
 import os

gptc/tokenizer.py

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: LGPL-3.0-or-later
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 from typing import List, Union
 

gptc/weighting.py

@@ -1,4 +1,4 @@
-# SPDX-License-Identifier: LGPL-3.0-or-later
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import math
 from typing import Sequence, Union, Tuple, List