Add emoji checks, improve docs
This commit is contained in:
parent
73b800d60d
commit
185692790f
43
README.md
43
README.md
|
@ -33,6 +33,13 @@ stdout (or "None" if it cannot determine anything).
|
|||
|
||||
This will print the compiled model in JSON to stdout.
|
||||
|
||||
### Packing models
|
||||
|
||||
gptc pack <dir>
|
||||
|
||||
This will print the raw model in JSON to stdout. See `models/unpacked/` for an
|
||||
example of the format. Any exceptions will be printed to stderr.
|
||||
|
||||
## Library
|
||||
|
||||
### `gptc.Classifier(model, max_ngram_length=1)`
|
||||
|
@ -52,12 +59,44 @@ category:probability, ...}`
|
|||
Classify `text`. Returns the category into which the text is placed (as a
|
||||
string), or `None` when it cannot classify the text.
|
||||
|
||||
#### `Classifier.model`
|
||||
|
||||
The classifier's model.
|
||||
|
||||
#### `Classifier.has_emoji`
|
||||
|
||||
Check whether emojis are supported by the `Classifier`. (See section "Emoji.")
|
||||
Equivalent to `gptc.has_emoji and gptc.model_has_emoji(model)`.
|
||||
|
||||
### `gptc.compile(raw_model, max_ngram_length=1)`
|
||||
|
||||
Compile a raw model (as a list, not JSON) and return the compiled model (as a
|
||||
dict).
|
||||
|
||||
For information about `max_ngram_length`, see section "Ngrams."
|
||||
|
||||
### `gptc.pack(directory, print_exceptions=False)
|
||||
|
||||
Pack the model in `directory` and return a tuple of the format:
|
||||
|
||||
(raw_model, [(exception,),(exception,)...])
|
||||
|
||||
Note that the exceptions are contained in single-item tuples. This is to allow
|
||||
more information to be provided without breaking the API in future versions of
|
||||
GPTC.
|
||||
|
||||
See `models/unpacked/` for an example of the format.
|
||||
|
||||
### `gptc.has_emoji`
|
||||
|
||||
`True` if the `emoji` package is installed (see section "Emoji"), `False`
|
||||
otherwise.
|
||||
|
||||
### `gptc.model_has_emoji(compiled_model)`
|
||||
|
||||
Returns `True` if `compiled_model` was compiled with emoji support, `False`
|
||||
otherwise.
|
||||
|
||||
## Ngrams
|
||||
|
||||
GPTC optionally supports using ngrams to improve classification accuracy. They
|
||||
|
@ -84,6 +123,10 @@ If the [`emoji`](https://pypi.org/project/emoji/) package is installed, GPTC
|
|||
will automatically handle emojis the same way as words. If it is not installed,
|
||||
GPTC will still work but will ignore emojis.
|
||||
|
||||
`emoji` must be installed on both the system used to compile the model and the
|
||||
system used to classify text. Emojis are ignored if it is missing on either
|
||||
system.
|
||||
|
||||
## Model format
|
||||
|
||||
This section explains the raw model format, which is how you should create and
|
||||
|
|
|
@ -5,6 +5,8 @@
|
|||
from gptc.compiler import compile as compile
|
||||
from gptc.classifier import Classifier as Classifier
|
||||
from gptc.pack import pack as pack
|
||||
from gptc.tokenizer import has_emoji as has_emoji
|
||||
from gptc.model_info import model_has_emoji as model_has_emoji
|
||||
from gptc.exceptions import (
|
||||
GPTCError as GPTCError,
|
||||
ModelError as ModelError,
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
# SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
import gptc.tokenizer, gptc.compiler, gptc.exceptions, gptc.weighting
|
||||
import gptc.tokenizer, gptc.compiler, gptc.exceptions, gptc.weighting, gptc.model_info
|
||||
import warnings
|
||||
from typing import Dict, Union, cast, List
|
||||
|
||||
|
@ -33,6 +33,7 @@ class Classifier:
|
|||
self.model = model
|
||||
model_ngrams = cast(int, model.get("__ngrams__", 1))
|
||||
self.max_ngram_length = min(max_ngram_length, model_ngrams)
|
||||
self.has_emoji = gptc.tokenizer.has_emoji and gptc.model_info.model_has_emoji(model)
|
||||
|
||||
def confidence(self, text: str) -> Dict[str, float]:
|
||||
"""Classify text with confidence.
|
||||
|
|
8
gptc/model_info.py
Executable file
8
gptc/model_info.py
Executable file
|
@ -0,0 +1,8 @@
|
|||
# SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
import gptc.compiler
|
||||
from typing import Dict, Union, cast, List
|
||||
|
||||
|
||||
def model_has_emoji(model: gptc.compiler.MODEL) -> bool:
|
||||
return cast(int, model.get("__emoji__]", 0)) == 1
|
Loading…
Reference in New Issue
Block a user