|
|
|
@ -14,11 +14,15 @@
|
|
|
|
|
# details. |
|
|
|
|
# |
|
|
|
|
# You should have received a copy of the GNU General Public License along with |
|
|
|
|
# this program. If not, see <https://www.gnu.org/licenses/>. |
|
|
|
|
# this program. If not, see <https://www.gnu.org/licenses/>. |
|
|
|
|
|
|
|
|
|
import sqlite3 |
|
|
|
|
import tomli |
|
|
|
|
import gptc |
|
|
|
|
|
|
|
|
|
with open("compiler.toml", "rb") as f: |
|
|
|
|
config = tomli.load(f) |
|
|
|
|
|
|
|
|
|
con = sqlite3.connect("articles.db") |
|
|
|
|
con.execute("CREATE TABLE IF NOT EXISTS articles(source, category, url, text);") |
|
|
|
|
|
|
|
|
@ -29,7 +33,11 @@ raw_model = [
|
|
|
|
|
|
|
|
|
|
with open("model.gptc", "w+b") as f: |
|
|
|
|
f.write( |
|
|
|
|
gptc.compile(raw_model, max_ngram_length=3, min_count=5).serialize() |
|
|
|
|
gptc.compile( |
|
|
|
|
raw_model, |
|
|
|
|
max_ngram_length=config["max_ngram_length"], |
|
|
|
|
min_count=config["min_count"], |
|
|
|
|
).serialize() |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
con.commit() |
|
|
|
|