gptc-news-model/compile.py

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later

# Copyright (c) 2022 Samuel L Sloniker
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program. If not, see <https://www.gnu.org/licenses/>. 

import sqlite3
import gptc

con = sqlite3.connect("articles.db")
con.execute("CREATE TABLE IF NOT EXISTS articles(source, category, url, text);")

raw_model = [
    {"text": i[0], "category": i[1]}
    for i in con.execute("SELECT text, category FROM articles;")
]

with open("model.gptc", "w+b") as f:
    f.write(
        gptc.compile(raw_model, max_ngram_length=3, min_count=5).serialize()
    )

con.commit()
con.close()