88 lines
2.4 KiB
Python
88 lines
2.4 KiB
Python
|
import feedparser
|
||
|
|
import hashlib
|
||
|
|
import sqlite3
|
||
|
|
import goose3
|
||
|
|
import tomli
|
||
|
|
import gptc
|
||
|
|
import bs4
|
||
|
|
|
||
|
|
with open("sources.toml", "rb") as f:
|
||
|
|
sources = tomli.load(f)
|
||
|
|
|
||
|
|
g = goose3.Goose()
|
||
|
|
|
||
|
|
con = sqlite3.connect("articles.db")
|
||
|
|
con.execute("CREATE TABLE IF NOT EXISTS articles(source, category, url, text);")
|
||
|
|
|
||
|
|
|
||
|
|
known = [i[0] for i in con.execute("SELECT url FROM articles;")]
|
||
|
|
|
||
|
|
try:
|
||
|
|
for (url,) in con.execute("SELECT url FROM articles WHERE (text = '***');"):
|
||
|
|
print(f"Downloading {url}...")
|
||
|
|
con.execute(
|
||
|
|
"UPDATE articles SET text = ? WHERE (url = ?);",
|
||
|
|
(
|
||
|
|
g.extract(url).cleaned_text,
|
||
|
|
url,
|
||
|
|
),
|
||
|
|
)
|
||
|
|
print("Done downloading.")
|
||
|
|
|
||
|
|
for name, config in sources.items():
|
||
|
|
url = config["feed"]
|
||
|
|
category = config["category"]
|
||
|
|
contains_articles = config["contains_articles"]
|
||
|
|
|
||
|
|
print(f"Fetching {name} feed ({url})...")
|
||
|
|
entries = [
|
||
|
|
entry
|
||
|
|
for entry in feedparser.parse(url)["entries"]
|
||
|
|
if not entry["link"] in known
|
||
|
|
]
|
||
|
|
print(f"Fetched feed. Found {len(entries)} new articles.")
|
||
|
|
|
||
|
|
if contains_articles:
|
||
|
|
for entry in entries:
|
||
|
|
print(f"Saving {entry['link']}")
|
||
|
|
con.execute(
|
||
|
|
"INSERT INTO articles VALUES (?, ?, ?, ?);",
|
||
|
|
(
|
||
|
|
name,
|
||
|
|
category,
|
||
|
|
entry["link"],
|
||
|
|
bs4.BeautifulSoup(
|
||
|
|
entry["content"][0]["value"], features="lxml"
|
||
|
|
).text,
|
||
|
|
),
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
for entry in entries:
|
||
|
|
print(f"Downloading {entry['link']}...")
|
||
|
|
con.execute(
|
||
|
|
"INSERT INTO articles VALUES (?, ?, ?, ?);",
|
||
|
|
(
|
||
|
|
name,
|
||
|
|
category,
|
||
|
|
entry["link"],
|
||
|
|
g.extract(entry["link"]).cleaned_text,
|
||
|
|
),
|
||
|
|
)
|
||
|
|
print(f"Done downloading.")
|
||
|
|
finally:
|
||
|
|
con.commit()
|
||
|
|
|
||
|
|
print("Compiling model...")
|
||
|
|
|
||
|
|
raw_model = [
|
||
|
|
{"text": i[0], "category": i[1]}
|
||
|
|
for i in con.execute("SELECT text, category FROM articles;")
|
||
|
|
]
|
||
|
|
|
||
|
|
with open("model.gptc", "w+b") as f:
|
||
|
|
f.write(
|
||
|
|
gptc.compile(raw_model, max_ngram_length=3, min_count=3).serialize()
|
||
|
|
)
|
||
|
|
|
||
|
|
con.close()
|