.gitignore

@@ -139,5 +139,4 @@ cython_debug/
 
 # Model
 *.db
-*.db-journal
 *.gptc

README.md

@@ -9,7 +9,6 @@ Inclusion of a site in this model is not an endorsement of the site.
 
 ### Left
 
-* ABC News
 * The Atlantic
 * CBS News
 * CNBC
@@ -22,7 +21,6 @@ Inclusion of a site in this model is not an endorsement of the site.
 
 * American Thinker
 * Breitbart
-* Daily Caller
 * Epoch Times
 * The Federalist
 * Fox News

compile.py

@@ -1,18 +0,0 @@
-import sqlite3
-import gptc
-
-con = sqlite3.connect("articles.db")
-con.execute("CREATE TABLE IF NOT EXISTS articles(source, category, url, text);")
-
-raw_model = [
-    {"text": i[0], "category": i[1]}
-    for i in con.execute("SELECT text, category FROM articles;")
-]
-
-with open("model.gptc", "w+b") as f:
-    f.write(
-        gptc.compile(raw_model, max_ngram_length=3, min_count=3).serialize()
-    )
-
-con.commit()
-con.close()

download.py

@@ -1,7 +1,9 @@
 import feedparser
+import hashlib
 import sqlite3
 import goose3
 import tomli
+import gptc
 import bs4
 
 
@@ -83,4 +85,17 @@ try:
                     print("Not enough information. Skipping.")
 finally:
     con.commit()
+
+    print("Compiling model...")
+
+    raw_model = [
+        {"text": i[0], "category": i[1]}
+        for i in con.execute("SELECT text, category FROM articles;")
+    ]
+
+    with open("model.gptc", "w+b") as f:
+        f.write(
+            gptc.compile(raw_model, max_ngram_length=3, min_count=3).serialize()
+        )
+
     con.close()

sources.toml

@@ -40,11 +40,6 @@ feed="https://www.theatlantic.com/feed/all/"
 category="left"
 contains_articles=true
 
-[abc_news]
-feed="https://abcnews.go.com/abcnews/usheadlines"
-category="left"
-contains_articles=false
-
 #[bloomberg]
 #feed="https://www.bloomberg.com/politics/feeds/site.xml"
 #category="left"
@@ -99,11 +94,6 @@ feed="https://notthebee.com/feed"
 category="right"
 contains_articles=false
 
-[daily_caller]
-feed="https://feeds.feedburner.com/dailycaller"
-category="right"
-contains_articles=false
-
 #[newsmax]
 #feed="https://www.newsmax.com/rss/Newsfront/16/"
 #category="right"

stats.sh

@@ -7,11 +7,7 @@ right=$(sqlite3 articles.db "SELECT url FROM articles WHERE category = 'right'"
 left_sources=$(sqlite3 articles.db "SELECT source FROM articles WHERE category = 'left'" | sort | uniq)
 right_sources=$(sqlite3 articles.db "SELECT source FROM articles WHERE category = 'right'" | sort | uniq)
 
-total_source_count=$(sqlite3 articles.db "SELECT source FROM articles" | sort | uniq | wc -l)
+echo "This model contains a total of $total articles ($left left, $right right)."
-left_source_count=$(echo $left_sources | wc -w)
-right_source_count=$(echo $right_sources | wc -w)
-
-echo "This model contains a total of $total articles from $total_source_count sources."
 echo ""
 echo "## Left"
 echo ""
@@ -19,12 +15,8 @@ for i in $left_sources; do
     echo "* $i: $(sqlite3 articles.db "SELECT url FROM articles WHERE source = '$i'" | wc -l)"
 done
 echo ""
-echo "Left total: $left articles from $left_source_count sources"
-echo ""
 echo "## Right"
 echo ""
 for i in $right_sources; do
     echo "* $i: $(sqlite3 articles.db "SELECT url FROM articles WHERE source = '$i'" | wc -l)"
 done
-echo ""
-echo "Right total: $right articles from $right_source_count sources"