Add Slate

README.md

@@ -25,7 +25,9 @@ Inclusion of a site in this model is not an endorsement of the site.
 * CNN
 * HuffPost (formerly Huffington Post)
 * Los Angeles Times
+* Slate
 * PBS NewsHour
+* The Washington Post
 
 ### Right
 

compile.py

@@ -14,11 +14,15 @@
 # details.
 #
 # You should have received a copy of the GNU General Public License along with
-# this program. If not, see <https://www.gnu.org/licenses/>. 
+# this program. If not, see <https://www.gnu.org/licenses/>.
 
 import sqlite3
+import tomli
 import gptc
 
+with open("compiler.toml", "rb") as f:
+    config = tomli.load(f)
+
 con = sqlite3.connect("articles.db")
 con.execute("CREATE TABLE IF NOT EXISTS articles(source, category, url, text);")
 
@@ -29,7 +33,11 @@ raw_model = [
 
 with open("model.gptc", "w+b") as f:
     f.write(
-        gptc.compile(raw_model, max_ngram_length=3, min_count=3).serialize()
+        gptc.compile(
+            raw_model,
+            max_ngram_length=config["max_ngram_length"],
+            min_count=config["min_count"],
+        ).serialize()
     )
 
 con.commit()

compiler.toml

@@ -0,0 +1,2 @@
+max_ngram_length=3
+min_count=5

sources.toml

@@ -5,6 +5,31 @@
 # ? Newsmax (read timeout errors)
 # ? Bloomberg (CAPTCHA on RSS feed?)
 
+[abc_news]
+feed="https://abcnews.go.com/abcnews/usheadlines"
+category="left"
+contains_articles=false
+name="ABC News"
+
+[atlantic]
+feed="https://www.theatlantic.com/feed/all/"
+category="left"
+contains_articles=true
+name="The Atlantic"
+sort="Atlantic"
+
+[cbs_news]
+feed="https://www.cbsnews.com/latest/rss/main"
+category="left"
+contains_articles=false
+name="CBS News"
+
+[cnbc]
+feed="https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114"
+category="left"
+contains_articles=false
+name="CNBC"
+
 [cnn]
 feed="http://rss.cnn.com/rss/cnn_latest.rss"
 category="left"
@@ -17,43 +42,6 @@ category="left"
 contains_articles=false
 name="HuffPost"
 
-[cnbc]
-feed="https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114"
-category="left"
-contains_articles=false
-name="CNBC"
-
-[pbs_newshour]
-feed="https://www.pbs.org/newshour/feeds/rss/headlines"
-category="left"
-contains_articles=false
-name="PBS NewsHour"
-
-[latimes]
-feed="https://www.latimes.com/local/rss2.0.xml"
-category="left"
-contains_articles=false
-name="Los Angeles Times"
-
-[cbs_news]
-feed="https://www.cbsnews.com/latest/rss/main"
-category="left"
-contains_articles=false
-name="CBS News"
-
-[atlantic]
-feed="https://www.theatlantic.com/feed/all/"
-category="left"
-contains_articles=true
-name="The Atlantic"
-sort="Atlantic"
-
-[abc_news]
-feed="https://abcnews.go.com/abcnews/usheadlines"
-category="left"
-contains_articles=false
-name="ABC News"
-
 [intercept]
 feed="https://theintercept.com/feed/?lang=en"
 category="left"
@@ -61,45 +49,42 @@ contains_articles=true
 name="The Intercept"
 sort="Intercept"
 
+[latimes]
+feed="https://www.latimes.com/local/rss2.0.xml"
+category="left"
+contains_articles=false
+name="Los Angeles Times"
+
+[pbs_newshour]
+feed="https://www.pbs.org/newshour/feeds/rss/headlines"
+category="left"
+contains_articles=false
+name="PBS NewsHour"
+
+[slate]
+feed="http://www.slate.com/articles/news_and_politics.fulltext.all.10.rss"
+category="left"
+contains_articles=false
+name="Slate"
+
+[washington_post]
+feed="https://feeds.washingtonpost.com/rss/national"
+category="left"
+contains_articles=false
+name="The Washington Post"
+sort="Washington Post"
+
 #[bloomberg]
 #feed="https://www.bloomberg.com/politics/feeds/site.xml"
 #category="left"
 #contains_articles=false
 
-[fox_news]
-feed="https://moxie.foxnews.com/google-publisher/latest.xml"
+[american_conservative]
+feed="https://theamericanconservative.com/articles/feed/"
 category="right"
 contains_articles=true
-name="Fox News"
-
-[oann]
-feed="https://www.oann.com/category/newsroom/feed"
-category="right"
-contains_articles=true
-name="One America News Network"
-
-[nypost]
-feed="https://nypost.com/feed"
-category="right"
-contains_articles=false
-exclude=[
-    { type="startswith", pattern="https://pagesix.com" },
-    { type="startswith", pattern="https://decider.com" },
-]
-name="New York Post"
-
-[federalist]
-feed="https://thefederalist.com/feed"
-category="right"
-contains_articles=false
-name="The Federalist"
-sort="Federalist"
-
-[washington_examiner]
-feed="https://feeds.feedburner.com/dcexaminer/Politics"
-category="right"
-contains_articles=true
-name="Washington Examiner"
+name="The American Conservative"
+sort="American Conservative"
 
 [american_thinker]
 feed="https://feeds.feedburner.com/americanthinker_articles"
@@ -113,30 +98,54 @@ category="right"
 contains_articles=false
 name="Breitbart"
 
+[daily_caller]
+feed="https://feeds.feedburner.com/dailycaller"
+category="right"
+contains_articles=false
+name="Daily Caller"
+
 [epoch_times]
 feed="https://www.theepochtimes.com/feed/"
 category="right"
 contains_articles=false
 name="Epoch Times"
 
+[federalist]
+feed="https://thefederalist.com/feed"
+category="right"
+contains_articles=false
+name="The Federalist"
+sort="Federalist"
+
+[fox_news]
+feed="https://moxie.foxnews.com/google-publisher/latest.xml"
+category="right"
+contains_articles=true
+name="Fox News"
+
 [not_the_bee]
 feed="https://notthebee.com/feed"
 category="right"
 contains_articles=false
 name="Not the Bee"
 
-[daily_caller]
-feed="https://feeds.feedburner.com/dailycaller"
+[nypost]
+feed="https://nypost.com/news/feed"
 category="right"
 contains_articles=false
-name="Daily Caller"
+name="New York Post"
 
-[american_conservative]
-feed="https://theamericanconservative.com/articles/feed/"
+[oann]
+feed="https://www.oann.com/category/newsroom/feed"
 category="right"
 contains_articles=true
-name="The American Conservative"
-sort="American Conservative"
+name="One America News Network"
+
+[washington_examiner]
+feed="https://feeds.feedburner.com/dcexaminer/Politics"
+category="right"
+contains_articles=true
+name="Washington Examiner"
 
 #[newsmax]
 #feed="https://www.newsmax.com/rss/Newsfront/16/"