summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore1
-rw-r--r--requirements.txt2
-rwxr-xr-xunbiased/main.py82
-rwxr-xr-xunbiased/parser.py986
-rw-r--r--unbiased/sources/__init__.py10
-rw-r--r--unbiased/sources/abc.py42
-rw-r--r--unbiased/sources/base.py210
-rw-r--r--unbiased/sources/bbc.py26
-rw-r--r--unbiased/sources/cbs.py37
-rw-r--r--unbiased/sources/csm.py41
-rw-r--r--unbiased/sources/fox.py41
-rw-r--r--unbiased/sources/guardian.py37
-rw-r--r--unbiased/sources/npr.py29
-rw-r--r--unbiased/sources/thehill.py39
-rw-r--r--unbiased/sources/washtimes.py31
-rwxr-xr-xunbiased/spotCheck.py41
-rw-r--r--unbiased/unbiasedFunctions.py277
-rw-r--r--unbiased/unbiasedObjects.py97
-rw-r--r--unbiased/util.py113
19 files changed, 694 insertions, 1448 deletions
diff --git a/.gitignore b/.gitignore
index 9e0f924..ad2b57a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@ venv/
unbiased.egg-info/
#*
.#*
+*.swp
diff --git a/requirements.txt b/requirements.txt
index 3767095..0d53cea 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
beautifulsoup4~=4.6.0
Jinja2~=2.9.6
-lxml=~=3.8.0
+lxml~=3.8.0
Pillow~=4.2.1
requests~=2.18.4
diff --git a/unbiased/main.py b/unbiased/main.py
index 4ccda24..19fd05b 100755
--- a/unbiased/main.py
+++ b/unbiased/main.py
@@ -1,13 +1,13 @@
#!/usr/bin/env python3
import argparse
+import io
import logging
import logging.config
import time
-from unbiased.unbiasedObjects import *
-from unbiased.unbiasedFunctions import *
-from unbiased.parser import *
+from unbiased.util import pickStories, pullImage, buildOutput, write_files, write_static_files
+from unbiased.sources import get_sources
logger = logging.getLogger('unbiased')
@@ -52,6 +52,7 @@ def main():
parser.add_argument('-l', '--log-dir', help='location to write detailed logs')
parser.add_argument('-d', '--debug', action='store_true', help='run in debug mode')
parser.add_argument('-o', '--oneshot', action='store_true', help='run once and exit')
+ parser.add_argument('-s', '--sources', type=lambda x: x.split(','), default=None)
args = parser.parse_args()
if args.log_dir:
@@ -67,7 +68,7 @@ def main():
while True:
logger.info('Starting crawl')
start = time.time()
- run(args.webroot)
+ run(args.webroot, args.sources, args.debug)
finish = time.time()
runtime = finish - start
sleeptime = crawl_frequency - runtime
@@ -77,51 +78,34 @@ def main():
if sleeptime > 0:
time.sleep(sleeptime)
-def run(webroot):
- sources = []
-
- '''
- SOURCES TO ADD NEXT:
- -REUTERS
- -Town Hall
- '''
-
- logger.debug('Running with webroot="{}"'.format(webroot))
-
- ### These values have to be the second half of the function name
- ### E.g. Guardian calls buildGuardian(), etc.
- sourceFnArr = [
- 'Guardian',
- 'TheHill',
- 'NPR',
- 'BBC',
- 'NBC',
- 'CBS',
- 'FoxNews',
- 'WashTimes',
- 'CSM',
- 'ABC',
- ]
-
- for source in sourceFnArr:
- logger.info('Crawling {}'.format(source))
+def run(webroot, source_names, debug_mode=False):
+
+ logger.debug('Running with webroot="{}" for sources="{}"'.format(webroot, source_names))
+
+ sources = get_sources()
+ if source_names is None:
+ sources = sources.values()
+ else:
+ sources = [sources[x] for x in source_names]
+
+ built_sources = []
+ for source in sources:
+ logger.info('Crawling {}'.format(source.name))
tries = 0
while tries < 3:
time.sleep(tries)
try:
- fn = 'build' + source
- possibles = globals().copy()
- possibles.update(locals())
- method = possibles.get(fn)
- src = method()
- sources.append(src)
+ built_sources.append(source.build())
break
except Exception as ex:
+ if debug_mode is True:
+ raise
tries += 1
if tries == 3:
- logger.error('Build failed. source={} ex={}'.format(source, ex))
+ logger.error('Build failed. source={} ex={}'.format(source.name, ex))
else:
- logger.debug('Build failed, retrying. source={} ex={}'.format(source, ex))
+ logger.debug('Build failed, retrying. source={} ex={}'.format(source.name, ex))
+ sources = tuple(built_sources)
logger.info('Parsed home pages for: {}'.format([x.name for x in sources]))
top_stories, middle_stories, bottom_stories = pickStories(sources)
@@ -129,20 +113,26 @@ def run(webroot):
logger.info('Picked middle stories from: {}'.format([x.source for x in middle_stories]))
logger.info('Picked bottom stories from: {}'.format([x.source for x in bottom_stories]))
+ files_to_write = {}
+
# download images
img_idx = 0
for story in top_stories:
- story.img = pullImage(story.img, img_idx, webroot, 350, 200)
+ story.img, img_jpg = pullImage(story.img, img_idx, webroot, 350, 200)
+ files_to_write[story.img] = img_jpg
img_idx += 1
for story in middle_stories:
- story.img = pullImage(story.img, img_idx, webroot, 150, 100)
+ story.img, img_jpg = pullImage(story.img, img_idx, webroot, 150, 100)
+ files_to_write[story.img] = img_jpg
img_idx += 1
- #build the output file HTML
- outputHTML = buildOutput(top_stories, middle_stories, bottom_stories)
+ # build the output file HTML
+ output_html = buildOutput(top_stories, middle_stories, bottom_stories)
+ output_html = io.BytesIO(output_html.encode('utf8'))
+ files_to_write['index.html'] = output_html
- #print the output file HTML
- writeOutputHTML(outputHTML, webroot)
+ write_files(files_to_write, webroot)
+ write_static_files(webroot)
if __name__=="__main__":
main()
diff --git a/unbiased/parser.py b/unbiased/parser.py
deleted file mode 100755
index 399e0f2..0000000
--- a/unbiased/parser.py
+++ /dev/null
@@ -1,986 +0,0 @@
-#!/usr/bin/env python3
-
-import logging
-import os
-import re
-import urllib.parse
-
-from bs4 import BeautifulSoup
-import requests
-
-from unbiased.unbiasedObjects import *
-from unbiased.unbiasedFunctions import buildArticle
-
-logger = logging.getLogger('unbiased')
-
-
-'''
-Takes in a URL, downloads the file to a temp file,
-reads the file into a string, and returns that string
-'''
-def urlToContent(url, sourceEncoding='utf8'):
- res = requests.get(url)
- if res.status_code == 200:
- return res.text
- else:
- raise Exception("Failed to download {}".format(url))
-
-
-'''
-Creates a new newsSource2 object. For each URL in h1-h3URLs,
-calls the file scraper and appends the new Article object.
-Returns a newsSource2 object
-'''
-def buildNewsSource2(name, url, h1URLs, h2URLs, h3URLs):
-
- url_parts = urllib.parse.urlparse(url)
- scheme = url_parts.scheme
- h1URLs = [urllib.parse.urlparse(x, scheme=scheme).geturl() for x in h1URLs]
- h2URLs = [urllib.parse.urlparse(x, scheme=scheme).geturl() for x in h2URLs]
- h3URLs = [urllib.parse.urlparse(x, scheme=scheme).geturl() for x in h3URLs]
-
- h1Arr=[]
- a=buildArticle(h1URLs[0], name)
- if a==None:
- logger.debug('H1 Nonetype in '+name)
- else:
- h1Arr.append(a)
-
- h2Arr=[]
- for x in h2URLs:
- a=buildArticle(x, name)
- if a!=None:
- h2Arr.append(a)
- else:
- logger.debug('H2 Nonetype in '+name)
-
- h3Arr=[]
- for x in h3URLs:
- a=buildArticle(x, name)
- if a!=None:
- h3Arr.append(a)
- else:
- logger.debug('H3 Nonetype in '+name)
-
- #BUILD THE NEWS SOURCE
- newsSource=NewsSource2(name, url, h1Arr, h2Arr, h3Arr)
-
- return newsSource
-
-
-'''
-Some sites will replicate URLs across the page. This function removes them.
-Check hierarchically: if h3 exists in h1s or h2s, remove from h3s;
-if h2 exists in h1s, remove from h2s
-
-also check partial URLs (e.g. nytimes.com/story.html is the same as
-nytimes.com/story.html?var=x
-'''
-def removeDuplicates(h1s, h2s, h3s):
- #Assume h1s is one element, and keep it
-
- #remove h2 duplicates
- removeArr=[]
- for i in range(len(h2s)):
- #check internally
- for j in range(len(h2s)):
- if i==j:
- continue
- else:
- if h2s[i] in h2s[j]:
- removeArr.append(h2s[j])
- #check against h1s
- for k in range(len(h1s)):
- if (h2s[i] in h1s[k]) or (h1s[k] in h2s[i]):
- removeArr.append(h2s[i])
- for x in removeArr:
- h2s.remove(x)
-
- #remove h3 duplicates
- removeArr=[]
- for i in range(len(h3s)):
- #check internally
- for j in range(len(h3s)):
- if i==j:
- continue
- else:
- if h3s[i] in h3s[j]:
- removeArr.append(h3s[j])
- #check against h1s and h2s
- h1and2=h1s+h2s
- for k in range(len(h1and2)):
- if (h3s[i] in h1and2[k]) or (h1and2[k] in h3s[i]):
- removeArr.append(h3s[i])
- for x in removeArr:
- if x in h3s:
- h3s.remove(x)
-
-
- return h1s, h2s, h3s
-
-
-
-def removalNotification(source, title, reason, value):
- logger.debug("""Story removed
- SOURCE:\t{}
- TITLE:\t{})
- REASON:\t{}
- VALUE:\t{}""".format(source, title, reason, value))
-
-
-def removeBadStoriesHelper(source, element, badStringList, article_tiers):
- if badStringList is None:
- return
- for tier, articles in enumerate(article_tiers):
- for idx, article in enumerate(articles):
- if article is None:
- logger.debug("None type found in removeBadStoriesHelper for {}".format(source.name))
- break
- for item in badStringList:
- if item in getattr(article, element):
- article_tiers[tier].remove(article)
- # if it's in the h1 slot, bump up the
- # first h2 into the h1 slot
- if tier == 0 and len(article_tiers[1]) > 0:
- article_tiers[0].append(article_tiers[1][0])
- article_tiers[1].remove(article_tiers[1][0])
- removalNotification(source.name, article.title, element, item)
-
-
-def removeBadStories(source, badTitleArr, badDescArr, badAuthorArr, badImgArr, badURLArr=None):
-
- arr=[source.h1Arr, source.h2Arr, source.h3Arr]
-
- removeBadStoriesHelper(source, "title", badTitleArr, arr)
- removeBadStoriesHelper(source, "description", badDescArr, arr)
- removeBadStoriesHelper(source, "author", badAuthorArr, arr)
- removeBadStoriesHelper(source, "img", badImgArr, arr)
- removeBadStoriesHelper(source, "url", badURLArr, arr)
-
- return source
-
-
-
-
-def buildTheHill():
- url='http://thehill.com'
- name='The Hill'
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #get main headline
- h1=content
- h1=h1.split('<div class="headline-story-image">', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- h1=h1.split('"', 1)[0]
- h1s=[url+h1]
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- h2=h2.split('<div class="section-top-content">', 1)[1]
- h2=h2.split('</ul>', 1)[0]
- while '<div class="top-story-item' in h2 and len(h2s)<4:
- h2=h2.split('<div class="top-story-item', 1)[1]
- x=h2.split('<a href="', 1)[1]
- x=x.split('"', 1)[0]
- h2s.append(url+x)
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('<div class="section-top-content">', 1)[1]
- h3=h3.split('</ul>', 1)[0]
- while '<div class="top-story-item small' in h3:
- h3=h3.split('<div class="top-story-item small', 1)[1]
- x=h3.split('<a href="', 1)[1]
- x=x.split('"', 1)[0]
- h3s.append(url+x)
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
- hil=buildNewsSource2(name, url, h1s, h2s, h3s)
- hil=removeBadStories(hil, ['THE MEMO'], None, ['Matt Schlapp', 'Juan Williams', 'Judd Gregg'], None, None)
-
- return hil
-
-
-
-
-
-def buildGuardian():
- url='http://www.theguardian.com/us'
- name='The Guardian US'
-
-
- while True:
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url, 'utf8')
-
- #get main headline
- h1=content
- h1=h1.split('<h1', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- h1=h1.split('"', 1)[0]
-
- if h1!='https://www.theguardian.com/us':
- break
- else:
- logger.debug('Guardian loop')
-
- h1s=[h1]
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- #only the h1 and the two h2s have this, so split on it and grab
- #the second two
- h2=h2.split('<div class="fc-item__image-container u-responsive-ratio inlined-image">')[2:]
- for x in h2:
- if '<h2 class="fc-item__title"><a href="' in x:
- x=x.split('<h2 class="fc-item__title"><a href="', 1)[1]
- x=x.split('"', 1)[0]
- h2s.append(x)
- else:
- break
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('<div class="fc-slice-wrapper">', 1)[1]
- h3=h3.split('<div class="fc-container__inner">', 1)[0]#'<div class="js-show-more-placeholder">', 1)[0]
- #this story section goes on forever; just grab the first 5
- while '<h2 class="fc-item__title"><a href="' in h3:
- h3=h3.split('<h2 class="fc-item__title"><a href="', 1)[1]
- x=h3.split('"', 1)[0]
- h3s.append(x)
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
-
- gdn=buildNewsSource2(name, url, h1s, h2s, h3s)
- gdn=removeBadStories(gdn, None, ['Tom McCarthy', 'Andy Hunter'], ['https://www.theguardian.com/profile/ben-jacobs'], None)
-
- return gdn
-
-
-
-def buildWashTimes():
- url='http://www.washingtontimes.com/'
- name='Washington Times'
-
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #get main headline
- h1=content
- h1=h1.split('top-news', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- h1=h1.split('"', 1)[0]
-
- h1s=[url+h1]
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- h2=h2.split('class="top-news', 1)[1]
- h2=h2.split('</article>', 1)[1] #end of top-news article
- h2=h2.split('<article ', 1)[0] #note the space; we want unclassed articles
- h2=h2.split('<article>')[1:]
-
- for x in h2:
- x=x.split('<a href="', 1)[1]
- x=x.split('"', 1)[0]
- h2s.append(url+x)
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('more-from desktop-only', 1)[1]
- h3=h3.split('</section>', 1)[0]
- h3=h3.split('<a href="')[1:]
-
- for x in h3:
- x=x.split('"', 1)[0]
- h3s.append(url+x)
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
-
- wat=buildNewsSource2(name, url, h1s, h2s, h3s)
- wat=removeBadStories(wat, None, None, None, None)
-
- return wat
-
-
-def buildCSM():
- url='http://www.csmonitor.com/USA'
- name='Christian Science Monitor'
-
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #this makes sure we don't get '/USA' in the URL twice
- url=url.split('/USA')[0]
-
- #get main headline
- h1=content
- h1=h1.split('block-0-0', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- h1=h1.split('"', 1)[0]
-
- h1s=[url+h1]
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- h2=h2.split('block-1-0', 1)[1]
- h2=h2.split('ui-section-middle', 1)[0]
- h2=h2.split('<h3 class="story_headline">')[1:]
-
- for x in h2:
- temp=x.split('<a href="', 2)[1:]
- x=temp[0]
- x=x.split('"', 1)[0]
- if x=='/csmlists/special/first-look':
- x=temp[1]
- x=x.split('"', 1)[0]
-
- h2s.append(url+x)
- #also add in the floating story on the left
- h2=content
- h2=h2.split('block-0-1', 1)[1]
- h2=h2.split('<h3 class="story_headline">')[1]
- h2=h2.split('<a href="', 2)[2]
- h2=h2.split('"', 1)[0]
- h2s.append(url+h2)
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('block-0-2', 1)[1]
- h3=h3.split('ui-section-top-right', 1)[0]
- h3=h3.split('<h3 class="story_headline')[1:]
-
- for x in h3:
- x=x.split('<a href="', 2)[-1]
- x=x.split('"', 1)[0]
- h3s.append(url+x)
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
-
- csm=buildNewsSource2(name, url, h1s, h2s, h3s)
-
- badTitleArr=['Change Agent']
- badDescArr=None
- badAuthorArr=None
- badImgArr=['csm_logo']
- badURLArr=['difference-maker']
- csm=removeBadStories(csm, badTitleArr, badDescArr, badAuthorArr, badImgArr, badURLArr)
-
- return csm
-
-
-
-'''
-Function to fix the oddly short og:descriptions provided
-in The Blaze articles by grabbing the first portion of the story instead
-'''
-def blazeFixDesc(articleArr):
- TAG_RE = re.compile(r'<[^>]+>')
- for i in range(len(articleArr)):
- desc=urlToContent(articleArr[i].url)
- desc=desc.split('<div class="entry-content article-styles">', 1)[1]
- desc=desc.split('<p>', 1)[1]
- desc=TAG_RE.sub('', desc)
- desc=desc.replace('\n', ' ')
- desc=desc[:144]
- articleArr[i].description=desc
-
- return articleArr
-
-
-
-def buildBlaze():
- url='http://theblaze.com'
- name='The Blaze'
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #get main headline
- h1=content
- h1=h1.split('<!-- home -->', 1)[1]
- h1=h1.split('<a class="gallery-link" href="', 1)[1]
- h1=h1.split('"', 1)[0]
- h1s=[url+h1]
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- h2=h2.split('<!-- home -->', 1)[1]
- h2=h2.split('<!-- loop-home -->', 1)[0]
- while '<a class="gallery-link" href="' in h2:#'</figure>\n\n<figure class="gallery-item">' in h2:
- h2=h2.split('<a class="gallery-link" href="', 1)[1]#'</figure>\n\n<figure class="gallery-item">', 1)[1]
- #h2=h2.split('href="', 1)[1]
- x=h2.split('"', 1)[0]
- if h1 not in x:
- h2s.append(url+x)
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('<!-- loop-home -->', 1)[1]
- #this story section goes on forever; just grab the first 5
- while len(h3s)<5:
- h3=h3.split('<a class="feed-link" href="', 1)[1]
- x=h3.split('"', 1)[0]
- if h1 not in x:
- h3s.append(url+x)
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
-
- blz=buildNewsSource2(name, url, h1s, h2s, h3s)
-
- badTitleArr=['Tucker Carlson', 'Mark Levin']
- badDescArr=['Lawrence Jones', 'Mike Slater']
- badAuthorArr=['Matt Walsh', 'Tomi Lahren', 'Dana Loesch', 'Mike Opelka', 'Chris Salcedo', 'Justin Haskins', 'Sara Gonzales', 'Doc Thompson', 'Glenn Beck']
- badImgArr=None
- badURLArr=None
- blz=removeBadStories(blz, badTitleArr, badDescArr, badAuthorArr, badImgArr, badURLArr)
-
-
- #The Blaze has dumb, short description fields, so we need to grab
- #the first x characters of actual article text instead
- blz.h1Arr=blazeFixDesc(blz.h1Arr)
- blz.h2Arr=blazeFixDesc(blz.h2Arr)
- blz.h3Arr=blazeFixDesc(blz.h3Arr)
-
- return blz
-
-
-
-def buildCBS():
- url='http://cbsnews.com'
- name='CBS News'
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #get main headline
- h1=content
- if '<h1 class="title">' in content:
- h1=h1.split('<h1 class="title">', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- h1=h1.split('"', 1)[0]
- h1s=[url+h1]
- else:
- #for cases where they lead with a video, pull the first h2 as h1
- h1=h1.split('Big News Area Side Assets', 1)[1]
- h1=h1.split('</ul></div>', 1)[0]
- h1=h1.split('<li data-tb-region-item>', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- x=h1.split('"', 1)[0]
- h1s=[url+x]
-
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- h2=h2.split('Big News Area Side Assets', 1)[1]
- h2=h2.split('</ul></div>', 1)[0]
- while '<li data-tb-region-item>' in h2:
- h2=h2.split('<li data-tb-region-item>', 1)[1]
- h2=h2.split('<a href="', 1)[1]
- x=h2.split('"', 1)[0]
- if h1 not in x:
- h2s.append(url+x)
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('Latest News', 1)[1]
- #this story section goes on forever; just grab the first 5
- while len(h3s)<5:
- h3=h3.split('<li class="item-full-lead"', 1)[1]
- h3=h3.split('<a href="', 1)[1]
- x=h3.split('"', 1)[0]
- if h1 not in x:
- h3s.append(url+x)
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
- cbs=buildNewsSource2(name, url, h1s, h2s, h3s)
- cbs=removeBadStories(cbs, ['60 Minutes'], ['60 Minutes'], None, None, ['whats-in-the-news-coverart'])
-
- return cbs
-
-
-
-
-
-def buildNBC():
- url='http://nbcnews.com'
- name='NBC News'
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #get main headline
- h1=content
- h1=h1.split('top-stories-section', 1)[1]
- h1=h1.split('panel_hero', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- h1=h1.split('"', 1)[0]
- if '.com' not in h1:
- h1=url+h1
- h1s=[h1]
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- h2=h2.split('ad-content ad-xs mobilebox1', 1)[1]
- h2=h2.split('taboola-native-top-stories-thumbnail', 1)[0]
- while '<div class="story-link' in h2:
- h2=h2.split('<div class="story-link', 1)[1]
- h2=h2.split('<a href="', 1)[1]
- x=h2.split('"', 1)[0]
- if h1 not in x:
- if '.com' not in x:
- x=url+x
- h2s.append(x)
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('js-more-topstories', 1)[1]
- h3=h3.split('<div class="panel-section', 1)[0]
- while '<div class="story-link' in h3:
- h3=h3.split('<div class="story-link', 1)[1]
- h3=h3.split('<a href="', 1)[1]
- x=h3.split('"', 1)[0]
- if h1 not in x:
- if '.com' not in x:
- x=url+x
- h3s.append(x)
-
- #adjust for today.com urls
- '''
- for arr in [h1s, h2s, h3s]:
- for i in range(len(arr)):
- if 'today.com' in arr[i]:
- arr[i]=arr[i].split('.com', 1)[1]
- '''
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
- nbc=buildNewsSource2(name, url, h1s, h2s, h3s)
- nbc=removeBadStories(nbc, None, ['First Read'], None, None, None)
-
-
- return nbc
-
-
-
-
-def buildBBC():
- url='http://www.bbc.com/news/world/us_and_canada'
- name='BBC US & Canada'
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #get main headline
- h1=content
- h1=h1.split('buzzard-item', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- h1=h1.split('"', 1)[0]
- h1s=['http://www.bbc.com'+h1]
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- h2=h2.split('<div class="pigeon">', 1)[1]
- h2=h2.split('<div id=', 1)[0]
- while 'top_stories#' in h2:
- h2=h2.split('top_stories#', 1)[1]
- h2=h2.split('<a href="', 1)[1]
- x=h2.split('"', 1)[0]
- if h1 not in x:
- h2s.append('http://www.bbc.com'+x)
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('<div class="macaw">', 1)[1]
- h3=h3.split('Watch/Listen', 1)[0]
- while '<div class="macaw-item' in h3:
- h3=h3.split('<div class="macaw-item', 1)[1]
- h3=h3.split('<a href="', 1)[1]
- x=h3.split('"', 1)[0]
- if h1 not in x:
- h3s.append('http://www.bbc.com'+x)
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
- bbc=buildNewsSource2(name, url, h1s, h2s, h3s)
- badTitleArr=None
- badDescArr=None
- badAuthorArr=None
- badImgArr=['bbc_news_logo.png']
- bbc=removeBadStories(bbc, badTitleArr, badDescArr, badAuthorArr, badImgArr)
-
-
- #REMOVE ' - BBC News' from headlines
- for i in range(len(bbc.h1Arr)):
- if ' - BBC News' in bbc.h1Arr[i].title:
- bbc.h1Arr[i].title=bbc.h1Arr[i].title.split(' - BBC News', 1)[0]
- for i in range(len(bbc.h2Arr)):
- if ' - BBC News' in bbc.h2Arr[i].title:
- bbc.h2Arr[i].title=bbc.h2Arr[i].title.split(' - BBC News', 1)[0]
- for i in range(len(bbc.h3Arr)):
- if ' - BBC News' in bbc.h3Arr[i].title:
- bbc.h3Arr[i].title=bbc.h3Arr[i].title.split(' - BBC News', 1)[0]
-
- return bbc
-
-
-
-def buildWeeklyStandard():
- url='http://www.weeklystandard.com'
- name='Weekly Standard'
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #get main headline
- h1=content
- h1=h1.split('<div id="region_1"', 1)[1]
- h1=h1.split('<div id="region_2"', 1)[0]
- h1=h1.split('<div class="lead-photo">', 1)[1]
- h1=h1.split('href="', 1)[1]
- h1=h1.split('"', 1)[0]
- h1s=[h1]
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- h2=h2.split('<div class="widget lead-story layout-3col-feature" data-count="2">', 1)[1]
- h2=h2.split('<div id="region_2"', 1)[0]
- while '<div class="lead-photo">' in h2:
- h2=h2.split('<div class="lead-photo">', 1)[1]
- h2=h2.split('href="', 1)[1]
- x=h2.split('"', 1)[0]
- if h1 not in x:
- h2s.append(x)
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('Today\'s Standard', 1)[1]
- h3=h3.split('<div id="region_3"', 1)[0]
- while '<div class="lead-photo">' in h3:
- h3=h3.split('<div class="lead-photo">', 1)[1]
- h3=h3.split('href="', 1)[1]
- x=h3.split('"', 1)[0]
- if h1 not in x:
- h3s.append(x)
-
- #Need to add URL prefix to all URLs
- for i in range(len(h1s)):
- h1s[i]=url+h1s[i]
- for i in range(len(h2s)):
- h2s[i]=url+h2s[i]
- for i in range(len(h3s)):
- h3s[i]=url+h3s[i]
-
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
- wkl=buildNewsSource2(name, url, h1s, h2s, h3s)
-
- #REMOVE BAD STORIES
- badTitleArr=None
- ## if flagged again, remove Micah Mattix
- badDescArr=['Matt Labash']
- badAuthorArr=['MATT LABASH', 'TWS PODCAST', 'ERIC FELTEN', 'Steven J. Lenzner', 'MARK HEMINGWAY']
- badImgArr=['http://www.weeklystandard.com/s3/tws15/images/twitter/tws-twitter_1024x512.png']
- wkl=removeBadStories(wkl, badTitleArr, badDescArr, badAuthorArr, badImgArr)
-
- return wkl
-
-
-
-
-def buildNPR():
- url='http://www.npr.org/sections/news/'
- name='NPR'
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #get main headline
- h1=content
- h1=h1.split('<a id="mainContent">', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- h1=h1.split('"', 1)[0]
- h1s=[h1]
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- h2=h2.split('<article class="item has-image">', 1)[1]
- h2=h2.split('<!-- END CLASS=\'FEATURED-3-UP\' -->', 1)[0]
- while '<article class="item has-image">' in h2:
- h2=h2.split('<article class="item has-image">', 1)[1]
- h2=h2.split('<a href="', 1)[1]
- x=h2.split('"', 1)[0]
- if h1 not in x:
- h2s.append(x)
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('<div id="overflow" class="list-overflow"', 1)[1]
- h3=h3.split('<!-- END ID="OVERFLOW" CLASS="LIST-OVERFLOW"', 1)[0]
- while '<h2 class="title"><a href="' in h3:
- h3=h3.split('<h2 class="title"><a href="', 1)[1]
- x=h3.split('"', 1)[0]
- if h1 not in x:
- h3s.append(x)
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
-
- npr=buildNewsSource2(name, url, h1s, h2s, h3s)
-
- #REMOVE BAD STORIES
- badTitleArr=['The Two-Way']
- badDescArr=None
- badAuthorArr=['Domenico Montanaro']
- badImgArr=None
- npr=removeBadStories(npr, badTitleArr, badDescArr, badAuthorArr, badImgArr)
-
- return npr
-
-
-
-
-
-def buildABC():
- url='http://www.abcnews.go.com'
- name='ABC News'
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #get main headline
- h1=content
- h1=h1.split('id="row-1"', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- h1=h1.split('"', 1)[0]
- h1s=[h1]
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- h2=h2.split('id="row-2"', 1)[1]
- h2=h2.split('id="row-3"', 1)[0]
- h2=h2.split('card single row-item')[1:3] #should just be 2 of these
- for x in h2:
- x=x.split('<a href="', 1)[1]
- x=x.split('"', 1)[0]
- if h1 not in x:
- h2s.append(x)
-
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- h3=h3.split('id="row-1"', 1)[1]
- h3=h3.split('tab-data active', 1)[1]
- h3=h3.split('tab-data"', 1)[0] #note the trailing quotation
- while '<a href="' in h3:
- h3=h3.split('<a href="', 1)[1]
- x=h3.split('"', 1)[0]
- if h1 not in x:
- h3s.append(x)
-
- h1s, h2s, h3s = removeDuplicates([h1], h2s, h3s)
- abc=buildNewsSource2(name, url, h1s, h2s, h3s)
-
- #REMOVE BAD STORIES
- badTitleArr=None
- badDescArr=None
- badAuthorArr=None
- badImgArr=None
- badURLArr=None
- abc=removeBadStories(abc, badTitleArr, badDescArr, badAuthorArr, badImgArr, badURLArr)
-
- return abc
-
-
-
-
-def buildFoxNews():
- url = 'http://foxnews.com'
- name = 'Fox News'
-
- # DOWNLOAD HOMEPAGE CONTENT
- content = urlToContent(url)
- soup = BeautifulSoup(content, 'lxml')
-
- # get main headline
- h1 = soup.find('div', id='big-top')\
- .find('div', class_='primary')\
- .find('h1')\
- .find('a')
- h1 = h1['href']
- h1s = [h1]
- h1s = ['http:' + x if x.startswith('//') else x for x in h1s]
-
- #GET SECONDARY HEADLINES
- h2s = soup.find('div', id='big-top').find('div', class_='top-stories').select('li > a')
- h2s = [x['href'] for x in h2s]
- h2s = ['http:' + x if x.startswith('//') else x for x in h2s]
-
- #GET TERTIARY HEADLINES
- h3s = []
- for ul in soup.find('section', id='latest').find_all('ul', recursive=False):
- for li in ul.find_all('li', recursive=False):
- h3s.append(li.find('a')['href'])
- h3s = ['http:' + x if x.startswith('//') else x for x in h3s]
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
- fox=buildNewsSource2(name, url, h1s, h2s, h3s)
-
- #REMOVE BAD STORIES
- badTitleArr=['O&#039;Reilly', 'Fox News', 'Brett Baier', 'Tucker']
- badDescArr=['Sean Hannity']
- badAuthorArr=['Bill O\'Reilly', 'Sean Hannity', 'Howard Kurtz']
- badImgArr=['http://www.foxnews.com/content/dam/fox-news/logo/og-fn-foxnews.jpg']
- badURLArr=['http://www.foxnews.com/opinion', 'videos.foxnews.com']
- fox=removeBadStories(fox, badTitleArr, badDescArr, badAuthorArr, badImgArr, badURLArr)
-
- return fox
-
-
-
-def buildNYT():
- url='http://www.nytimes.com'
- name='New York Times'
-
- #DOWNLOAD HOMEPAGE CONTENT
- content=urlToContent(url)
-
- #get main headline
- #this will likely need if/else logic
- h1=content
-
- if 'story theme-summary banner' in h1:
- #This is with a large headline over a and b columns
- h1=h1.split('story theme-summary banner', 1)[1]
- h1=h1.split('<a href="', 1)[1]
- h1=h1.split('"', 1)[0]
- else:
- #otherwise, pull the first story from the A column
- h1=h1.split('<div class="a-column column">', 1)[1]
- h1=h1.split('<article class="story theme-summary lede"', 1)[1]
- h1=h1.split('<a href="', 1)[1].split('"', 1)[0]
- h1s=[h1]
-
-
- #GET SECONDARY HEADLINES
- h2=content
- h2s=[]
- #A column
- h2=h2.split('<div class="a-column column">', 1)[1]
- h2=h2.split('<!-- close a-column -->', 1)[0]
- #remove "collection" sets
- while '<div class="collection headlines">' in h2:
- arr=h2.split('<div class="collection headlines">', 1)
- h2=arr[0]+arr[1].split('</ul>', 1)[1]
- #Grab the remaining URLs
- while '<a href="' in h2:
- h2=h2.split('<a href="', 1)[1]
- x=h2.split('"', 1)[0]
- if h1 not in x:
- h2s.append(x)
-
- #GET TERTIARY HEADLINES
- h3s=[]
- #B column
- h3=content
- h3=h3.split('<div class="b-column column">', 1)[1]
- h3=h3.split('<!-- close b-column -->', 1)[0]
- #remove "collection" sets
- while '<div class="collection headlines">' in h3:
- arr=h3.split('<div class="collection headlines">', 1)
- h3=arr[0]+arr[1].split('</ul>', 1)[1]
- #Grab the remaining URLs
- while '<a href="' in h3:
- h3=h3.split('<a href="', 1)[1]
- x=h3.split('"', 1)[0]
- if (h1 not in x) and (x not in h3s):
- h3s.append(x)
-
- '''
- #GET TERTIARY HEADLINES
- h3=content
- h3s=[]
- if '<!-- close lede-package-region -->' in h3:
- h3=h3.split('<!-- close lede-package-region -->', 1)[1]
- h3=h3.split('<a href="https://www.nytimes.com/tips">', 1)[0]
- elif '/video/the-daily-360' in h3:
- h3=h3.split('/video/the-daily-360')[-1]
- h3=h3.split('More News', 1)[0]
- #remove "collection" sets
- while '<div class="collection headlines">' in h2:
- arr=h3.split('<div class="collection headlines">', 1)
- h3=arr[0]+arr[1].split('</ul>', 1)[1]
-
- #Grab the remaining URLs
- while '<a href="' in h3:
- h3=h3.split('<a href="', 1)[1]
- x=h3.split('"', 1)[0]
- if (h1 not in x) and (x not in h3s):
- h3s.append(x)
- '''
-
- h1s, h2s, h3s = removeDuplicates(h1s, h2s, h3s)
-
- nyt=buildNewsSource2(name, url, h1s, h2s, h3s)
- nyt=removeBadStories(nyt, None, None, None, None, ['https://www.nytimes.com/section/magazine', 'https://www.nytimes.com/newsletters/the-interpreter'])
-
-
- return nyt
-
-
-
-
-'''
-NYT
-EXAMPLE OF BIG HEADLINE SPANNING BOTH A AND B COLUMNS
-
-<div class="span-ab-layout layout">
-
- <div class="ab-column column">
-
- <section id="top-news" class="top-news">
- <h2 class="section-heading visually-hidden">Top News</h2>
-
- <div class="above-banner-region region">
-
- <div class="collection">
- <div class="hpHeader" id="top-megapackage-kicker">
- <h6><a href="http://www.nytimes.com/pages/politics/index.html?src=hpHeader">The 45th President</a></h6>
-</div>
-
-</div>
-
- </div><!-- close above-banner-region -->
-
- <div class="span-ab-top-region region">
-
- <div class="collection">
- <article class="story theme-summary banner" id="topnews-100000004932040" data-story-id="100000004932040" data-rank="0" data-collection-renderstyle="Banner">
- <h1 class="story-heading"><a href="https://www.nytimes.com/2017/02/14/us/politics/fbi-interviewed-mike-flynn.html">F.B.I. Questioned Flynn About Russia Call</a></h1>
-</article>
-</div>
-
- </div><!-- close span-ab-top-region -->
-'''
diff --git a/unbiased/sources/__init__.py b/unbiased/sources/__init__.py
new file mode 100644
index 0000000..e4a473a
--- /dev/null
+++ b/unbiased/sources/__init__.py
@@ -0,0 +1,10 @@
+import importlib
+import pkgutil
+
+from unbiased.sources.base import NewsSource
+
+def get_sources():
+ for loader, name, is_pkg in pkgutil.walk_packages(__path__):
+ if name != 'base':
+ importlib.import_module('unbiased.sources.' + name)
+ return {x.shortname.lower(): x for x in NewsSource.__subclasses__()}
diff --git a/unbiased/sources/abc.py b/unbiased/sources/abc.py
new file mode 100644
index 0000000..d9092a2
--- /dev/null
+++ b/unbiased/sources/abc.py
@@ -0,0 +1,42 @@
+from unbiased.sources.base import NewsSource
+
+class ABC(NewsSource):
+
+ name = 'ABC News'
+ shortname = 'ABC'
+ url = 'http://abcnews.go.com/'
+
+ @classmethod
+ def _fetch_urls(cls):
+ """
+ Returns three tuples of urls, one for each of
+ the three tiers.
+ """
+ soup = cls._fetch_content(cls.url)
+
+ # get primary headline
+ h1 = soup.find('article', class_='hero')\
+ .find('div', class_='caption-wrapper').h1.a['href']
+ h1s = (h1,)
+
+ # get secondary headlines
+ h2s = soup.find('div', id='row-2')\
+ .find_all('article', class_='card single row-item')
+ h2s = tuple(x.find('div', class_='caption-wrapper').h1.a['href'] for x in h2s)
+
+ # get tertiary headlines
+ h3s = soup.find('div', id='row-1')\
+ .find('article', class_='headlines')\
+ .find('div', id='tab-content')\
+ .find_all('li', class_=['story', 'wirestory'])
+ h3s = tuple(x.div.h1.a['href'] for x in h3s)
+
+ return h1s, h2s, h3s
+
+ @classmethod
+ def _normalize_url(cls, url):
+ """
+ ABC News urls include an 'id' query param that we need to
+ keep in order for the URL to work.
+ """
+ return NewsSource._normalize_url(url, ['id'])
diff --git a/unbiased/sources/base.py b/unbiased/sources/base.py
new file mode 100644
index 0000000..9f51287
--- /dev/null
+++ b/unbiased/sources/base.py
@@ -0,0 +1,210 @@
+import collections
+import html
+import logging
+import urllib
+
+from bs4 import BeautifulSoup
+import requests
+
+logger = logging.getLogger('unbiased')
+
+class Article(object):
+
+ def __init__(self, source, title, author, description, url, img):
+ self.source = source
+ self.title = title
+ self.author = author
+ self.description = description
+ self.url = url
+ self.img = img
+
+ def __repr__(self):
+ return 'Article({}, {}, {}, {}, {}, {})'.format(self.source, self.title, self.author, self.description, self.url, self.img)
+
+class NewsSource(object):
+ """
+ Abstract base class.
+ To implement:
+ - set 'name', 'shortname', and 'url'
+ - set 'bad_' variables to blacklist terms and phrases
+ - implement '_fetch_urls()', which should return three tuples
+ of urls, one for each tier
+ - override any of the '_get_*()' functions as necessary
+ """
+
+ name = None
+ shortname = None
+ url = None
+
+ bad_titles = None
+ bad_authors = None
+ bad_descriptions = None
+ bad_imgs = None
+ bad_urls = None
+
+ def __init__(self, h1s, h2s, h3s):
+ self.h1s = h1s
+ self.h2s = h2s
+ self.h3s = h3s
+
+ @classmethod
+ def build(cls):
+ h1s, h2s, h3s = cls._fetch_urls()
+ logger.info('Fetched {} h1s, {} h2s, {} h3s'.format(len(h1s), len(h2s), len(h3s)))
+ h1s = tuple(cls._normalize_url(x) for x in h1s)
+ h2s = tuple(cls._normalize_url(x) for x in h2s)
+ h3s = tuple(cls._normalize_url(x) for x in h3s)
+ h1s, h2s, h3s = cls._remove_duplicates(h1s, h2s, h3s)
+ h1s, h2s, h3s = cls._fetch_articles(h1s, h2s, h3s)
+ h1s, h2s, h3s = cls._remove_all_bad_stories(h1s, h2s, h3s)
+ return cls(h1s, h2s, h3s)
+
+ @classmethod
+ def _fetch_content(cls, url):
+ res = requests.get(url)
+ if res.status_code == 200:
+ content = res.text
+ else:
+ raise Exception("Failed to download {}".format(url))
+ return BeautifulSoup(content, 'lxml')
+
+ @classmethod
+ def _normalize_url(cls, url, keep_query_vars=None):
+ """
+ Make sure they have a scheme.
+ Make sure they have a host.
+ Trim any query string, params, or fragments.
+ """
+ cls_url = urllib.parse.urlparse(cls.url)
+ url = urllib.parse.urlparse(url)
+ if keep_query_vars is None:
+ query = ''
+ else:
+ query_vars = {}
+ qs = urllib.parse.parse_qs(url.query)
+ for v in keep_query_vars:
+ if v in qs:
+ query_vars[v] = qs[v]
+ query_pairs = []
+ for k, i in query_vars.items():
+ for v in i:
+ query_pairs.append('{}={}'.format(k, v))
+ query = '?'.join(query_pairs)
+ url = (url.scheme or cls_url.scheme, url.netloc or cls_url.netloc, url.path, '', query, '')
+ return urllib.parse.urlunparse(url)
+
+ @classmethod
+ def _remove_duplicates(cls, h1s, h2s, h3s):
+ h2s = tuple(x for x in h2s if x not in h1s)
+ h3s = tuple(x for x in h3s if x not in h1s and x not in h2s)
+ return h1s, h2s, h3s
+
+ @classmethod
+ def _remove_bad_stories(cls, articles, element, filters):
+ # TODO: replace string filters with regex filters
+ if filters is None:
+ return articles
+ new_articles = []
+ for article in articles:
+ save = True
+ for f in filters:
+ if getattr(article, element) and f in getattr(article, element):
+ save = False
+ break
+ if save:
+ new_articles.append(article)
+ return tuple(new_articles)
+
+ @classmethod
+ def _remove_all_bad_stories(cls, h1s, h2s, h3s):
+ new_articles = []
+ for articles in [h1s, h2s, h3s]:
+ articles = cls._remove_bad_stories(articles, 'title', cls.bad_titles)
+ articles = cls._remove_bad_stories(articles, 'description', cls.bad_descriptions)
+ articles = cls._remove_bad_stories(articles, 'author', cls.bad_authors)
+ articles = cls._remove_bad_stories(articles, 'img', cls.bad_imgs)
+ articles = cls._remove_bad_stories(articles, 'url', cls.bad_urls)
+ new_articles.append(articles)
+ if len(new_articles[0]) == 0 and len(new_articles[1]) > 0:
+ new_articles[0] = new_articles[0] + new_articles[1][:1]
+ new_articles[1] = new_articles[1][1:]
+ return tuple(tuple(x) for x in new_articles)
+
+ @classmethod
+ def _fetch_articles(cls, h1s, h2s, h3s):
+ ret = []
+ for urls in [h1s, h2s, h3s]:
+ articles = []
+ for url in urls:
+ article = cls._fetch_article(url)
+ if article is not None:
+ articles.append(article)
+ ret.append(articles)
+ return tuple(tuple(x) for x in ret)
+
+ @classmethod
+ def _fetch_article(cls, url):
+ logger.debug(cls.name)
+ logger.debug(url)
+
+ try:
+ soup = cls._fetch_content(url)
+ except Exception as ex:
+ logger.debug("""ARTICLE DOWNLOADING ERROR
+ SOURCE:\t{}
+ URL:\t{}""".format(cls.name, url))
+ return None
+
+ url_parts = urllib.parse.urlparse(url)
+ scheme = url_parts.scheme
+
+ try:
+ img = cls._get_image(soup)
+ img = urllib.parse.urlparse(img, scheme=scheme).geturl()
+ logger.debug(img)
+
+ title = cls._get_title(soup)
+ logger.debug(title)
+
+ author = cls._get_author(soup)
+ logger.debug(author)
+
+ description = cls._get_description(soup)
+ logger.debug(description)
+ description = cls._remove_self_refs(description)
+ logger.debug(description)
+ except Exception:
+ logger.debug("""ARTICLE PARSING ERROR
+ SOURCE:\t{}
+ URL:\t{}""".format(cls.name, url))
+ return None
+
+ return Article(cls.name, title, author, description, url, img)
+
+ @classmethod
+ def _get_image(cls, soup):
+ return soup.find('meta', property='og:image')['content']
+
+ @classmethod
+ def _get_title(cls, soup):
+ return soup.find('meta', property='og:title')['content']
+
+ @classmethod
+ def _get_author(cls, soup):
+ for author_tag in ['article:author', 'dc.creator', 'author']:
+ author = soup.find('meta', property=author_tag)
+ if author is None:
+ continue
+ return author['content']
+ return None
+
+ @classmethod
+ def _get_description(cls, soup):
+ return soup.find('meta', property='og:description')['content']
+
+ @classmethod
+ def _remove_self_refs(cls, description):
+ description = description.replace(cls.name + "'s", '***')
+ description = description.replace(cls.name + "'", '***')
+ description = description.replace(cls.name, '***')
+ return description
diff --git a/unbiased/sources/bbc.py b/unbiased/sources/bbc.py
new file mode 100644
index 0000000..0dd0f80
--- /dev/null
+++ b/unbiased/sources/bbc.py
@@ -0,0 +1,26 @@
+from unbiased.sources.base import NewsSource
+
+class BBC(NewsSource):
+
+ name = 'BBC News'
+ shortname = 'bbc'
+ url = 'http://www.bbc.com/news/world/us_and_canada'
+
+ bad_images = ['bbc_news_logo.png']
+
+ @classmethod
+ def _fetch_urls(cls):
+ soup = cls._fetch_content(cls.url)
+
+ h1s = soup.find('div', class_='buzzard-item')\
+ .find('a', class_='title-link')
+ h1s = (h1s['href'],)
+
+ h2s = soup.find_all('div', attrs={'class': 'pigeon__column', 'data-entityid': True})
+ h2s = tuple(x.find('a', class_='title-link')['href'] for x in h2s)
+
+ # get tertiary headlines
+ h3s = soup.find_all('div', attrs={'class': 'macaw-item', 'data-entityid': True})
+ h3s = tuple(x.find('a', class_='title-link')['href'] for x in h3s)
+
+ return h1s, h2s, h3s
diff --git a/unbiased/sources/cbs.py b/unbiased/sources/cbs.py
new file mode 100644
index 0000000..295e671
--- /dev/null
+++ b/unbiased/sources/cbs.py
@@ -0,0 +1,37 @@
+from unbiased.sources.base import NewsSource
+
+class CBS(NewsSource):
+
+ name = 'CBS News'
+ shortname = 'cbs'
+ url = 'https://www.cbsnews.com/'
+
+ bad_titles = ['60 Minutes']
+ bad_descriptions = ['60 Minutes']
+ bad_urls = ['whats-in-the-news-coverart']
+
+ @classmethod
+ def _fetch_urls(cls):
+ soup = cls._fetch_content(cls.url)
+
+ # get primary headline
+ h1 = soup.find('h1', class_='title')
+ # sometimes they lead with a video
+ # if so, we'll pull the first h2 into the h1 slot later
+ if h1 is not None:
+ h1s = (h1.a['href'],)
+
+ # get secondary headlines
+ h2s = soup.find('div', attrs={'data-tb-region': 'Big News Area Side Assets'})\
+ .ul.find_all('li', attrs={'data-tb-region-item': True})
+ h2s = tuple(x.a['href'] for x in h2s)
+ if h1 is None:
+ h1s = (h2s[0],)
+ h2s = tuple(h2s[1:])
+
+ # get tertiary headlines
+ h3s = soup.find('div', attrs={'data-tb-region': 'Hard News'})\
+ .ul.find_all('li', attrs={'data-tb-region-item': True})
+ h3s = tuple(x.a['href'] for x in h3s[:5])
+
+ return h1s, h2s, h3s
diff --git a/unbiased/sources/csm.py b/unbiased/sources/csm.py
new file mode 100644
index 0000000..4e1eea5
--- /dev/null
+++ b/unbiased/sources/csm.py
@@ -0,0 +1,41 @@
+from unbiased.sources.base import NewsSource
+
+class CSM(NewsSource):
+
+ name = 'Christian Science Monitor'
+ shortname = 'csm'
+ url = 'https://www.csmonitor.com/USA'
+
+ bad_titles = ['Change Agent']
+ bad_imgs = ['csm_logo']
+ bad_urls = ['difference-maker']
+
+ @classmethod
+ def _fetch_urls(cls):
+ soup = cls._fetch_content(cls.url)
+
+ # get primary headline
+ h1 = soup.find('div', id='block-0-0')\
+ .find('h3', class_='story_headline')\
+ .a['href']
+ h1s = (h1,)
+
+ # get secondary headlines
+ h2_blocks = soup.find_all('div', id=['block-1-0', 'block-0-1'])
+ h2s = []
+ for block in h2_blocks:
+ hblocks = block.find_all('h3', class_='story_headline')
+ for hblock in hblocks:
+ h2s += [x for x in hblock.find_all('a') if 'first-look' not in x['href']]
+ h2s = tuple(x['href'] for x in h2s)
+
+ # get tertiary headlines
+ h3_blocks = soup.find_all('div', id='block-0-2')
+ h3s = []
+ for block in h3_blocks:
+ hblocks = block.find_all('h3', class_='story_headline')
+ for hblock in hblocks:
+ h3s += [x for x in hblock.find_all('a') if 'first-look' not in x['href']]
+ h3s = tuple(x['href'] for x in h3s)
+
+ return h1s, h2s, h3s
diff --git a/unbiased/sources/fox.py b/unbiased/sources/fox.py
new file mode 100644
index 0000000..ce7730f
--- /dev/null
+++ b/unbiased/sources/fox.py
@@ -0,0 +1,41 @@
+from unbiased.sources.base import NewsSource
+
+class Fox(NewsSource):
+
+ name = 'Fox News'
+ shortname = 'Fox'
+ url = 'http://www.foxnews.com'
+
+ bad_titles = ['O&#039;Reilly', 'Fox News', 'Brett Baier', 'Tucker']
+ bad_descriptions = ['Sean Hannity']
+ bad_authors = ['Bill O\'Reilly', 'Sean Hannity', 'Howard Kurtz']
+ bad_imgs = ['http://www.foxnews.com/content/dam/fox-news/logo/og-fn-foxnews.jpg']
+ bad_urls = ['http://www.foxnews.com/opinion', 'videos.foxnews.com']
+
+ @classmethod
+ def _fetch_urls(cls):
+ """
+ Returns three tuples of urls, one for each of
+ the three tiers.
+ """
+ soup = cls._fetch_content(cls.url)
+
+ # get primary headline
+ h1 = soup.find('div', id='big-top')\
+ .find('div', class_='primary')\
+ .find('h1')\
+ .find('a')['href']
+ h1s = (h1,)
+
+ # get secondary headlines
+ h2s = soup.find('div', id='big-top').find('div', class_='top-stories').select('li > a')
+ h2s = tuple(x['href'] for x in h2s)
+
+ # get tertiary headlines
+ h3s = []
+ for ul in soup.find('section', id='latest').find_all('ul', recursive=False):
+ for li in ul.find_all('li', recursive=False):
+ h3s.append(li.find('a')['href'])
+ h3s = tuple(h3s)
+
+ return h1s, h2s, h3s
diff --git a/unbiased/sources/guardian.py b/unbiased/sources/guardian.py
new file mode 100644
index 0000000..5a1c3dd
--- /dev/null
+++ b/unbiased/sources/guardian.py
@@ -0,0 +1,37 @@
+import html
+
+from unbiased.sources.base import NewsSource
+
+class TheGuardian(NewsSource):
+
+ name = 'The Guardian'
+ shortname = 'Guardian'
+ url = 'https://www.theguardian.com/us'
+
+ bad_authors = ['Tom McCarthy', 'Andy Hunter']
+ bad_urls = ['https://www.theguardian.com/profile/ben-jacobs']
+
+ @classmethod
+ def _fetch_urls(cls):
+ soup = cls._fetch_content(cls.url)
+
+ url_groups = []
+ for htag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+ hblocks = soup.find('section', id='headlines').find_all(htag)
+ urls = [x.a['href'] for x in hblocks]
+ url_groups.append(urls)
+ url_groups = [x for x in url_groups if len(url_groups) > 0]
+ if len(url_groups) < 3:
+ raise Exception('not enough article groups on Guardian home page!')
+
+ return tuple(url_groups[0]), tuple(url_groups[1]), tuple(url_groups[2])
+
+ @classmethod
+ def _get_image(cls, soup):
+ if soup.find('img', class_='maxed'):
+ img = soup.find('img', class_='maxed')['src']
+ if soup.find('meta', itemprop='image'):
+ img = soup.find('meta', itemprop='image')['content']
+ if soup.find('img', class_='immersive-main-media__media'):
+ img = soup.find('img', class_='immersive-main-media__media')['src']
+ return html.unescape(img)
diff --git a/unbiased/sources/npr.py b/unbiased/sources/npr.py
new file mode 100644
index 0000000..e52459f
--- /dev/null
+++ b/unbiased/sources/npr.py
@@ -0,0 +1,29 @@
+from unbiased.sources.base import NewsSource
+
+class NPR(NewsSource):
+
+ name = 'NPR News'
+ shortname = 'npr'
+ url = 'http://www.npr.org/sections/news/'
+
+ bad_titles = ['The Two-Way']
+ bad_authors = ['Domenico Montanaro']
+
+ @classmethod
+ def _fetch_urls(cls):
+ soup = cls._fetch_content(cls.url)
+
+ featured = soup.find('div', class_='featured-3-up')\
+ .find_all('article', recursive=False)
+
+ h1s = featured[:1]
+ h1s = tuple(x.find('h2', class_='title').a['href'] for x in h1s)
+ h2s = featured[1:]
+ h2s = tuple(x.find('h2', class_='title').a['href'] for x in h2s)
+
+ # get tertiary headlines
+ h3s = soup.find('div', id='overflow')\
+ .find_all('article', recursive=False)
+ h3s = tuple(x.find('h2', class_='title').a['href'] for x in h3s[:5])
+
+ return h1s, h2s, h3s
diff --git a/unbiased/sources/thehill.py b/unbiased/sources/thehill.py
new file mode 100644
index 0000000..862204e
--- /dev/null
+++ b/unbiased/sources/thehill.py
@@ -0,0 +1,39 @@
+
+from unbiased.sources.base import NewsSource
+
+class TheHill(NewsSource):
+
+ name = 'The Hill'
+ shortname = 'Hill'
+ url = 'http://thehill.com'
+
+ bad_titles = ['THE MEMO']
+ bad_authors = ['Matt Schlapp', 'Juan Williams', 'Judd Gregg']
+
+ @classmethod
+ def _fetch_urls(cls):
+ soup = cls._fetch_content(cls.url)
+
+ h1 = soup.find('h1', class_='top-story-headline')\
+ .find('a')['href']
+ h1s = (h1,)
+
+ h23s = soup.find('div', class_='section-top-content')\
+ .find_all('div', class_='top-story-item')
+ h2s = set([x.h4.a['href'] for x in h23s if 'small' not in x['class']])
+ h2s = tuple(h2s)
+
+ h3s = set([x.h4.a['href'] for x in h23s if 'small' in x['class']])
+ h3s = tuple(h3s)
+
+ return h1s, h2s, h3s
+
+ @classmethod
+ def _get_description(cls, soup):
+ try:
+ return NewsSource._get_description(soup)
+ except Exception:
+ # fall back on grabbing text from the article
+ desc = soup.find('div', class_='field-items')
+ return desc.text[:200].rsplit(' ', 1)[0]
+
diff --git a/unbiased/sources/washtimes.py b/unbiased/sources/washtimes.py
new file mode 100644
index 0000000..1be1838
--- /dev/null
+++ b/unbiased/sources/washtimes.py
@@ -0,0 +1,31 @@
+from unbiased.sources.base import NewsSource
+
+class TheWashingtonTimes(NewsSource):
+
+ name = 'The Washington Times'
+ shortname = 'WashTimes'
+ url = 'http://www.washingtontimes.com/'
+
+ @classmethod
+ def _fetch_urls(cls):
+ soup = cls._fetch_content(cls.url)
+
+ h1 = soup.find('article', class_='lead-story')\
+ .find(class_='article-headline')\
+ .a['href']
+ h1s = (h1,)
+
+ top_articles = soup.find('section', class_='top-news')\
+ .find_all('article', recursive=False)
+ h2s = []
+ for a in top_articles:
+ if a.attrs.get('class') is None:
+ h2s.append(a.a['href'])
+ h2s = tuple(h2s)
+
+ h3s = soup.find('section', class_='more-from desktop-only')\
+ .ul.find_all('a')
+ h3s = [x['href'] for x in h3s]
+ h3s = tuple(h3s)
+
+ return h1s, h2s, h3s
diff --git a/unbiased/spotCheck.py b/unbiased/spotCheck.py
deleted file mode 100755
index 7ce50d3..0000000
--- a/unbiased/spotCheck.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-
-from unbiased.parser import *
-from unbiased.unbiasedObjects import *
-
-def spotCheck(src):
-
- fns = {'hil' : buildTheHill,
- 'cbs' : buildCBS,
- 'npr' : buildNPR,
- 'fox' : buildFoxNews,
- 'gdn' : buildGuardian,
- 'blz' : buildBlaze,
- 'bbc' : buildBBC,
- 'nbc' : buildNBC,
- 'wat' : buildWashTimes,
- 'csm' : buildCSM,
- 'abc' : buildABC}
-
- data=fns[src]()
-
- print('H1s:\n--------------')
- for h in data.h1Arr:
- print(h.title)
-
- print('\n\nH2s:\n--------------')
- for h in data.h2Arr:
- print(h.title)
-
- print('\n\nH3s:\n--------------')
- for h in data.h3Arr:
- print(h.title)
-
- print('\n\n')
-
-
-
-if __name__=='__main__':
- spotCheck(sys.argv[1])
diff --git a/unbiased/unbiasedFunctions.py b/unbiased/unbiasedFunctions.py
deleted file mode 100644
index 6ec89b7..0000000
--- a/unbiased/unbiasedFunctions.py
+++ /dev/null
@@ -1,277 +0,0 @@
-import html
-import io
-import logging
-import os
-import pkgutil
-import random
-import re
-import time
-import urllib.parse
-
-from PIL import Image
-import requests
-
-from unbiased.unbiasedObjects import *
-
-logger = logging.getLogger('unbiased')
-
-#take in a url and delimiters, return twitter card
-def buildArticle(url, sourceName, encoding=None):#, titleDelStart, titleDelEnd, imgDelStart, imgDelEnd):
-
- debugging=False
- if debugging:
- logger.debug(sourceName)
- logger.debug(url)
-
- url_parts = urllib.parse.urlparse(url)
- scheme = url_parts.scheme
-
- #download url
- try:
- res = requests.get(url)
- except Exception as ex:
- logger.debug("""ARTICLE DOWNLOADING ERROR
- SOURCE:\t{}
- URL:\t{}""".format(sourceName, url))
- return None
-
- if res.status_code == 200:
- content = res.text
- else:
- logger.debug("""ARTICLE DOWNLOADING ERROR
- SOURCE:\t{}
- URL:\t{}""".format(sourceName, url))
- return None
-
- try:
- if sourceName=='The Guardian US':
- #The Guardian puts an identifying banner on their og:images
- #grab the main image from the page instead
-
- #scenario 1: regular image
- if '<img class="maxed' in content:
- img=content.split('<img class="maxed', 1)[1]
- img=img.split('src="', 1)[1].split('"')[0]
- #scenario 2: video in image spot
- elif '<meta itemprop="image"' in content:
- img=content.split('<meta itemprop="image"', 1)[1]
- img=img.split('content="', 1)[1].split('"')[0]
- #scenario 3: photo essays
- elif '<img class="immersive-main-media__media"' in content:
- img=content.split('<img class="immersive-main-media__media"', 1)[1]
- img=img.split('src="', 1)[1].split('"')[0]
- img = html.unescape(img)
-
- else:
- if 'og:image' in content:
- img=content.split('og:image" content=')[1][1:].split('>')[0]
- elif sourceName=='ABC News':
- img='https://c1.staticflickr.com/7/6042/6276688407_12900948a2_b.jpgX'
- if img[-1]=='/':
- #because the quote separator could be ' or ",
- #trim to just before it then lop it off
- img=img[:-1].strip()
- img=img[:-1]
- # fix the scheme if it's missing
- img = urllib.parse.urlparse(img, scheme=scheme).geturl()
-
- if debugging:
- logger.debug(img)
-
- title=content.split('og:title" content=')[1][1:].split('>')[0]
- if title[-1]=='/':
- title=title[:-1].strip()
- title=title[:-1]
-
- if debugging:
- logger.debug(title)
-
-
- author=''
- if sourceName=='The Blaze':
- if 'class="article-author">' in content:
- author=content.split('class="article-author">')[1].split('<')[0]
- elif 'class="article-author" href="' in content:
- author=content.split('class="article-author" href="')[1]
- author=author.split('>')[1].split('<')[0].strip()
- else:
- authorTags=['article:author', 'dc.creator', 'property="author']
- for tag in authorTags:
- if tag in content:
- author=content.split(tag+'" content=')[1][1:].split('>')[0]
- author=author[:-1]
- #trim an extra quotation mark for The Hill
- if sourceName=='The Hill':
- author=author.split('"', 1)[0]
- break
-
- if debugging:
- logger.debug(author)
-
-
- if 'og:description' in content:
- description=content.split('og:description" content=')[1][1:].split('>')[0]
- if description[-1]=='/':
- description=description[:-1].strip()
- description=description[:-1]
- else:
- if sourceName=='The Hill':
- description=content.split('div class="field-items"')[-1]
- description=re.sub('<[^<]+?>', '', description)
- description=description[1:200]
- else:
- logger.debug("SHOULDN'T GET HERE")
-
- #strip out self-references
- description=description.replace(sourceName+"'s", '***')
- description=description.replace(sourceName+"'", '***')
- description=description.replace(sourceName, '***')
-
- if debugging:
- logger.debug(description)
-
-
- a=Article(html.unescape(title), url, img, html.unescape(description), sourceName, html.unescape(author))
- return a
-
- except Exception:
- logger.debug("""ARTICLE PARSING ERROR
- SOURCE:\t{}
- URL:\t{}""".format(sourceName, url))
- return None
-
-
-def pickStories(newsSourceArr):
- # TODO: refactor to avoid infinite loops
- #set the random order for sources
- h1RandomSources=[]
- guard = 0
- while len(h1RandomSources)<4:
- x=random.sample(range(len(newsSourceArr)), 1)[0]
- if len(newsSourceArr[x].h1Arr)>0:
- if x not in h1RandomSources:
- h1RandomSources.append(x)
- else:
- logger.debug('No H1 stories in '+newsSourceArr[x].name)
- guard += 1
- if guard > 100:
- return [], [], []
-
- #For h2s and h3s, select N random sources (can repeat), then
- #a non-repetitive random article from within
- h2RandomPairs=[]
- while len(h2RandomPairs) < 6:
- x=random.sample(range(len(newsSourceArr)), 1)[0]
- if len(newsSourceArr[x].h2Arr) > 0:
- y=random.sample(range(len(newsSourceArr[x].h2Arr)), 1)[0]
- pair=[x,y]
- if not pair in h2RandomPairs:
- h2RandomPairs.append(pair)
- else:
- logger.debug('No H2 stories in '+newsSourceArr[x].name)
-
- h3RandomPairs=[]
- while len(h3RandomPairs) < 12:
- x=random.sample(range(len(newsSourceArr)), 1)[0]
- if len(newsSourceArr[x].h3Arr) > 0:
- y=random.sample(range(len(newsSourceArr[x].h3Arr)), 1)[0]
- pair=[x,y]
- if not pair in h3RandomPairs:
- h3RandomPairs.append(pair)
- else:
- logger.debug('No H3 stories in '+newsSourceArr[x].name)
-
- # collect articles for each section
- image_index = 0
-
- top_stories = []
- for i in range(len(h1RandomSources)):
- source=newsSourceArr[h1RandomSources[i]]
- randomArticle=random.sample(range(len(source.h1Arr)), 1)[0]
- article=source.h1Arr[randomArticle]
- top_stories.append(article)
-
- middle_stories = []
- for i in range(len(h2RandomPairs)):
- pair=h2RandomPairs[i]
- article=newsSourceArr[pair[0]].h2Arr[pair[1]]
- middle_stories.append(article)
-
- bottom_stories = []
- for i in range(len(h3RandomPairs)):
- pair=h3RandomPairs[i]
- article=newsSourceArr[pair[0]].h3Arr[pair[1]]
- bottom_stories.append(article)
-
- return top_stories, middle_stories, bottom_stories
-
-def buildOutput(top_stories, middle_stories, bottom_stories):
- #read in the template html file
- from jinja2 import Environment, PackageLoader, select_autoescape
- env = Environment(
- loader=PackageLoader('unbiased', 'html_template'),
- autoescape=select_autoescape(['html', 'xml'])
- )
- template = env.get_template('unbiased.jinja.html')
-
- timestamp = time.strftime("%a, %b %-d, %-I:%M%P %Z", time.localtime())
- utime = int(time.time())
-
- sourcesStr = ', '.join(set([x.source for x in top_stories] + [x.source for x in middle_stories] + [x.source for x in bottom_stories]))
-
- html = template.render(
- timestamp = timestamp,
- utime = utime,
- top_stories = top_stories,
- middle_stories = middle_stories,
- bottom_stories = bottom_stories,
- sources = sourcesStr,
- )
-
- #return updated text
- return html
-
-def writeOutputHTML(outputHTML, outDir):
- timestamp = time.strftime("%a, %b %-d, %-I:%M%P %Z", time.localtime())
-
- with open(os.path.join(outDir, 'index.html'), 'w') as fp:
- fp.write(outputHTML)
-
- # copy over static package files
- for filename in ['unbiased.css', 'favicon.ico', 'favicon.png', 'apple-touch-icon.png']:
- data = pkgutil.get_data('unbiased', os.path.join('html_template', filename))
- with open(os.path.join(outDir, filename), 'wb') as fp:
- fp.write(data)
-
-def pullImage(url, index, webroot, target_width=350, target_height=200):
- extension = url.split('.')[-1].split('?')[0]
- img_name = 'img{}.{}'.format(index, extension)
- res = requests.get(url)
- if res.status_code == 200:
- content = res.content
- else:
- logger.debug('Image not found: url={}'.format(url))
- return ''
- img = Image.open(io.BytesIO(content))
- # crop to aspect ratio
- target_ar = target_width / target_height
- left, top, right, bottom = img.getbbox()
- height = bottom - top
- width = right - left
- ar = width / height
- if target_ar > ar:
- new_height = (target_height / target_width) * width
- bbox = (left, top + ((height - new_height) / 2), right, bottom - ((height - new_height) / 2))
- img = img.crop(bbox)
- elif target_ar < ar:
- new_width = (target_width / target_height) * height
- bbox = (left + ((width - new_width) / 2), top, right - ((width - new_width) / 2), bottom)
- img = img.crop(bbox)
- # resize if larger
- if target_width * 2 < width or target_height * 2 < height:
- img = img.resize((target_width*2, target_height*2), Image.LANCZOS)
- # TODO: create retina images
- jpg_name = 'img{}.jpg'.format(index)
- out_file = os.path.join(webroot, jpg_name)
- img.save(out_file, 'JPEG')
- return jpg_name
diff --git a/unbiased/unbiasedObjects.py b/unbiased/unbiasedObjects.py
deleted file mode 100644
index 9a8a78a..0000000
--- a/unbiased/unbiasedObjects.py
+++ /dev/null
@@ -1,97 +0,0 @@
-import logging
-
-logger = logging.getLogger('unbiased')
-
-class Article():
- title=''
- url=''
- img=''
- description=''
- source=''
- author=''
-
- def __init__(self, title, url, img, description, source, author):
- self.title=title
- self.url=url
- self.img=img
- self.description=description
- self.source=source
- self.author=author
-
- def __str__(self):
- return '-----------\ntitle: {}\nauthor: {}\nsource: {}\ndescription: {}\nurl: {}\nimg: {}\n-----------'.format(self.title, self.author, self.source, self.description, self.url, self.img)
-
- def __repr__(self):
- return '{}({}, {}, {})'.format(self.source.replace(' ', ''), self.title, self.author, self.url)
-
-
-class NewsSource2():
- name=''
- url=''
- h1Arr=[]
- h2Arr=[]
- h3Arr=[]
- def __init__(self, name, url, h1Arr, h2Arr, h3Arr):
- self.name=name
- self.url=url
- self.h1Arr=h1Arr
- self.h2Arr=h2Arr
- self.h3Arr=h3Arr
-
-
-
-class NewsSource():
- name=''
- url=''
- #multiple start values to step through file. end value default to '"'
- h1SectionDividerStart=None
- h1SectionDividerEnd=None
- h1DelStart=[]
- h1DelEnd='"'
- h2SectionDividerStart=None
- h2SectionDividerEnd=None
- h2DelStart=[]
- h2DelEnd='"'
- h3SectionDividerStart=None
- h3SectionDividerEnd=None
- h3DelStart=[]
- h3DelEnd='"'
- #arrays of Article object types
- h1Arr=None
- h2Arr=None
- h3Arr=None
- #url to attach to stub links
- stubURL=''
-
- def __init__(self, name, url,
- h1DelStart, h2DelStart, h3DelStart,
- h1SectionDividerStart=None, h1SectionDividerEnd=None,
- h2SectionDividerStart=None, h2SectionDividerEnd=None,
- h3SectionDividerStart=None, h3SectionDividerEnd=None,
- stubURL=None):
- self.name=name
- self.url=url
- self.h1DelStart=h1DelStart
- self.h2DelStart=h2DelStart
- self.h3DelStart=h3DelStart
- self.h1SectionDividerStart=h1SectionDividerStart
- self.h2SectionDividerStart=h2SectionDividerStart
- self.h3SectionDividerStart=h3SectionDividerStart
- self.h1SectionDividerEnd=h1SectionDividerEnd
- self.h2SectionDividerEnd=h2SectionDividerEnd
- self.h3SectionDividerEnd=h3SectionDividerEnd
- self.h1Arr=[]
- self.h2Arr=[]
- self.h3Arr=[]
- self.stubURL=stubURL
-
- def addArticle(self, article, level):
- if level==1:
- self.h1Arr.append(article)
- elif level==2:
- self.h2Arr.append(article)
- elif level==3:
- self.h3Arr.append(article)
- else:
- logger.debug("Invalid level in NewsSource.addArtlce: " + level)
-
diff --git a/unbiased/util.py b/unbiased/util.py
new file mode 100644
index 0000000..12003b1
--- /dev/null
+++ b/unbiased/util.py
@@ -0,0 +1,113 @@
+import io
+import logging
+import os
+import pkgutil
+import random
+import shutil
+import time
+
+from PIL import Image
+import requests
+
+logger = logging.getLogger('unbiased')
+
+
+def pick_randoms(story_lists, length, per_source):
+ """
+ Return a randomly chosen list of 'length' stories, picking at
+ most 'per_source' stories from each source.
+ """
+ # TODO: weighting is incorrect if a source has fewer than 'per_source' articles
+ urandom = random.SystemRandom()
+ candidates = []
+ for stories in story_lists:
+ indexes = list(range(len(stories)))
+ urandom.shuffle(indexes)
+ random_indexes = indexes[:per_source]
+ candidates.extend([stories[x] for x in random_indexes])
+ indexes = list(range(len(candidates)))
+ urandom.shuffle(indexes)
+ random_indexes = indexes[:length]
+ return tuple(candidates[x] for x in random_indexes)
+
+
+def pickStories(newsSourceArr):
+ h1s = pick_randoms([x.h1s for x in newsSourceArr], 4, 1)
+ h2s = pick_randoms([x.h2s for x in newsSourceArr], 6, 2)
+ h3s = pick_randoms([x.h3s for x in newsSourceArr], 12, 2)
+ return h1s, h2s, h3s
+
+
+def buildOutput(top_stories, middle_stories, bottom_stories):
+ # read in the template html file
+ from jinja2 import Environment, PackageLoader, select_autoescape
+ env = Environment(
+ loader=PackageLoader('unbiased', 'html_template'),
+ autoescape=select_autoescape(['html', 'xml'])
+ )
+ template = env.get_template('unbiased.jinja.html')
+
+ timestamp = time.strftime("%a, %b %-d, %-I:%M%P %Z", time.localtime())
+ utime = int(time.time())
+
+ sourcesStr = ', '.join(set([x.source for x in top_stories] + [x.source for x in middle_stories] + [x.source for x in bottom_stories]))
+
+ html = template.render(
+ timestamp=timestamp,
+ utime=utime,
+ top_stories=top_stories,
+ middle_stories=middle_stories,
+ bottom_stories=bottom_stories,
+ sources=sourcesStr,
+ )
+
+ return html
+
+
+def write_files(files_to_write, outDir):
+ for name, bytesio in files_to_write.items():
+ with open(os.path.join(outDir, name), 'wb') as fp:
+ shutil.copyfileobj(bytesio, fp)
+
+
+def write_static_files(outDir):
+ # copy over static package files
+ for filename in ['unbiased.css', 'favicon.ico', 'favicon.png', 'apple-touch-icon.png']:
+ data = pkgutil.get_data('unbiased', os.path.join('html_template', filename))
+ with open(os.path.join(outDir, filename), 'wb') as fp:
+ fp.write(data)
+
+
+def pullImage(url, index, webroot, target_width=350, target_height=200):
+ res = requests.get(url)
+ if res.status_code == 200:
+ content = res.content
+ else:
+ logger.debug('Image not found: url={}'.format(url))
+ return ''
+ img = Image.open(io.BytesIO(content))
+ # crop to aspect ratio
+ target_ar = target_width / target_height
+ left, top, right, bottom = img.getbbox()
+ height = bottom - top
+ width = right - left
+ ar = width / height
+ if target_ar > ar:
+ new_height = (target_height / target_width) * width
+ bbox = (left, top + ((height - new_height) / 2), right, bottom - ((height - new_height) / 2))
+ img = img.crop(bbox)
+ elif target_ar < ar:
+ new_width = (target_width / target_height) * height
+ bbox = (left + ((width - new_width) / 2), top, right - ((width - new_width) / 2), bottom)
+ img = img.crop(bbox)
+ # resize if larger
+ if target_width * 2 < width or target_height * 2 < height:
+ img = img.resize((target_width * 2, target_height * 2), Image.LANCZOS)
+ # TODO: fill with a neutral color instead of just discarding alpha channel
+ img = img.convert('RGB')
+ # TODO: create retina images
+ jpg_name = 'img{}.jpg'.format(index)
+ jpg_file = io.BytesIO()
+ img.save(jpg_file, 'JPEG')
+ jpg_file.seek(0)
+ return jpg_name, jpg_file
generated by cgit v1.2.3 (git 2.39.1) at 2024-10-18 10:20:48 +0000