#!/usr/bin/env python3 from unbiasedObjects import * from unbiasedFunctions import buildArticle import os ''' Takes in a URL, downloads the file to a temp file, reads the file into a string, and returns that string ''' def urlToContent(url): #download file os.system('wget -q -O scratch/temp1.html --no-check-certificate '+url) #read file f=open('scratch/temp1.html', 'r')#, encoding="utf8") content=f.read() f.close() return content ''' Creates a new newsSource2 object. For each URL in h1-h3URLs, calls the file scraper and appends the new Article object. Returns a newsSource2 object ''' def buildNewsSource2(name, url, h1URLs, h2URLs, h3URLs): h1Arr=[] h1Arr.append(buildArticle(h1URLs[0], name)) h2Arr=[] for x in h2URLs: h2Arr.append(buildArticle(x, name)) h3Arr=[] for x in h3URLs: h3Arr.append(buildArticle(x, name)) #BUILD THE NEWS SOURCE newsSource=NewsSource2(name, url, h1Arr, h2Arr, h3Arr) return newsSource ''' Some sites will replicate URLs across the page. This function removes them. Check hierarchically: if h3 exists in h1s or h2s, remove from h3s; if h2 exists in h1s, remove from h2s also check partial URLs (e.g. nytimes.com/story.html is the same as nytimes.com/story.html?var=x ''' def removeDuplicates(h1s, h2s, h3s): #Assume h1s is one element, and keep it #remove h2 duplicates removeArr=[] for i in range(len(h2s)): #check internally for j in range(len(h2s)): if i==j: continue else: if h2s[i] in h2s[j]: removeArr.append(h2s[j]) #check against h1s for k in range(len(h1s)): if (h2s[i] in h1s[k]) or (h1s[k] in h2s[i]): removeArr.append(h2s[i]) for x in removeArr: h2s.remove(x) #remove h3 duplicates removeArr=[] for i in range(len(h3s)): #check internally for j in range(len(h3s)): if i==j: continue else: if h3s[i] in h3s[j]: removeArr.append(h3s[j]) #check against h1s and h2s h1and2=h1s+h2s for k in range(len(h1and2)): if (h3s[i] in h1and2[k]) or (h1and2[k] in h3s[i]): removeArr.append(h3s[i]) for x in removeArr: h3s.remove(x) return h1s, h2s, h3s def buildFoxNews(): url='http://foxnews.com' name='Fox News' #DOWNLOAD HOMEPAGE CONTENT content=urlToContent(url) #get main headline h1=content h1=h1.split('

', 1)[1] h2=h2.split('
', 1)[0] while '', 1)[1] h2=h2.split('', 1)[0] #remove "collection" sets while '
' in h2: arr=h2.split('
', 1) h2=arr[0]+arr[1].split('', 1)[1] #Grab the remaining URLs while '', 1)[1] h2=h2.split('', 1)[0] #remove "collection" sets while '
' in h2: arr=h2.split('
', 1) h2=arr[0]+arr[1].split('', 1)[1] #Grab the remaining URLs while '', 1)[1] h3=h3.split('', 1)[0] #remove "collection" sets while '
' in h2: arr=h3.split('
', 1) h3=arr[0]+arr[1].split('', 1)[1] #Grab the remaining URLs while '
'''