normalize urls using urllib

1 files changed, 8 insertions, 10 deletions

diff --git a/unbiased/sources/base.py b/unbiased/sources/base.py
index 9dc14fd..51c4cff 100644
--- a/unbiased/sources/base.py
+++ b/unbiased/sources/base.py
@@ -51,9 +51,9 @@ class NewsSource(object):
     @classmethod
     def build(cls):
         h1s, h2s, h3s = cls._fetch_urls()
-        h1s = tuple(cls._fix_url(x) for x in h1s)
-        h2s = tuple(cls._fix_url(x) for x in h2s)
-        h3s = tuple(cls._fix_url(x) for x in h3s)
+        h1s = tuple(cls._normalize_url(x) for x in h1s)
+        h2s = tuple(cls._normalize_url(x) for x in h2s)
+        h3s = tuple(cls._normalize_url(x) for x in h3s)
         h1s, h2s, h3s = cls._remove_duplicates(h1s, h2s, h3s)
         h1s, h2s, h3s = cls._fetch_articles(h1s, h2s, h3s)
         h1s, h2s, h3s = cls._remove_all_bad_stories(h1s, h2s, h3s)
@@ -69,16 +69,14 @@ class NewsSource(object):
         return BeautifulSoup(content, 'lxml')
 
     @classmethod
-    def _fix_url(cls, url, scheme='http'):
+    def _normalize_url(cls, url, scheme='http'):
         """
         Make sure they have a scheme.
-        Trim any query parameters.
+        Trim any query string, params, or fragments.
         """
-        # TODO: proper URL parsing
-        if url.startswith('//'):
-            url = '{}:{}'.format(scheme, x)
-        url = url.split('?')[0]
-        return url
+        url = urllib.parse.urlparse(url)
+        url = (url.scheme or scheme, url.netloc, url.path, '', '', '')
+        return urllib.parse.urlunparse(url)
 
     @classmethod
     def _remove_duplicates(cls, h1s, h2s, h3s):