diff options
| -rw-r--r-- | unbiased/sources/base.py | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/unbiased/sources/base.py b/unbiased/sources/base.py index 68211c5..ea81f25 100644 --- a/unbiased/sources/base.py +++ b/unbiased/sources/base.py @@ -61,7 +61,11 @@ class NewsSource(object): @classmethod def _fetch_content(cls, url): - res = requests.get(url, timeout=3) + # fake a regular browser user agent for sites that sniff + headers = { + 'User-Agent': u'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36' + } + res = requests.get(url, timeout=3, headers=headers) if res.status_code == 200: content = res.text else: |