diff options
| author | Matt Singleton <matt@xcolour.net> | 2023-03-04 20:04:30 -0600 |
|---|---|---|
| committer | Matt Singleton <matt@xcolour.net> | 2023-03-04 20:04:30 -0600 |
| commit | 84ef462fe22d339420db562e7d1673807dea7fc4 (patch) | |
| tree | 275b56297099b77719e5dc48a395cb8594683808 | |
| parent | 256875f2403441e0c1627c89de51cc2015945a66 (diff) | |
fake user agent
| -rw-r--r-- | unbiased/sources/base.py | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/unbiased/sources/base.py b/unbiased/sources/base.py index 68211c5..ea81f25 100644 --- a/unbiased/sources/base.py +++ b/unbiased/sources/base.py @@ -61,7 +61,11 @@ class NewsSource(object): @classmethod def _fetch_content(cls, url): - res = requests.get(url, timeout=3) + # fake a regular browser user agent for sites that sniff + headers = { + 'User-Agent': u'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36' + } + res = requests.get(url, timeout=3, headers=headers) if res.status_code == 200: content = res.text else: |