From 01b4fd03f3c2c09eb0337aeade83daf93856c408 Mon Sep 17 00:00:00 2001 From: Matt Singleton Date: Mon, 7 Sep 2020 14:31:13 -0500 Subject: use stdlib url parsing in gemini module --- .pylintrc | 7 +++- Pipfile | 2 + Pipfile.lock | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- TODO.md | 10 +++++ browser.py | 19 ++++----- gemini.py | 46 +++++++++++----------- test_gemini.py | 19 +++++++++ 7 files changed, 189 insertions(+), 34 deletions(-) create mode 100644 TODO.md create mode 100644 test_gemini.py diff --git a/.pylintrc b/.pylintrc index 0ace322..5342c9f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -139,7 +139,9 @@ disable=print-statement, deprecated-sys-function, exception-escape, comprehension-escape, - c-extension-no-member + c-extension-no-member, + missing-module-docstring, + missing-function-docstring # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -423,7 +425,8 @@ good-names=i, k, ex, Run, - _ + _, + fp # Good variable names regexes, separated by a comma. If names match any regex, # they will always be accepted diff --git a/Pipfile b/Pipfile index d087bf0..c1178de 100644 --- a/Pipfile +++ b/Pipfile @@ -5,6 +5,8 @@ verify_ssl = true [dev-packages] pylint = "*" +pytest = "*" +mypy = "*" [packages] pyside2 = "*" diff --git a/Pipfile.lock b/Pipfile.lock index f071e66..71b7109 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6ab125c809493f48b2455c80c8297db7ea44a5f654424211b1a3538067433d5b" + "sha256": "3218b2f6e241e522a76bfce33f6cea9b42fe7a8127340096e0a20effd637aa6f" }, "pipfile-spec": 6, "requires": { @@ -52,6 +52,20 @@ ], "version": "==2.4.2" }, + "attrs": { + "hashes": [ + "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", + "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" + ], + "version": "==20.2.0" + }, + "iniconfig": { + "hashes": [ + "sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437", + "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69" + ], + "version": "==1.0.1" + }, "isort": { "hashes": [ "sha256:92533892058de0306e51c88f22ece002a209dc8e80288aa3cec6d443060d584f", @@ -92,6 +106,61 @@ ], "version": "==0.6.1" }, + "more-itertools": { + "hashes": [ + "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20", + "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c" + ], + "version": "==8.5.0" + }, + "mypy": { + "hashes": [ + "sha256:2c6cde8aa3426c1682d35190b59b71f661237d74b053822ea3d748e2c9578a7c", + "sha256:3fdda71c067d3ddfb21da4b80e2686b71e9e5c72cca65fa216d207a358827f86", + "sha256:5dd13ff1f2a97f94540fd37a49e5d255950ebcdf446fb597463a40d0df3fac8b", + "sha256:6731603dfe0ce4352c555c6284c6db0dc935b685e9ce2e4cf220abe1e14386fd", + "sha256:6bb93479caa6619d21d6e7160c552c1193f6952f0668cdda2f851156e85186fc", + "sha256:81c7908b94239c4010e16642c9102bfc958ab14e36048fa77d0be3289dda76ea", + "sha256:9c7a9a7ceb2871ba4bac1cf7217a7dd9ccd44c27c2950edbc6dc08530f32ad4e", + "sha256:a4a2cbcfc4cbf45cd126f531dedda8485671545b43107ded25ce952aac6fb308", + "sha256:b7fbfabdbcc78c4f6fc4712544b9b0d6bf171069c6e0e3cb82440dd10ced3406", + "sha256:c05b9e4fb1d8a41d41dec8786c94f3b95d3c5f528298d769eb8e73d293abc48d", + "sha256:d7df6eddb6054d21ca4d3c6249cae5578cb4602951fd2b6ee2f5510ffb098707", + "sha256:e0b61738ab504e656d1fe4ff0c0601387a5489ca122d55390ade31f9ca0e252d", + "sha256:eff7d4a85e9eea55afa34888dfeaccde99e7520b51f867ac28a48492c0b1130c", + "sha256:f05644db6779387ccdb468cc47a44b4356fc2ffa9287135d05b70a98dc83b89a" + ], + "index": "pypi", + "version": "==0.782" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, + "packaging": { + "hashes": [ + "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", + "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" + ], + "version": "==20.4" + }, + "pluggy": { + "hashes": [ + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" + ], + "version": "==0.13.1" + }, + "py": { + "hashes": [ + "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2", + "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342" + ], + "version": "==1.9.0" + }, "pylint": { "hashes": [ "sha256:bb4a908c9dadbc3aac18860550e870f58e1a02c9f2c204fdf5693d73be061210", @@ -100,6 +169,21 @@ "index": "pypi", "version": "==2.6.0" }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "version": "==2.4.7" + }, + "pytest": { + "hashes": [ + "sha256:85228d75db9f45e06e57ef9bf4429267f81ac7c0d742cc9ed63d09886a9fe6f4", + "sha256:8b6007800c53fdacd5a5c192203f4e531eb2a1540ad9c752e052ec0f7143dbad" + ], + "index": "pypi", + "version": "==6.0.1" + }, "six": { "hashes": [ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", @@ -114,6 +198,40 @@ ], "version": "==0.10.1" }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "version": "==1.4.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "version": "==3.7.4.3" + }, "wrapt": { "hashes": [ "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7" diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..837dd54 --- /dev/null +++ b/TODO.md @@ -0,0 +1,10 @@ + - [ ] open non-gemini links externally and display differently + - [ ] keyboard navigation + - vim-style + - scroll + - highlight and follow links + - forward and back + - url bar + - [ ] search in page + - [ ] work with search pages + - [x] stdlib url parsing in gemini module diff --git a/browser.py b/browser.py index 58b650c..21e9293 100755 --- a/browser.py +++ b/browser.py @@ -20,42 +20,43 @@ class GViewport(QtWidgets.QTextBrowser): def mouseMoveEvent(self, event): cur = self.cursorForPosition(event.localPos().toPoint()) hover_url = cur.charFormat().anchorHref() - hover_url = gemini.absolute_url(self._current_url, QtCore.QUrl(hover_url)) + hover_url = gemini.urljoin(self._current_url, hover_url) if hover_url != self._hover_url: self._hover_url = hover_url - self.hoverUrlChanged.emit(self._hover_url.toString()) + self.hoverUrlChanged.emit(self._hover_url) return super().mouseMoveEvent(event) def loadResource(self, type_, url): + current_url = QtCore.QUrl(self._current_url) if self._last_redirect[0].toString() == url.toString(): gem = self._last_redirect[1] else: if not url.scheme(): - url.setScheme(self._current_url.scheme()) + url.setScheme(current_url.scheme()) if not url.host(): - url.setHost(self._current_url.host()) + url.setHost(current_url.host()) if url.port() == -1: url.setPort(1965) if not url.path().startswith('/'): - url.setPath(self._current_url.path().rsplit('/', 1)[0] + '/' + url.path()) - gem = gemini.get(gemini.absolute_url(self._current_url, url)) + url.setPath(current_url.path().rsplit('/', 1)[0] + '/' + url.path()) + gem = gemini.get(gemini.urljoin(current_url.toString(), url.toString())) if 'body' in gem: html = gemini.gem2html(gem['body']) else: html = '

{} {}

'.format(gem['status'], gem['meta']) - self._current_url = url + self._current_url = url.toString() return html def setSource(self, url): if url.scheme() != 'gemini': return - gem = gemini.get(gemini.absolute_url(self._current_url, url)) + gem = gemini.get(gemini.urljoin(self._current_url, url.toString())) while gem['status'][0] == '3': url = QtCore.QUrl(gem['meta']) if url.port() == 1965: url.setPort(-1) print('redirect: {}'.format(url)) - gem = gemini.get(gemini.absolute_url(self._current_url, url)) + gem = gemini.get(gemini.urljoin(self._current_url, url.toString())) self._last_redirect = (url, gem) if url.port() == 1965: url.setPort(-1) diff --git a/gemini.py b/gemini.py index c0d09c1..58f3358 100644 --- a/gemini.py +++ b/gemini.py @@ -1,10 +1,12 @@ +import re import socket import ssl +import urllib.parse -def htmlescape(text): +def htmlescape(text: str) -> str: return text.replace('<', '<').replace('>', '>') -def gem2html(gem): +def gem2html(gem: str) -> str: html = [] state = 'text' blanklines = 0 @@ -69,33 +71,33 @@ def gem2html(gem): html.append('

{}

'.format(htmlescape(line))) return '\n'.join(html) -def absolute_url(base, url): - """ - modifies `url` in place - """ - if not url.scheme(): - url.setScheme(base.scheme()) - if not url.host(): - url.setHost(base.host()) - if not url.path().startswith('/'): - url.setPath(base.path().rsplit('/', 1)[0] + '/' + url.path()) - if url.port() == -1: - url.setPort(1965) - return url +def urljoin(base: str, url: str) -> str: + if base is None: + return url + base = re.sub('^gemini:', 'http:', base) + url = re.sub('^gemini:', 'http:', url) + return re.sub('^http:', 'gemini:', urllib.parse.urljoin(base, url)) -def get(url): - if len(url.path()) == 0: - url.setPath('/') +def get(url: str) -> dict: + url_parts = urllib.parse.urlsplit(url) + if len(url_parts.path) == 0: return { 'status': '32', - 'meta': url.toDisplayString(), + 'meta': urllib.parse.urlunsplit(( + url_parts.scheme, + url_parts.netloc, + '/', + url_parts.query, + url_parts.fragment, + )) } context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE - with socket.create_connection((url.host(), url.port())) as sock: - with context.wrap_socket(sock, server_hostname=url.host()) as ssock: - ssock.sendall('gemini://{}{}\r\n'.format(url.host(), url.path()).encode('utf8')) + port = 1965 if url_parts.port is None else url_parts.port + with socket.create_connection((url_parts.hostname, port)) as sock: + with context.wrap_socket(sock, server_hostname=url_parts.hostname) as ssock: + ssock.sendall('gemini://{}{}\r\n'.format(url_parts.hostname, url_parts.path).encode('utf8')) fp = ssock.makefile(mode='rb') header = fp.readline(1027) status, meta = header.decode('utf8').split(None, 1) diff --git a/test_gemini.py b/test_gemini.py new file mode 100644 index 0000000..9d67f6e --- /dev/null +++ b/test_gemini.py @@ -0,0 +1,19 @@ +import unittest + +from gemini import urljoin + +class TestUrljoin(unittest.TestCase): + + def test_relative(self): + self.assertEqual(urljoin('gemini://example.com:1965/foo/', '/bar/'), + 'gemini://example.com:1965/bar/') + self.assertEqual(urljoin('gemini://example.com:1965/foo/', 'bar/'), + 'gemini://example.com:1965/foo/bar/') + self.assertEqual(urljoin('gemini://example.com:1965/foo/', 'baz.gem'), + 'gemini://example.com:1965/foo/baz.gem') + self.assertEqual(urljoin('gemini://example.com:1965/foo/bar.gem', '/bar/'), + 'gemini://example.com:1965/bar/') + self.assertEqual(urljoin('gemini://example.com:1965/foo/bar.gem', 'bar/'), + 'gemini://example.com:1965/foo/bar/') + self.assertEqual(urljoin('gemini://example.com:1965/foo/bar.gem', 'baz.gem'), + 'gemini://example.com:1965/foo/baz.gem') -- cgit v1.2.3