use stdlib url parsing in gemini module

7 files changed, 189 insertions, 34 deletions

diff --git a/.pylintrc b/.pylintrc
index 0ace322..5342c9f 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -139,7 +139,9 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        c-extension-no-member
+        c-extension-no-member,
+        missing-module-docstring,
+        missing-function-docstring
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -423,7 +425,8 @@ good-names=i,
            k,
            ex,
            Run,
-           _
+           _,
+           fp
 
 # Good variable names regexes, separated by a comma. If names match any regex,
 # they will always be accepted
diff --git a/Pipfile b/Pipfile
index d087bf0..c1178de 100644
--- a/Pipfile
+++ b/Pipfile
@@ -5,6 +5,8 @@ verify_ssl = true
 
 [dev-packages]
 pylint = "*"
+pytest = "*"
+mypy = "*"
 
 [packages]
 pyside2 = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index f071e66..71b7109 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "6ab125c809493f48b2455c80c8297db7ea44a5f654424211b1a3538067433d5b"
+            "sha256": "3218b2f6e241e522a76bfce33f6cea9b42fe7a8127340096e0a20effd637aa6f"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -52,6 +52,20 @@
             ],
             "version": "==2.4.2"
         },
+        "attrs": {
+            "hashes": [
+                "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594",
+                "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"
+            ],
+            "version": "==20.2.0"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437",
+                "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69"
+            ],
+            "version": "==1.0.1"
+        },
         "isort": {
             "hashes": [
                 "sha256:92533892058de0306e51c88f22ece002a209dc8e80288aa3cec6d443060d584f",
@@ -92,6 +106,61 @@
             ],
             "version": "==0.6.1"
         },
+        "more-itertools": {
+            "hashes": [
+                "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20",
+                "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c"
+            ],
+            "version": "==8.5.0"
+        },
+        "mypy": {
+            "hashes": [
+                "sha256:2c6cde8aa3426c1682d35190b59b71f661237d74b053822ea3d748e2c9578a7c",
+                "sha256:3fdda71c067d3ddfb21da4b80e2686b71e9e5c72cca65fa216d207a358827f86",
+                "sha256:5dd13ff1f2a97f94540fd37a49e5d255950ebcdf446fb597463a40d0df3fac8b",
+                "sha256:6731603dfe0ce4352c555c6284c6db0dc935b685e9ce2e4cf220abe1e14386fd",
+                "sha256:6bb93479caa6619d21d6e7160c552c1193f6952f0668cdda2f851156e85186fc",
+                "sha256:81c7908b94239c4010e16642c9102bfc958ab14e36048fa77d0be3289dda76ea",
+                "sha256:9c7a9a7ceb2871ba4bac1cf7217a7dd9ccd44c27c2950edbc6dc08530f32ad4e",
+                "sha256:a4a2cbcfc4cbf45cd126f531dedda8485671545b43107ded25ce952aac6fb308",
+                "sha256:b7fbfabdbcc78c4f6fc4712544b9b0d6bf171069c6e0e3cb82440dd10ced3406",
+                "sha256:c05b9e4fb1d8a41d41dec8786c94f3b95d3c5f528298d769eb8e73d293abc48d",
+                "sha256:d7df6eddb6054d21ca4d3c6249cae5578cb4602951fd2b6ee2f5510ffb098707",
+                "sha256:e0b61738ab504e656d1fe4ff0c0601387a5489ca122d55390ade31f9ca0e252d",
+                "sha256:eff7d4a85e9eea55afa34888dfeaccde99e7520b51f867ac28a48492c0b1130c",
+                "sha256:f05644db6779387ccdb468cc47a44b4356fc2ffa9287135d05b70a98dc83b89a"
+            ],
+            "index": "pypi",
+            "version": "==0.782"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8",
+                "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"
+            ],
+            "version": "==20.4"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
+                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
+            ],
+            "version": "==0.13.1"
+        },
+        "py": {
+            "hashes": [
+                "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2",
+                "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"
+            ],
+            "version": "==1.9.0"
+        },
         "pylint": {
             "hashes": [
                 "sha256:bb4a908c9dadbc3aac18860550e870f58e1a02c9f2c204fdf5693d73be061210",
@@ -100,6 +169,21 @@
             "index": "pypi",
             "version": "==2.6.0"
         },
+        "pyparsing": {
+            "hashes": [
+                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
+                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
+            ],
+            "version": "==2.4.7"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:85228d75db9f45e06e57ef9bf4429267f81ac7c0d742cc9ed63d09886a9fe6f4",
+                "sha256:8b6007800c53fdacd5a5c192203f4e531eb2a1540ad9c752e052ec0f7143dbad"
+            ],
+            "index": "pypi",
+            "version": "==6.0.1"
+        },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
@@ -114,6 +198,40 @@
             ],
             "version": "==0.10.1"
         },
+        "typed-ast": {
+            "hashes": [
+                "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
+                "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
+                "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
+                "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
+                "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
+                "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
+                "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
+                "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
+                "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
+                "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
+                "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
+                "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
+                "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
+                "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
+                "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
+                "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
+                "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
+                "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
+                "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
+                "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
+                "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
+            ],
+            "version": "==1.4.1"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
+                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
+                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+            ],
+            "version": "==3.7.4.3"
+        },
         "wrapt": {
             "hashes": [
                 "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"
diff --git a/TODO.md b/TODO.md
new file mode 100644
index 0000000..837dd54
--- /dev/null
+++ b/TODO.md
@@ -0,0 +1,10 @@
+ - [ ] open non-gemini links externally and display differently
+ - [ ] keyboard navigation
+   - vim-style
+   - scroll
+   - highlight and follow links
+   - forward and back
+   - url bar
+ - [ ] search in page
+ - [ ] work with search pages
+ - [x] stdlib url parsing in gemini module
diff --git a/browser.py b/browser.py
index 58b650c..21e9293 100755
--- a/browser.py
+++ b/browser.py
@@ -20,42 +20,43 @@ class GViewport(QtWidgets.QTextBrowser):
     def mouseMoveEvent(self, event):
         cur = self.cursorForPosition(event.localPos().toPoint())
         hover_url = cur.charFormat().anchorHref()
-        hover_url = gemini.absolute_url(self._current_url, QtCore.QUrl(hover_url))
+        hover_url = gemini.urljoin(self._current_url, hover_url)
         if hover_url != self._hover_url:
             self._hover_url = hover_url
-            self.hoverUrlChanged.emit(self._hover_url.toString())
+            self.hoverUrlChanged.emit(self._hover_url)
         return super().mouseMoveEvent(event)
 
     def loadResource(self, type_, url):
+        current_url = QtCore.QUrl(self._current_url)
         if self._last_redirect[0].toString() == url.toString():
             gem = self._last_redirect[1]
         else:
             if not url.scheme():
-                url.setScheme(self._current_url.scheme())
+                url.setScheme(current_url.scheme())
             if not url.host():
-                url.setHost(self._current_url.host())
+                url.setHost(current_url.host())
             if url.port() == -1:
                 url.setPort(1965)
             if not url.path().startswith('/'):
-                url.setPath(self._current_url.path().rsplit('/', 1)[0] + '/' + url.path())
-            gem = gemini.get(gemini.absolute_url(self._current_url, url))
+                url.setPath(current_url.path().rsplit('/', 1)[0] + '/' + url.path())
+            gem = gemini.get(gemini.urljoin(current_url.toString(), url.toString()))
         if 'body' in gem:
             html = gemini.gem2html(gem['body'])
         else:
             html = '<h1>{} {}</h1>'.format(gem['status'], gem['meta'])
-        self._current_url = url
+        self._current_url = url.toString()
         return html
 
     def setSource(self, url):
         if url.scheme() != 'gemini':
             return
-        gem = gemini.get(gemini.absolute_url(self._current_url, url))
+        gem = gemini.get(gemini.urljoin(self._current_url, url.toString()))
         while gem['status'][0] == '3':
             url = QtCore.QUrl(gem['meta'])
             if url.port() == 1965:
                 url.setPort(-1)
             print('redirect: {}'.format(url))
-            gem = gemini.get(gemini.absolute_url(self._current_url, url))
+            gem = gemini.get(gemini.urljoin(self._current_url, url.toString()))
         self._last_redirect = (url, gem)
         if url.port() == 1965:
             url.setPort(-1)
diff --git a/gemini.py b/gemini.py
index c0d09c1..58f3358 100644
--- a/gemini.py
+++ b/gemini.py
@@ -1,10 +1,12 @@
+import re
 import socket
 import ssl
+import urllib.parse
 
-def htmlescape(text):
+def htmlescape(text: str) -> str:
     return text.replace('<', '&lt;').replace('>', '&gt;')
 
-def gem2html(gem):
+def gem2html(gem: str) -> str:
     html = []
     state = 'text'
     blanklines = 0
@@ -69,33 +71,33 @@ def gem2html(gem):
                 html.append('<p>{}</p>'.format(htmlescape(line)))
     return '\n'.join(html)
 
-def absolute_url(base, url):
-    """
-    modifies `url` in place
-    """
-    if not url.scheme():
-        url.setScheme(base.scheme())
-    if not url.host():
-        url.setHost(base.host())
-    if not url.path().startswith('/'):
-        url.setPath(base.path().rsplit('/', 1)[0] + '/' + url.path())
-    if url.port() == -1:
-        url.setPort(1965)
-    return url
+def urljoin(base: str, url: str) -> str:
+    if base is None:
+        return url
+    base = re.sub('^gemini:', 'http:', base)
+    url = re.sub('^gemini:', 'http:', url)
+    return re.sub('^http:', 'gemini:', urllib.parse.urljoin(base, url))
 
-def get(url):
-    if len(url.path()) == 0:
-        url.setPath('/')
+def get(url: str) -> dict:
+    url_parts = urllib.parse.urlsplit(url)
+    if len(url_parts.path) == 0:
         return {
             'status': '32',
-            'meta': url.toDisplayString(),
+            'meta': urllib.parse.urlunsplit((
+                url_parts.scheme,
+                url_parts.netloc,
+                '/',
+                url_parts.query,
+                url_parts.fragment,
+            ))
         }
     context = ssl.create_default_context()
     context.check_hostname = False
     context.verify_mode = ssl.CERT_NONE
-    with socket.create_connection((url.host(), url.port())) as sock:
-        with context.wrap_socket(sock, server_hostname=url.host()) as ssock:
-            ssock.sendall('gemini://{}{}\r\n'.format(url.host(), url.path()).encode('utf8'))
+    port = 1965 if url_parts.port is None else url_parts.port
+    with socket.create_connection((url_parts.hostname, port)) as sock:
+        with context.wrap_socket(sock, server_hostname=url_parts.hostname) as ssock:
+            ssock.sendall('gemini://{}{}\r\n'.format(url_parts.hostname, url_parts.path).encode('utf8'))
             fp = ssock.makefile(mode='rb')
             header = fp.readline(1027)
             status, meta = header.decode('utf8').split(None, 1)
diff --git a/test_gemini.py b/test_gemini.py
new file mode 100644
index 0000000..9d67f6e
--- /dev/null
+++ b/test_gemini.py
@@ -0,0 +1,19 @@
+import unittest
+
+from gemini import urljoin
+
+class TestUrljoin(unittest.TestCase):
+
+    def test_relative(self):
+        self.assertEqual(urljoin('gemini://example.com:1965/foo/', '/bar/'),
+                         'gemini://example.com:1965/bar/')
+        self.assertEqual(urljoin('gemini://example.com:1965/foo/', 'bar/'),
+                         'gemini://example.com:1965/foo/bar/')
+        self.assertEqual(urljoin('gemini://example.com:1965/foo/', 'baz.gem'),
+                         'gemini://example.com:1965/foo/baz.gem')
+        self.assertEqual(urljoin('gemini://example.com:1965/foo/bar.gem', '/bar/'),
+                         'gemini://example.com:1965/bar/')
+        self.assertEqual(urljoin('gemini://example.com:1965/foo/bar.gem', 'bar/'),
+                         'gemini://example.com:1965/foo/bar/')
+        self.assertEqual(urljoin('gemini://example.com:1965/foo/bar.gem', 'baz.gem'),
+                         'gemini://example.com:1965/foo/baz.gem')