From 37bc08c699f48461be5e000b2da9212237a1ca0f Mon Sep 17 00:00:00 2001 From: JohnJamesUtley Date: Tue, 25 Apr 2023 16:01:03 -0400 Subject: [PATCH 1/4] Adds checks to ensure that bracketed hosts found by urlsplit are of IPv6 or IPvFuture format --- Lib/test/test_urlparse.py | 23 +++++++++++++++++++ Lib/urllib/parse.py | 18 ++++++++++++--- ...-04-26-09-54-25.gh-issue-103848.aDSnpR.rst | 2 ++ 3 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 80fb9e5cd2a445..8dfc1545387435 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1037,6 +1037,29 @@ def test_issue14072(self): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') + def test_splitting_bracketed_hosts(self): + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[important.com:8000]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123r.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v12ae]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123.]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af::2309::fae7:1234]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af:2309::fae7:1234:2342:438e:192.0.2.146]/Path?Query') + p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]/path?query') + self.assertEqual(p1.hostname, 'v6a.ip') + self.assertEqual(p1.username, 'user') + self.assertEqual(p1.path, '/path') + p2 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query') + self.assertEqual(p2.hostname, '0439:23af:2309::fae7%test') + self.assertEqual(p2.username, 'user') + self.assertEqual(p2.path, '/path') + p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query') + self.assertEqual(p3.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test') + self.assertEqual(p3.username, 'user') + self.assertEqual(p3.path, '/path') + def test_port_casting_failure_message(self): message = "Port could not be cast to integer value as 'oracle'" p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 5f95c5ff7f9c1c..5a7a136631afa3 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -33,6 +33,7 @@ import re import types import warnings +import ipaddress __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "urlsplit", "urlunsplit", "urlencode", "parse_qs", @@ -199,7 +200,7 @@ def _hostinfo(self): _, _, hostinfo = netloc.rpartition('@') _, have_open_br, bracketed = hostinfo.partition('[') if have_open_br: - hostname, _, port = bracketed.partition(']') + hostname, _, port = bracketed.rpartition(']') _, _, port = port.partition(':') else: hostname, _, port = hostinfo.partition(':') @@ -229,7 +230,7 @@ def _hostinfo(self): _, _, hostinfo = netloc.rpartition(b'@') _, have_open_br, bracketed = hostinfo.partition(b'[') if have_open_br: - hostname, _, port = bracketed.partition(b']') + hostname, _, port = bracketed.rpartition(b']') _, _, port = port.partition(b':') else: hostname, _, port = hostinfo.partition(b':') @@ -426,6 +427,15 @@ def _checknetloc(netloc): if c in netloc2: raise ValueError("netloc '" + netloc + "' contains invalid " + "characters under NFKC normalization") + +def _check_bracketed_host(hostname): + if hostname.startswith('v'): + if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", hostname): + raise ValueError(f"IPvFuture address is invalid") + else: + ip = ipaddress.ip_address(hostname) # Throws Value Error if not IPv6 or IPv4 + if isinstance(ip, ipaddress.IPv4Address): + raise ValueError(f"An IPv4 address cannot be in brackets") # typed=True avoids BytesWarnings being emitted during cache key # comparison since this API supports both bytes and str input. @@ -466,12 +476,14 @@ def urlsplit(url, scheme='', allow_fragments=True): break else: scheme, url = url[:i].lower(), url[i+1:] - if url[:2] == '//': netloc, url = _splitnetloc(url, 2) if (('[' in netloc and ']' not in netloc) or (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") + if '[' in netloc and ']' in netloc: + bracketed_host = netloc.partition('[')[2].rpartition(']')[0] + _check_bracketed_host(bracketed_host) if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: diff --git a/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst new file mode 100644 index 00000000000000..4ba17597bfe781 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst @@ -0,0 +1,2 @@ +Add checks to ensure that bracketed hosts found by urlsplit are of IPv6 or +IPvFuture format From 4057d356af14c998b48deae7bd23290e75dafcb2 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Tue, 9 May 2023 08:41:46 -0700 Subject: [PATCH 2/4] ReSTify NEWS. --- .../Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst index 4ba17597bfe781..81e5904aa6cca2 100644 --- a/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst +++ b/Misc/NEWS.d/next/Library/2023-04-26-09-54-25.gh-issue-103848.aDSnpR.rst @@ -1,2 +1,2 @@ -Add checks to ensure that bracketed hosts found by urlsplit are of IPv6 or -IPvFuture format +Add checks to ensure that ``[`` bracketed ``]`` hosts found by +:func:`urllib.parse.urlsplit` are of IPv6 or IPvFuture format. From 20335ce758704944aa6f1ba2a184ef5bc23a89c1 Mon Sep 17 00:00:00 2001 From: JohnJamesUtley Date: Tue, 9 May 2023 16:21:02 -0400 Subject: [PATCH 3/4] Splits bracketed host tests in two, replaces rpartition for host brackets, adds comments, and a new test --- Lib/test/test_urlparse.py | 5 ++++- Lib/urllib/parse.py | 10 ++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 8dfc1545387435..9272e32ac8f71f 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1037,7 +1037,7 @@ def test_issue14072(self): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') - def test_splitting_bracketed_hosts(self): + def test_invalid_bracketed_hosts(self): self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[192.0.2.146]/Path?Query') self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[important.com:8000]/Path?Query') self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123r.IP]/Path?Query') @@ -1047,6 +1047,9 @@ def test_splitting_bracketed_hosts(self): self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v]/Path?Query') self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af::2309::fae7:1234]/Path?Query') self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af:2309::fae7:1234:2342:438e:192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@]v6a.ip[/Path') + + def test_splitting_bracketed_hosts(self): p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]/path?query') self.assertEqual(p1.hostname, 'v6a.ip') self.assertEqual(p1.username, 'user') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 5a7a136631afa3..15d99ce89e5893 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -200,7 +200,7 @@ def _hostinfo(self): _, _, hostinfo = netloc.rpartition('@') _, have_open_br, bracketed = hostinfo.partition('[') if have_open_br: - hostname, _, port = bracketed.rpartition(']') + hostname, _, port = bracketed.partition(']') _, _, port = port.partition(':') else: hostname, _, port = hostinfo.partition(':') @@ -230,7 +230,7 @@ def _hostinfo(self): _, _, hostinfo = netloc.rpartition(b'@') _, have_open_br, bracketed = hostinfo.partition(b'[') if have_open_br: - hostname, _, port = bracketed.rpartition(b']') + hostname, _, port = bracketed.partition(b']') _, _, port = port.partition(b':') else: hostname, _, port = hostinfo.partition(b':') @@ -427,7 +427,9 @@ def _checknetloc(netloc): if c in netloc2: raise ValueError("netloc '" + netloc + "' contains invalid " + "characters under NFKC normalization") - + +# Valid bracketed hosts are defined in +# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/ def _check_bracketed_host(hostname): if hostname.startswith('v'): if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", hostname): @@ -482,7 +484,7 @@ def urlsplit(url, scheme='', allow_fragments=True): (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") if '[' in netloc and ']' in netloc: - bracketed_host = netloc.partition('[')[2].rpartition(']')[0] + bracketed_host = netloc.partition('[')[2].partition(']')[0] _check_bracketed_host(bracketed_host) if allow_fragments and '#' in url: url, fragment = url.split('#', 1) From 1783d437ae4a5827f930b5b8c362ec3828ce18ef Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Tue, 9 May 2023 16:53:54 -0700 Subject: [PATCH 4/4] remove trailing spaces --- Lib/test/test_urlparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 9272e32ac8f71f..73d7e12a965794 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1048,7 +1048,7 @@ def test_invalid_bracketed_hosts(self): self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af::2309::fae7:1234]/Path?Query') self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af:2309::fae7:1234:2342:438e:192.0.2.146]/Path?Query') self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@]v6a.ip[/Path') - + def test_splitting_bracketed_hosts(self): p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]/path?query') self.assertEqual(p1.hostname, 'v6a.ip')