fix TLS validation for requirements.txt

Previously, due to a probable typo in the code for importing a
requirements file to create a new pipenv project, SSL/TLS validation was
disabled by default for any package index servers specified in the
requirements file with the `--index-url` or `--extra-index-url` options.

In addition, `--trusted-host` options in the requirements file would not
work as intended, because any host or host:port pair provided with these
options was incorrectly being matched against the full URLs of the
configured index server(s) (i.e. including the scheme, path, etc.),
instead of extracting and comparing with the host and port parts only,
as intended.

This PR fixes both of these issues, flipping the existing behavior to
require SSL/TLS validation by default, and optionally allowing TLS
validation to be disabled explicitly for specific host:port with the
`--trusted-host` option if provided.

Author: milo-minderbinder

pipenv/core.py

@@ -17,7 +17,7 @@
 from pipenv.patched import crayons
 from pipenv.utils import (
     cmd_list_to_shell, convert_deps_to_pip, create_spinner, download_file,
-    find_python, get_canonical_names, get_source_list, is_pinned,
+    find_python, get_canonical_names, get_host_and_port, get_source_list, is_pinned,
     is_python_command, is_required_version, is_star, is_valid_url,
     parse_indexes, pep423_name, prepare_pip_source_args, proper_case,
     python_version, run_command, subprocess_run, venv_resolve_deps
@@ -169,7 +169,7 @@ def import_requirements(project, r=None, dev=False):
         if extra_index:
             indexes.append(extra_index)
         if trusted_host:
-            trusted_hosts.append(trusted_host)
+            trusted_hosts.append(get_host_and_port(trusted_host))
     indexes = sorted(set(indexes))
     trusted_hosts = sorted(set(trusted_hosts))
     reqs = [install_req_from_parsed_requirement(f) for f in parse_requirements(r, session=pip_requests)]
@@ -185,8 +185,13 @@ def import_requirements(project, r=None, dev=False):
             else:
                 project.add_package_to_pipfile(str(package.req), dev=dev)
     for index in indexes:
-        trusted = index in trusted_hosts
-        project.add_index_to_pipfile(index, verify_ssl=trusted)
+        # don't require HTTPS for trusted hosts (see: https://pip.pypa.io/en/stable/cli/pip/#cmdoption-trusted-host)
+        host_and_port = get_host_and_port(index)
+        require_valid_https = not any((v in trusted_hosts for v in (
+            host_and_port,
+            host_and_port.partition(':')[0],  # also check if hostname without port is in trusted_hosts
+        )))
+        project.add_index_to_pipfile(index, verify_ssl=require_valid_https)
     project.recase_pipfile()
 
 

pipenv/utils.py

@@ -1643,6 +1643,30 @@ def get_url_name(url):
     return urllib3_util.parse_url(url).host
 
 
+def get_host_and_port(url):
+    """Get the host, or the host:port pair if port is explicitly included, for the given URL.
+
+    Examples:
+    >>> get_host_and_port('example.com')
+    'example.com'
+    >>> get_host_and_port('example.com:443')
+    'example.com:443'
+    >>> get_host_and_port('http://example.com')
+    'example.com'
+    >>> get_host_and_port('https://example.com/')
+    'example.com'
+    >>> get_host_and_port('https://example.com:8081')
+    'example.com:8081'
+    >>> get_host_and_port('ssh://example.com')
+    'example.com'
+
+    :param url: the URL string to parse
+    :return: a string with the host:port pair if the URL includes port number explicitly; otherwise, returns host only
+    """
+    url = urllib3_util.parse_url(url)
+    return '{}:{}'.format(url.host, url.port) if url.port else url.host
+
+
 def get_canonical_names(packages):
     """Canonicalize a list of packages and return a set of canonical names"""
     from .vendor.packaging.utils import canonicalize_name

tests/unit/test_utils.py

@@ -142,7 +142,7 @@ def test_convert_deps_to_pip_unicode():
     ("--extra-index-url=https://example.com/simple/", (None, "https://example.com/simple/", None, [])),
     ("--trusted-host=example.com", (None, None, "example.com", [])),
     ("# -i https://example.com/simple/", (None, None, None, [])),
-    ("requests", (None, None, None, ["requests"]))
+    ("requests # -i https://example.com/simple/", (None, None, None, ["requests"])),
 ])
 @pytest.mark.utils
 def test_parse_indexes(line, result):