Added known issue to atempt to forestall python2 openssl bug reports

Added test certs to manifest so the demos work out of the box
pinned ruamel.yaml in install controls
added requirement for requests to use new security enhancements per best practice
tweaked security functions to make it clearer when the cert is failing on handshake

Author: Chaffelson

MANIFEST.in

@@ -4,6 +4,7 @@ include docs/history.rst
 include LICENSE
 include README.rst
 include requirements.txt
+include nipyapi/demo/resources/keys/*
 
 recursive-include nipyapi *
 recursive-exclude * *.py[co]

README.rst

@@ -105,4 +105,5 @@ Python Requirements
 -------------------
 
 | Python 2.7 or 3.6 supported, though other versions may work.
+| Tested on Ubuntu and OSX 10.11.x - Windows automated testing not attempted
 | Outside of the standard Python modules, we make use of lxml, DeepDiff and ruamel.yaml

docs/history.rst

@@ -18,6 +18,10 @@ History
 * Standardised bad user submission on AssertionError, bad API submission errors on ValueError, and general API errors on ApiException. This standard should flow forwards
 * Switched ruamel.yaml from >15 to <15 as advised in the project documentation, as >15 is not considered production ready
 
+**Known Issues**
+
+* Python2 environments with older versions of openssl may run into errors like 'SSLV3_ALERT_HANDSHAKE_FAILURE' when working in secured environments. This is not a NiPyApi bug, it's a problem with py2/openssl which is fixed by either upgrading openssl or moving to Python3 like you know you should
+
 **New Features**
 
 * Added support for working with secured NiFi environments, contributed by KevDoran
@@ -35,6 +39,7 @@ History
 * Implemented import/export to json/yaml in versioning
 * Added regression/backtesting for many functions going back through major release versions to NiFi-1.1.2. More details will be obvious from reading tests/conftest.py
 * Test suites now more reliably clean up after themselves when executed on long-running environments
+* Apparently logging is popular, so standard Python logging is now included
 
 **Other notes**
 

nipyapi/config.py

@@ -46,7 +46,6 @@
 long_max_wait = 120
 
 
-
 # --- Object Filters ------
 # This sets the mappings of where in the native datatype objects to find
 # particularly useful fields, like UUID or NAME.

nipyapi/demo/secure_connection.py

@@ -13,6 +13,9 @@
 
 log = logging.getLogger(__name__)
 log.setLevel(logging.INFO)
+logging.getLogger('nipyapi.utils').setLevel(logging.INFO)
+logging.getLogger('nipyapi.security').setLevel(logging.INFO)
+logging.getLogger('nipyapi.versioning').setLevel(logging.INFO)
 
 # Uncomment the block below to enable debug logging
 # nipyapi.config.nifi_config.debug=True
@@ -29,8 +32,9 @@
 secured_registry_url = 'https://localhost:18443/nifi-registry-api'
 secured_nifi_url = 'https://localhost:8443/nifi-api'
 
-host_certs_path = path.abspath(
-    nipyapi.config.PROJECT_ROOT_DIR + "/demo/resources/keys"
+host_certs_path = path.join(
+    nipyapi.config.PROJECT_ROOT_DIR,
+    "demo/resources/keys"
 )
 
 tls_env_vars = {
@@ -191,11 +195,12 @@ def bootstrap_nifi_access_policies():
 
 log.info("Creating Registry security context")
 nipyapi.utils.set_endpoint(secured_registry_url)
+log.info("Using demo certs from %s", host_certs_path)
 nipyapi.security.set_service_ssl_context(
     service='registry',
-    ca_file=host_certs_path + '/localhost-ts.pem',
-    client_cert_file=host_certs_path + '/client-cert.pem',
-    client_key_file=host_certs_path + '/client-key.pem',
+    ca_file=path.join(host_certs_path, 'localhost-ts.pem'),
+    client_cert_file=path.join(host_certs_path, 'client-cert.pem'),
+    client_key_file=path.join(host_certs_path, 'client-key.pem'),
     client_key_password='clientPassword'
 )
 log.info("Waiting for Registry to be ready for login")

nipyapi/security.py

@@ -195,7 +195,7 @@ def get_service_access_status(service='nifi', bool_response=False):
     Returns:
         (bool) if bool_response, else the Service Access Status of the User
     """
-    log.info("Called get_registry_access_status with args %s", locals())
+    log.info("Called get_service_access_status with args %s", locals())
     assert service in _valid_services
     assert isinstance(bool_response, bool)
     if bool_response:
@@ -204,12 +204,14 @@ def get_service_access_status(service='nifi', bool_response=False):
         log.debug("- bool_response is True, disabling urllib3 warnings")
         logging.getLogger('urllib3').setLevel(logging.ERROR)
     try:
-        return getattr(nipyapi, service).AccessApi().get_access_status()
+        out = getattr(nipyapi, service).AccessApi().get_access_status()
+        log.info("Got server response, returning")
+        return out
     except urllib3.exceptions.MaxRetryError as e:
         log.debug("- Caught exception %s", type(e))
         if bool_response:
-            log.debug("- bool_response is True, returning False instead of"
-                      " raising exception")
+            log.debug("Connection failed with error %s and bool_response is "
+                      "True, returning False", e)
             return False
         log.debug("- bool_response is False, raising Exception")
         raise e
@@ -507,7 +509,8 @@ def set_service_ssl_context(
             ssl_context.load_cert_chain(
                 certfile=client_cert_file,
                 keyfile=client_key_file,
-                password=client_key_password)
+                password=client_key_password
+            )
         except FileNotFoundError as e:
             raise FileNotFoundError(
                 "Unable to read keyfile {0} or certfile {1}, error was "

nipyapi/utils.py

@@ -329,29 +329,29 @@ def start_docker_containers(docker_containers, network_name='demo'):
     Returns: Nothing
 
     """
-    log.info("- Creating Docker client using Environment Variables")
+    log.info("Creating Docker client using Environment Variables")
     d_client = docker.from_env()
 
     for target in docker_containers:
         assert isinstance(target, DockerContainer)
 
     # Pull relevant Images
-    log.info("- Pulling relevant Docker Images")
+    log.info("Pulling relevant Docker Images")
     for image in set([(c.image_name + ':' + c.image_tag)
                       for c in docker_containers]):
-        log.info("- - Pulling %s", image)
+        log.info("Pulling %s", image)
         d_client.images.pull(image)
 
     # Clear previous containers
-    log.info("- Clearing previous containers for this demo")
+    log.info("Clearing previous containers for this demo")
     d_clear_list = [li for li in d_client.containers.list(all=True)
                     if li.name in [i.name for i in docker_containers]]
     for c in d_clear_list:
-        log.info("- - Removing old container %s", c.name)
+        log.info("Removing old container %s", c.name)
         c.remove(force=True)
 
     # Deploy/Get Network
-    log.info("- Getting Docker bridge network")
+    log.info("Getting Docker bridge network")
     d_n_list = [li for li in d_client.networks.list()
                 if network_name in li.name]
     if not d_n_list:
@@ -364,13 +364,13 @@ def start_docker_containers(docker_containers, network_name='demo'):
         raise EnvironmentError("Too many test networks found")
     else:
         d_network = d_n_list[0]
-    log.info("- - Using Docker network: %s", d_network.name)
+    log.info("Using Docker network: %s", d_network.name)
 
     # Deploy Containers
-    log.info("- Starting relevant Docker Containers")
+    log.info("Starting relevant Docker Containers")
     c_hooks = {}
     for c in docker_containers:
-        log.info("- - Starting Container %s", c.name)
+        log.info("Starting Container %s", c.name)
         c_hooks[c.name] = d_client.containers.run(
             image=c.image_name + ':' + c.image_tag,
             detach=True,

nipyapi/versioning.py

@@ -582,8 +582,8 @@ def export_flow_version(bucket_id, flow_id, version=None, file_path=None,
     Args:
         bucket_id (str): the UUID of the bucket containing the Flow
         flow_id (str): the UUID of the Flow to be retrieved from the Bucket
-        version (Optional [None, Str]): 'None' to retrieve the latest version, or a version
-            number as a string to get that version
+        version (Optional [None, Str]): 'None' to retrieve the latest version,
+            or a version number as a string to get that version
         file_path (str): The path and filename to write to. Defaults to None
             which returns the serialised obj
         mode (str): 'json' or 'yaml' to specific the encoding format

requirements.txt

@@ -14,4 +14,4 @@ deepdiff>=3.3.0
 
 # Demo deployment automation
 docker>=2.5.1
-requests>=2.18
+requests[security]>=2.18

setup.py

@@ -18,9 +18,9 @@
     'lxml',  # Required for parsing NiFi Templates
     'deepdiff',  # Required for comparing configurations
     'six',  # Required for managing Python version compatibility
-    'ruamel.yaml',  # Required for parsing Json/Yaml consistently
+    'ruamel.yaml==0.14.12',  # Required for parsing Json/Yaml consistently
     'docker',  # Used to deploy demo assemblies
-    'requests'  # Used in utils functions
+    'requests[security]'  # Used in utils functions, security extras for Py2
 ]
 
 setup_requirements = [