Update NiPyAPI to be roughly forward compatible with NiFi-2.x line. (#368)

* Instruct pyup to ignore pytest in requirements txt
* instruct pytest to not report insecure http requests to localhost for docker testing
* Templates are deprecated in nifi-2x, remove from tests for newer versions
* add function to check registry version using newer aboutapi() call
* update utils.check_version to ignore any non-semver elements of a version string and use newer registry version api call if available
* Update registry client creation to handle nifi-2x structure
* Update Testing to treat NiFI-2.x as regression target
* Deprecate very old nifi-1x versions from regression testing
* Fix get processor test to use a processor available in all nifi versions, GenerateFlowfile
* Update readme about NiFi2.x support.
* Switch 'latest' dockerfile to use 'latest' nifi tags.

Author: Chaffelson

README.rst

@@ -96,7 +96,12 @@ Background and Documentation
 NiFi Version Support
 --------------------
 
-| Currently we are testing against NiFi versions 1.1.2 - 1.27.0, and NiFi-Registry versions 0.1.0 - 1.27.0.
+| Currently we are testing against NiFi versions 1.9.2 - 1.27.0, and NiFi-Registry versions 0.3.0 - 1.27.0.
+
+| We have also tested against the latest 2.0.0-M4 release candidates using the 1.x SDK and have found that basic functionality works as expected.
+| Apache NiFi offers no compatibility guarantees between major versions, and while many functions may work the same, you should test carefully for your specific use case.
+| In future we will create a specific branch of NiPyAPI for NiFi 2.x SDKs, and maintain separate NiFi 1.x and NiFi 2.x clients.
+
 | If you find a version compatibility problem please raise an `issue <https://github.com/Chaffelson/nipyapi/issues>`_
 
 Python Support

nipyapi/canvas.py

@@ -402,10 +402,11 @@ def delete_process_group(process_group, force=False, refresh=True):
                         delete_controller(x, True)
                         removed_controllers_id.append(x.component.id)
 
-            # Remove templates
-            for template in nipyapi.templates.list_all_templates(native=False):
-                if target.id == template.template.group_id:
-                    nipyapi.templates.delete_template(template.id)
+            # Templates are not supported in NiFi 2.x
+            if nipyapi.utils.check_version('2', service='nifi') < 0:
+                for template in nipyapi.templates.list_all_templates(native=False):
+                    if target.id == template.template.group_id:
+                        nipyapi.templates.delete_template(template.id)
             # have to refresh revision after changes
             target = nipyapi.nifi.ProcessGroupsApi().get_process_group(pg_id)
             return nipyapi.nifi.ProcessGroupsApi().remove_process_group(

nipyapi/system.py

@@ -58,3 +58,13 @@ def get_nifi_version_info():
     """
     diags = get_system_diagnostics()
     return diags.system_diagnostics.aggregate_snapshot.version_info
+
+
+def get_registry_version_info():
+    """
+    Returns the version information of the connected NiFi Registry instance
+
+    Returns (VersionInfoDTO):
+    """
+    details = nipyapi.registry.AboutApi().get_version()
+    return details.registry_about_version

nipyapi/utils.py

@@ -8,7 +8,6 @@
 from __future__ import absolute_import, unicode_literals
 import logging
 import json
-import re
 import io
 import time
 from copy import copy
@@ -479,17 +478,6 @@ def start_docker_containers(docker_containers, network_name='demo'):
         ))
 
 
-def strip_snapshot(java_version):
-    """
-    Strips the -SNAPSHOT suffix from a version string
-
-    Args:
-        java_version (str): the version string
-    """
-    assert isinstance(java_version, six.string_types)
-    return re.sub("-SNAPSHOT", "", java_version)
-
-
 def check_version(base, comparator=None, service='nifi',
                   default_version='0.2.0'):
     """
@@ -511,36 +499,43 @@ def check_version(base, comparator=None, service='nifi',
     Returns (int): -1/0/1 if base is lower/equal/newer than comparator
 
     """
+
+    def strip_version_string(version_string):
+        # Reduces the string to only the major.minor.patch version
+        return '.'.join(version_string.split('-')[0].split('.')[:3])
+
     assert isinstance(base, six.string_types)
     assert comparator is None or isinstance(comparator, six.string_types)
     assert service in ['nifi', 'registry']
-    ver_a = version.parse(base)
+    ver_a = version.parse(strip_version_string(base))
     if comparator:
-        # if b is set, we compare the passed versions
-        comparator = strip_snapshot(comparator)
-        ver_b = version.parse(comparator)
+        ver_b = version.parse(strip_version_string(comparator))
     elif service == 'registry':
         try:
-            config = nipyapi.config.registry_config
-            if config.api_client is None:
-                config.api_client = nipyapi.registry.ApiClient()
-            reg_swagger_def = config.api_client.call_api(
-                resource_path='/swagger/swagger.json',
-                method='GET', _preload_content=False,
-                auth_settings=['tokenAuth', 'Authorization']
-            )
-            reg_json = load(reg_swagger_def[0].data)
-            ver_b = version.parse(reg_json['info']['version'])
+            reg_ver = nipyapi.system.get_registry_version_info()
+            ver_b = version.parse(strip_version_string(reg_ver))
         except nipyapi.registry.rest.ApiException:
             log.warning(
-                "Unable to get registry swagger.json, assuming version %s",
-                default_version)
-            ver_b = version.parse(default_version)
+                "Unable to get registry version, trying swagger.json")
+            try:
+                config = nipyapi.config.registry_config
+                if config.api_client is None:
+                    config.api_client = nipyapi.registry.ApiClient()
+                reg_swagger_def = config.api_client.call_api(
+                    resource_path='/swagger/swagger.json',
+                    method='GET', _preload_content=False,
+                    auth_settings=['tokenAuth', 'Authorization']
+                )
+                reg_json = load(reg_swagger_def[0].data)
+                ver_b = version.parse(reg_json['info']['version'])
+            except nipyapi.registry.rest.ApiException:
+                log.warning(
+                    "Can't get registry swagger.json, assuming version %s",
+                    default_version)
+                ver_b = version.parse(default_version)
     else:
-        # Working with NiFi
         ver_b = version.parse(
-            strip_snapshot(
-                # This call currently only supports NiFi
+            strip_version_string(
                 nipyapi.system.get_nifi_version_info().ni_fi_version
             )
         )

nipyapi/versioning.py

@@ -27,7 +27,7 @@
 log = logging.getLogger(__name__)
 
 
-def create_registry_client(name, uri, description):
+def create_registry_client(name, uri, description, reg_type=None):
     """
     Creates a Registry Client in the NiFi Controller Services
 
@@ -42,14 +42,28 @@ def create_registry_client(name, uri, description):
     assert isinstance(uri, six.string_types) and uri is not False
     assert isinstance(name, six.string_types) and name is not False
     assert isinstance(description, six.string_types)
+    if nipyapi.utils.check_version('2', service='nifi') > 0:
+        component = {
+            'uri': uri,
+            'name': name,
+            'description': description
+        }
+    else:
+        component = {
+            'name': name,
+            'description': description,
+            'type': (
+                reg_type or
+                'org.apache.nifi.registry.flow.NifiRegistryFlowRegistryClient'),
+            'properties': {
+                'url': uri
+            }
+        }
+
     with nipyapi.utils.rest_exceptions():
         return nipyapi.nifi.ControllerApi().create_flow_registry_client(
             body={
-                'component': {
-                    'uri': uri,
-                    'name': name,
-                    'description': description
-                },
+                'component': component,
                 'revision': {
                     'version': 0
                 }

requirements_dev.txt

@@ -14,7 +14,7 @@ tox>=3.28.0
 flake8>=3.6.0
 coverage>=4.4.1
 coveralls>=1.2.0
-pytest  # handled in setup.py
+pytest  # pyup: ignore
 nose>=1.3.7
 pluggy>=0.3.1
 pylint>=1.7.4

resources/docker/latest/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   nifi:
-    image: apache/nifi:1.27.0
+    image: apache/nifi:latest
     container_name: nifi
     hostname: nifi
     ports:
@@ -9,7 +9,7 @@ services:
       - SINGLE_USER_CREDENTIALS_USERNAME=nobel
       - SINGLE_USER_CREDENTIALS_PASSWORD=supersecret1!
   registry:
-    image: apache/nifi-registry:1.27.0
+    image: apache/nifi-registry:latest
     container_name: registry
     hostname: registry
     ports:

resources/docker/secure/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   secure-nifi:
-    image: apache/nifi:1.27.0
+    image: apache/nifi:latest
     container_name: secure-nifi
     hostname: secure-nifi
     ports:
@@ -25,7 +25,7 @@ services:
       - LDAP_URL=ldap://ldap.forumsys.com:389
       - NIFI_WEB_PROXY_HOST=localhost:9443,localhost:8443
   secure-registry:
-    image: apache/nifi-registry:1.27.0
+    image: apache/nifi-registry:latest
     container_name: secure-registry
     hostname: secure-registry
     ports:

resources/docker/tox-full/docker-compose.yml

@@ -1,45 +1,29 @@
 services:
-  nifi-112:
-    image: chaffelson/nifi:1.1.2
-    container_name: nifi-112
-    hostname: nifi-112
-    ports:
-      - "10112:8080"
-  nifi-120:
-    image: apache/nifi:1.2.0
-    container_name: nifi-120
-    hostname: nifi-120
-    ports:
-      - "10120:8080"
-  nifi-180:
-    image: apache/nifi:1.8.0
-    container_name: nifi-180
-    hostname: nifi-180
-    ports:
-      - "10180:8080"
   nifi-192:
     image: apache/nifi:1.9.2
     container_name: nifi-192
     hostname: nifi-192
     ports:
       - "10192:8080"
-  nifi:
+  nifi-127:
     image: apache/nifi:1.27.0
+    container_name: nifi-127
+    hostname: nifi-127
+    ports:
+      - "10127:10127"
+    environment:
+      - SINGLE_USER_CREDENTIALS_USERNAME=nobel
+      - SINGLE_USER_CREDENTIALS_PASSWORD=supersecret1!
+      - NIFI_WEB_HTTPS_PORT=10127
+  nifi:
+    image: apache/nifi:2.0.0-M4
     container_name: nifi
     hostname: nifi
     ports:
       - "8443:8443"
     environment:
       - SINGLE_USER_CREDENTIALS_USERNAME=nobel
       - SINGLE_USER_CREDENTIALS_PASSWORD=supersecret1!
-  registry-010:
-    image: apache/nifi-registry:0.1.0
-    container_name: registry-010
-    hostname: registry-010
-    ports:
-      - "18010:18010"
-    environment:
-      - NIFI_REGISTRY_WEB_HTTP_PORT=18010
   registry-030:
     image: apache/nifi-registry:0.3.0
     container_name: registry-030
@@ -48,8 +32,16 @@ services:
       - "18030:18030"
     environment:
       - NIFI_REGISTRY_WEB_HTTP_PORT=18030
-  registry:
+  registry-127:
     image: apache/nifi-registry:1.27.0
+    container_name: registry-127
+    hostname: registry-127
+    ports:
+      - "18127:18127"
+    environment:
+      - NIFI_REGISTRY_WEB_HTTP_PORT=18127
+  registry:
+    image: apache/nifi-registry:2.0.0-M4
     container_name: registry
     hostname: registry
     ports:

resources/docker/tox-full/readme.md

@@ -14,12 +14,6 @@ Stop a cluster:
 
 - ```docker-compose stop```
 
-## Environment Details
-
-- Apache NiFi 1.2.0 available on port 10120
-- Apache NiFi 1.4.0 available on port 10140
-- Apache NiFi 1.5.0 available on port 8080
-- Apache NiFi-Registry 0.1.0 available on port 18080
 
 ### Notes
-Where possible the latest version will be available on the defaul port, with older versions on identifiable ports for back testing
+Where possible the latest version will be available on the default port, with older versions on identifiable ports for back testing