iot: switch to Cloud Client (#2418)

* First part of Cloud client library migration, separating as this part affects tests

Author: gguuss

iot/api-client/http_example/cloudiot_http_example_test.py

@@ -74,10 +74,6 @@ def test_event(test_topic, capsys):
             'hello', 'event', _BASE_URL, project_id, cloud_region,
             registry_id, device_id, jwt_token))
 
-    manager.get_state(
-            service_account_json, project_id, cloud_region, registry_id,
-            device_id)
-
     manager.delete_device(
             service_account_json, project_id, cloud_region, registry_id,
             device_id)
@@ -87,7 +83,6 @@ def test_event(test_topic, capsys):
 
     out, _ = capsys.readouterr()
     assert 'format : RSA_X509_PEM' in out
-    assert 'State: {' in out
     assert '200' in out
 
 
@@ -125,8 +120,7 @@ def test_state(test_topic, capsys):
 
     out, _ = capsys.readouterr()
     assert 'format : RSA_X509_PEM' in out
-    assert 'State: {' in out
-    assert 'aGVsbG8=' in out
+    assert 'binary_data: "hello"' in out
     assert '200' in out
 
 
@@ -164,5 +158,4 @@ def test_config(test_topic, capsys):
 
     out, _ = capsys.readouterr()
     assert 'format : RSA_X509_PEM' in out
-    assert 'State: {' in out
     assert '"version": "1"' in out

iot/api-client/http_example/requirements.txt

@@ -2,6 +2,8 @@ cryptography==2.5
 google-api-python-client==1.7.8
 google-auth-httplib2==0.0.3
 google-auth==1.6.2
-google-cloud-pubsub==0.39.1
+google-cloud-iot==0.3.0
+google-cloud-pubsub==1.0.0
+grpc-google-iam-v1==0.12.3
 pyjwt==1.7.1
 requests==2.21.0

iot/api-client/manager/manager.py

@@ -35,6 +35,7 @@
 import sys
 import time
 
+from google.cloud import iot_v1
 from google.cloud import pubsub
 from google.oauth2 import service_account
 from googleapiclient import discovery
@@ -236,29 +237,38 @@ def get_device(
     """Retrieve the device with the given id."""
     # [START iot_get_device]
     print('Getting device')
-    client = get_client(service_account_json)
-    registry_name = 'projects/{}/locations/{}/registries/{}'.format(
-            project_id, cloud_region, registry_id)
+    client = iot_v1.DeviceManagerClient()
+    device_path = client.device_path(
+        project_id, cloud_region, registry_id, device_id)
 
-    device_name = '{}/devices/{}'.format(registry_name, device_id)
-    devices = client.projects().locations().registries().devices()
-    device = devices.get(name=device_name).execute()
+    device = client.get_device(device_path)
 
-    print('Id : {}'.format(device.get('id')))
-    print('Name : {}'.format(device.get('name')))
+    print('Id : {}'.format(device.id))
+    print('Name : {}'.format(device.name))
     print('Credentials:')
-    if device.get('credentials') is not None:
-        for credential in device.get('credentials'):
-            keyinfo = credential.get('publicKey')
-            print('\tcertificate: \n{}'.format(keyinfo.get('key')))
-            print('\tformat : {}'.format(keyinfo.get('format')))
-            print('\texpiration: {}'.format(credential.get('expirationTime')))
+
+    if device.credentials is not None:
+        for credential in device.credentials:
+            keyinfo = credential.public_key
+            print('\tcertificate: \n{}'.format(keyinfo.key))
+
+            if keyinfo.format == 4:
+                keyformat = 'ES256_X509_PEM'
+            elif keyinfo.format == 3:
+                keyformat = 'RSA_PEM'
+            elif keyinfo.format == 2:
+                keyformat = 'ES256_PEM'
+            elif keyinfo.format == 1:
+                keyformat = 'RSA_X509_PEM'
+            else:
+                keyformat = 'UNSPECIFIED_PUBLIC_KEY_FORMAT'
+            print('\tformat : {}'.format(keyformat))
+            print('\texpiration: {}'.format(credential.expiration_time))
 
     print('Config:')
-    print('\tdata: {}'.format(device.get('config').get('data')))
-    print('\tversion: {}'.format(device.get('config').get('version')))
-    print('\tcloudUpdateTime: {}'.format(device.get('config').get(
-            'cloudUpdateTime')))
+    print('\tdata: {}'.format(device.config.binary_data))
+    print('\tversion: {}'.format(device.config.version))
+    print('\tcloudUpdateTime: {}'.format(device.config.cloud_update_time))
 
     return device
     # [END iot_get_device]
@@ -269,17 +279,19 @@ def get_state(
         device_id):
     """Retrieve a device's state blobs."""
     # [START iot_get_device_state]
-    client = get_client(service_account_json)
-    registry_name = 'projects/{}/locations/{}/registries/{}'.format(
-            project_id, cloud_region, registry_id)
+    client = iot_v1.DeviceManagerClient()
+    device_path = client.device_path(
+        project_id, cloud_region, registry_id, device_id)
 
-    device_name = '{}/devices/{}'.format(registry_name, device_id)
-    devices = client.projects().locations().registries().devices()
-    state = devices.states().list(name=device_name, numStates=5).execute()
+    device = client.get_device(device_path)
+    print('Last state: {}'.format(device.state))
 
-    print('State: {}\n'.format(state))
+    print('State history')
+    states = client.list_device_states(device_path).device_states
+    for state in states:
+        print('State: {}'.format(state))
 
-    return state
+    return states
     # [END iot_get_device_state]
 
 
@@ -288,16 +300,13 @@ def list_devices(
     """List all devices in the registry."""
     # [START iot_list_devices]
     print('Listing devices')
-    registry_path = 'projects/{}/locations/{}/registries/{}'.format(
-            project_id, cloud_region, registry_id)
-    client = get_client(service_account_json)
-    devices = client.projects().locations().registries().devices(
-            ).list(parent=registry_path).execute().get('devices', [])
 
+    client = iot_v1.DeviceManagerClient()
+    registry_path = client.registry_path(project_id, cloud_region, registry_id)
+
+    devices = list(client.list_devices(parent=registry_path))
     for device in devices:
-        print('Device: {} : {}'.format(
-                device.get('numId'),
-                device.get('id')))
+        print('Device: {} : {}'.format(device.num_id, device.id))
 
     return devices
     # [END iot_list_devices]
@@ -307,16 +316,14 @@ def list_registries(service_account_json, project_id, cloud_region):
     """List all registries in the project."""
     # [START iot_list_registries]
     print('Listing Registries')
-    registry_path = 'projects/{}/locations/{}'.format(
-            project_id, cloud_region)
-    client = get_client(service_account_json)
-    registries = client.projects().locations().registries().list(
-        parent=registry_path).execute().get('deviceRegistries', [])
+    client = iot_v1.DeviceManagerClient()
+    parent = client.location_path(project_id, cloud_region)
 
+    registries = list(client.list_device_registries(parent))
     for registry in registries:
         print('id: {}\n\tname: {}'.format(
-            registry.get('id'),
-            registry.get('name')))
+            registry.id,
+            registry.name))
 
     return registries
     # [END iot_list_registries]

iot/api-client/manager/manager_test.py

@@ -43,11 +43,11 @@
 
 @pytest.fixture(scope="session", autouse=True)
 def clean_up_registries():
-    all_registries = manager.list_registries(
-        service_account_json, project_id, cloud_region)
+    all_registries = list(manager.list_registries(
+        service_account_json, project_id, cloud_region))
 
     for registry in all_registries:
-        registry_id = registry.get('id')
+        registry_id = registry.id
         if registry_id.find('test-registry-') == 0:
             time_str = registry_id[
                 registry_id.rfind('-') + 1: len(registry_id)]
@@ -61,19 +61,19 @@ def clean_up_registries():
                 client = manager.get_client(service_account_json)
                 gateways = client.projects().locations().registries().devices(
                     ).list(
-                        parent=registry.get('name'),
+                        parent=registry.name,
                         fieldMask='config,gatewayConfig'
                     ).execute().get('devices', [])
                 devices = client.projects().locations().registries().devices(
-                    ).list(parent=registry.get('name')).execute().get(
+                    ).list(parent=registry.name).execute().get(
                         'devices', [])
 
                 # Unbind devices from each gateway and delete
                 for gateway in gateways:
                     gateway_id = gateway.get('id')
                     bound = client.projects().locations().registries().devices(
                         ).list(
-                            parent=registry.get('name'),
+                            parent=registry.name,
                             gatewayListOptions_associationsGatewayId=gateway_id
                         ).execute()
                     if 'devices' in bound:
@@ -87,15 +87,15 @@ def clean_up_registries():
                                     parent=registry.get('name'),
                                     body=bind_request).execute()
                     gateway_name = '{}/devices/{}'.format(
-                        registry.get('name'), gateway_id)
+                        registry.name, gateway_id)
                     client.projects().locations().registries().devices(
                         ).delete(name=gateway_name).execute()
 
                 # Delete the devices
                 # Assumption is that the devices are not bound to gateways
                 for device in devices:
                     device_name = '{}/devices/{}'.format(
-                        registry.get('name'), device.get('id'))
+                        registry.name, device.get('id'))
                     print(device_name)
                     remove_device = True
                     try:
@@ -111,7 +111,7 @@ def clean_up_registries():
 
                 # Delete the old test registry
                 client.projects().locations().registries().delete(
-                    name=registry.get('name')).execute()
+                    name=registry.name).execute()
 
 
 @pytest.fixture(scope='module')
@@ -193,6 +193,9 @@ def test_add_delete_unauth_device(test_topic, capsys):
             service_account_json, project_id, cloud_region, registry_id,
             device_id)
 
+    manager.delete_registry(
+            service_account_json, project_id, cloud_region, registry_id)
+
     out, _ = capsys.readouterr()
     assert 'UNAUTH' in out
 
@@ -219,6 +222,9 @@ def test_add_config_unauth_device(test_topic, capsys):
             service_account_json, project_id, cloud_region, registry_id,
             device_id)
 
+    manager.delete_registry(
+            service_account_json, project_id, cloud_region, registry_id)
+
     out, _ = capsys.readouterr()
     assert 'Set device configuration' in out
     assert 'UNAUTH' in out
@@ -252,7 +258,6 @@ def test_add_delete_rs256_device(test_topic, capsys):
 
     out, _ = capsys.readouterr()
     assert 'format : RSA_X509_PEM' in out
-    assert 'State: {' in out
 
 
 def test_add_delete_es256_device(test_topic, capsys):
@@ -282,7 +287,6 @@ def test_add_delete_es256_device(test_topic, capsys):
 
     out, _ = capsys.readouterr()
     assert 'format : ES256_PEM' in out
-    assert 'State: {' in out
 
 
 def test_add_patch_delete_rs256(test_topic, capsys):

iot/api-client/manager/requirements.txt

@@ -4,7 +4,8 @@ gcp-devrel-py-tools==0.0.15
 google-api-python-client==1.7.8
 google-auth-httplib2==0.0.3
 google-auth==1.6.2
-google-cloud-pubsub==0.39.1
+google-cloud-iot==0.3.0
+google-cloud-pubsub==1.0.0
 oauth2client==4.1.3
 paho-mqtt==1.4.0
 pyjwt==1.7.1

iot/api-client/mqtt_example/cloudiot_mqtt_example_test.py

@@ -139,7 +139,7 @@ def test_state(test_topic, capsys):
 
     out, _ = capsys.readouterr()
     assert 'on_publish' in out
-    assert 'c3RhdGUgdGVzdA' in out
+    assert 'binary_data: "state test"' in out
 
 
 def test_config(test_topic, capsys):
@@ -354,6 +354,12 @@ def trigger_error(client):
                 service_account_json, project_id, cloud_region, registry_id,
                 device_id, gateway_id, num_messages, rsa_private_path,
                 'RS256', ca_cert_path, 'mqtt.googleapis.com', 443,
+                20, 42, trigger_error)
+    # Try to connect the gateway aagin on 8883
+    cloudiot_mqtt_example.listen_for_messages(
+                service_account_json, project_id, cloud_region, registry_id,
+                device_id, gateway_id, num_messages, rsa_private_path,
+                'RS256', ca_cert_path, 'mqtt.googleapis.com', 8883,
                 20, 15, trigger_error)
 
     # Clean up

iot/api-client/mqtt_example/requirements.txt

@@ -4,6 +4,8 @@ gcp-devrel-py-tools==0.0.15
 google-api-python-client==1.7.8
 google-auth-httplib2==0.0.3
 google-auth==1.6.2
-google-cloud-pubsub==0.39.1
+google-cloud-pubsub==1.0.0
+google-cloud-iot==0.3.0
+grpc-google-iam-v1==0.12.3
 pyjwt==1.7.1
 paho-mqtt==1.4.0