Merge pull request #96 from GoogleCloudPlatform/test-cleanup

Cleaning up base test case

Author: Jonathan Wayne Parrott

CONTRIBUTING.md

@@ -81,16 +81,22 @@ If you want to run the Google App Engine tests, you will need:
 
         $ export GAE_PYTHONPATH=<path your AppeEngine sdk>
 
-To run the bigquery tests, you'll need to create a bigquery dataset:
+To run the bigquery tests:
 
 * Create a dataset in your project named `test_dataset`.
-* Create a table named `test_table2`, upload ``tests/resources/data.csv`` and give it the following schema:
 
-        Name STRING
-        Age INTEGER
-        Weight FLOAT
-        IsMagic BOOLEAN
+        gcloud alpha bigquery datasets create test_dataset
 
+* Load sample data into google cloud storage (for import tests):
+
+        gsutil cp tests/resources/data.csv gs://$TEST_BUCKET_NAME/data.csv
+
+* Load the sample data into a table named `test_table` (for export and streaming tests):
+
+        gcloud alpha bigquery import \
+          gs://$TEST_BUCKET_NAME/data.csv \
+          test_dataset/test_table \
+          --schema-file tests/resources/schema.json
 
 ### Test environments
 

appengine/bigquery/main_test.py

@@ -28,6 +28,7 @@ class TestAuthSample(tests.AppEngineTestbedCase):
     def setUp(self):
         super(TestAuthSample, self).setUp()
         self.app = webtest.TestApp(main.app)
+        main.PROJECTID = self.project_id
 
     def test_anonymous_get(self):
         response = self.app.get('/')
@@ -55,12 +56,7 @@ def test_oauthed_get(self, *args):
             {'status': '200'})
 
         with mock.patch.object(main.decorator, 'http', return_value=mock_http):
-            original_projectid = main.PROJECTID
-            try:
-                main.PROJECTID = self.constants['projectId']
-                response = self.app.get('/')
-            finally:
-                main.PROJECTID = original_projectid
+            response = self.app.get('/')
 
         # Should make the api call
         self.assertEqual(response.status_int, 200)

bigquery/samples/async_query_test.py

@@ -21,13 +21,17 @@
 class TestAsyncQuery(tests.CloudBaseTest):
 
     def test_async_query(self):
+        query = (
+            'SELECT corpus FROM publicdata:samples.shakespeare '
+            'GROUP BY corpus;')
+
         with tests.capture_stdout() as stdout:
             main(
-                self.constants['projectId'],
-                self.constants['query'],
-                False,
-                5,
-                5)
+                project_id=self.project_id,
+                query_string=query,
+                batch=False,
+                num_retries=5,
+                interval=1)
 
         value = stdout.getvalue().strip().split('\n').pop()
 

bigquery/samples/export_data_to_cloud_storage_test.py

@@ -19,33 +19,41 @@
 
 
 class TestExportTableToGCS(CloudBaseTest):
+    dataset_id = 'test_dataset'
+    table_id = 'test_table'
 
     def test_export_table_csv(self):
+        cloud_storage_output_uri = \
+            'gs://{}/output.csv'.format(self.bucket_name)
         main(
-            self.constants['cloudStorageOutputURI'],
-            self.constants['projectId'],
-            self.constants['datasetId'],
-            self.constants['newTableId'],
-            5,
-            1,
+            cloud_storage_output_uri,
+            self.project_id,
+            self.dataset_id,
+            self.table_id,
+            num_retries=5,
+            interval=1,
             export_format="CSV")
 
     def test_export_table_json(self):
+        cloud_storage_output_uri = \
+            'gs://{}/output.json'.format(self.bucket_name)
         main(
-            self.constants['cloudStorageOutputURI'],
-            self.constants['projectId'],
-            self.constants['datasetId'],
-            self.constants['newTableId'],
-            5,
-            1,
+            cloud_storage_output_uri,
+            self.project_id,
+            self.dataset_id,
+            self.table_id,
+            num_retries=5,
+            interval=1,
             export_format="NEWLINE_DELIMITED_JSON")
 
     def test_export_table_avro(self):
+        cloud_storage_output_uri = \
+            'gs://{}/output.avro'.format(self.bucket_name)
         main(
-            self.constants['cloudStorageOutputURI'],
-            self.constants['projectId'],
-            self.constants['datasetId'],
-            self.constants['newTableId'],
-            5,
-            1,
+            cloud_storage_output_uri,
+            self.project_id,
+            self.dataset_id,
+            self.table_id,
+            num_retries=5,
+            interval=1,
             export_format="AVRO")

bigquery/samples/getting_started_test.py

@@ -21,7 +21,7 @@
 class TestGettingStarted(tests.CloudBaseTest):
     def test_main(self):
         with tests.capture_stdout() as mock_stdout:
-            main(self.constants['projectId'])
+            main(self.project_id)
 
         stdout = mock_stdout.getvalue()
         self.assertRegexpMatches(stdout, re.compile(

bigquery/samples/list_datasets_projects_test.py

@@ -22,7 +22,7 @@ class TestListDatasetsProjects(tests.CloudBaseTest):
 
     def test_main(self):
         with tests.capture_stdout() as mock_stdout:
-            main(self.constants['projectId'])
+            main(self.project_id)
 
         stdout = mock_stdout.getvalue()
 

bigquery/samples/load_data_from_csv_test.py

@@ -20,12 +20,18 @@
 
 
 class TestLoadDataFromCSV(CloudBaseTest):
+    dataset_id = 'test_dataset'
+    table_id = 'test_import_table'
+
     def test_load_table(self):
+        cloud_storage_input_uri = 'gs://{}/data.csv'.format(self.bucket_name)
+        schema_file = os.path.join(self.resource_path, 'schema.json')
+
         main(
-            self.constants['projectId'],
-            self.constants['datasetId'],
-            self.constants['newTableId'],
-            os.path.join(self.resource_path, 'schema.json'),
-            self.constants['cloudStorageInputURI'],
-            1,
-            5)
+            self.project_id,
+            self.dataset_id,
+            self.table_id,
+            schema_file=schema_file,
+            data_path=cloud_storage_input_uri,
+            poll_interval=1,
+            num_retries=5)

bigquery/samples/streaming_test.py

@@ -21,6 +21,8 @@
 
 
 class TestStreaming(CloudBaseTest):
+    dataset_id = 'test_dataset'
+    table_id = 'test_table'
 
     def test_stream_row_to_bigquery(self):
         with open(
@@ -33,10 +35,10 @@ def test_stream_row_to_bigquery(self):
 
         with capture_stdout() as stdout:
             streaming.main(
-                self.constants['projectId'],
-                self.constants['datasetId'],
-                self.constants['newTableId'],
-                5)
+                self.project_id,
+                self.dataset_id,
+                self.table_id,
+                num_retries=5)
 
         results = stdout.getvalue().split('\n')
         self.assertIsNotNone(json.loads(results[0]))

bigquery/samples/sync_query_test.py

@@ -21,12 +21,16 @@
 class TestSyncQuery(CloudBaseTest):
 
     def test_sync_query(self):
+        query = (
+            'SELECT corpus FROM publicdata:samples.shakespeare '
+            'GROUP BY corpus;')
+
         with capture_stdout() as stdout:
             main(
-                self.constants['projectId'],
-                self.constants['query'],
-                30,
-                5)
+                project_id=self.project_id,
+                query=query,
+                timeout=30,
+                num_retries=5)
 
         result = stdout.getvalue().split('\n')[0]
         self.assertIsNotNone(json.loads(result))

blog/introduction_to_data_models_in_cloud_datastore/blog_test.py

@@ -20,4 +20,4 @@ class BlogTestCase(CloudBaseTest):
     """Simple test case that ensures the blog code doesn't throw any errors."""
 
     def test_main(self):
-        main(self.constants['projectId'])
+        main(self.project_id)