From a61a82efbee2d1429d60dcc35a2df79574977e55 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Wed, 3 Apr 2024 19:42:29 -0700 Subject: [PATCH 01/11] Add ZSTD to compression types --- google/cloud/bigquery/enums.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index d75037ad1..9fbf61f4e 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -39,6 +39,9 @@ class Compression(object): SNAPPY = "SNAPPY" """Specifies SNAPPY format.""" + ZSTD = "ZSTD" + """Specifies ZSTD format.""" + NONE = "NONE" """Specifies no compression.""" From df23c7dab9464cd681b0298fd181d688568580ee Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 8 Apr 2024 14:27:47 +0000 Subject: [PATCH 02/11] feat: adds tests re Compression types --- google/cloud/bigquery/enums.py | 2 +- tests/unit/test_enums.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_enums.py diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 9fbf61f4e..8a5ae0851 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -22,7 +22,7 @@ class AutoRowIDs(enum.Enum): GENERATE_UUID = enum.auto() -class Compression(object): +class Compression(enum.Enum): """The compression type to use for exported files. The default value is :attr:`NONE`. diff --git a/tests/unit/test_enums.py b/tests/unit/test_enums.py new file mode 100644 index 000000000..e58e86251 --- /dev/null +++ b/tests/unit/test_enums.py @@ -0,0 +1,8 @@ +def test_compression_enums(): + from google.cloud.bigquery.enums import Compression + + expected_comps_sorted = ["DEFLATE", "GZIP", "NONE", "SNAPPY", "ZSTD"] + + result_comps_sorted = sorted(comp.value for comp in Compression) + + assert result_comps_sorted == expected_comps_sorted From b26a974acea38842b97d28b81a4cb40e68cdf646 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 8 Apr 2024 15:17:40 +0000 Subject: [PATCH 03/11] revise datatype from Enum to object --- google/cloud/bigquery/enums.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 8a5ae0851..9fbf61f4e 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -22,7 +22,7 @@ class AutoRowIDs(enum.Enum): GENERATE_UUID = enum.auto() -class Compression(enum.Enum): +class Compression(object): """The compression type to use for exported files. The default value is :attr:`NONE`. From b801aafc327c74b13caf039cbf24000d72f88f98 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Mon, 8 Apr 2024 16:13:34 +0000 Subject: [PATCH 04/11] adds license text and docstring --- tests/unit/test_enums.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/unit/test_enums.py b/tests/unit/test_enums.py index e58e86251..ddf040cea 100644 --- a/tests/unit/test_enums.py +++ b/tests/unit/test_enums.py @@ -1,4 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + def test_compression_enums(): + """Ensures that the compression enum has all the allowed types of + compression algorithms present. + """ + from google.cloud.bigquery.enums import Compression expected_comps_sorted = ["DEFLATE", "GZIP", "NONE", "SNAPPY", "ZSTD"] From f6da457a1a9f82dbac5e3387066758d1a6049e32 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 10 Apr 2024 15:16:17 +0000 Subject: [PATCH 05/11] change object back to enum datatype --- google/cloud/bigquery/enums.py | 2 +- tests/unit/job/test_extract.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 9fbf61f4e..8a5ae0851 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -22,7 +22,7 @@ class AutoRowIDs(enum.Enum): GENERATE_UUID = enum.auto() -class Compression(object): +class Compression(enum.Enum): """The compression type to use for exported files. The default value is :attr:`NONE`. diff --git a/tests/unit/job/test_extract.py b/tests/unit/job/test_extract.py index 76ee72f28..86ab12b69 100644 --- a/tests/unit/job/test_extract.py +++ b/tests/unit/job/test_extract.py @@ -50,7 +50,7 @@ def test_to_api_repr(self): resource, { "extract": { - "compression": "SNAPPY", + "compression": job.Compression.SNAPPY, "destinationFormat": "AVRO", "fieldDelimiter": "ignored for avro", "printHeader": False, From b57dd6678b6482bb4a5c1ac5a0a5eea105a591e0 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 10 Apr 2024 15:49:17 +0000 Subject: [PATCH 06/11] updates compression object comparison --- tests/unit/job/test_extract.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/job/test_extract.py b/tests/unit/job/test_extract.py index 86ab12b69..28049b134 100644 --- a/tests/unit/job/test_extract.py +++ b/tests/unit/job/test_extract.py @@ -39,7 +39,7 @@ def test_to_api_repr(self): from google.cloud.bigquery import job config = self._make_one() - config.compression = job.Compression.SNAPPY + config.compression = job.Compression.SNAPPY.value config.destination_format = job.DestinationFormat.AVRO config.field_delimiter = "ignored for avro" config.print_header = False @@ -50,7 +50,7 @@ def test_to_api_repr(self): resource, { "extract": { - "compression": job.Compression.SNAPPY, + "compression": "SNAPPY", "destinationFormat": "AVRO", "fieldDelimiter": "ignored for avro", "printHeader": False, From 867e5bc89d24daf48f016100f7467611477d95c4 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 10 Apr 2024 16:08:22 +0000 Subject: [PATCH 07/11] updates Compression class --- google/cloud/bigquery/enums.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 8a5ae0851..1abe28381 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -22,7 +22,7 @@ class AutoRowIDs(enum.Enum): GENERATE_UUID = enum.auto() -class Compression(enum.Enum): +class Compression(str, enum.Enum): """The compression type to use for exported files. The default value is :attr:`NONE`. From 3b0f4b0d1dde2b10f089cf172a36f30d73e58913 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Apr 2024 17:22:05 +0000 Subject: [PATCH 08/11] jsonify and sort the input and output for testing --- tests/unit/job/test_extract.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/unit/job/test_extract.py b/tests/unit/job/test_extract.py index 28049b134..38b87d801 100644 --- a/tests/unit/job/test_extract.py +++ b/tests/unit/job/test_extract.py @@ -37,6 +37,7 @@ def test_ctor_w_properties(self): def test_to_api_repr(self): from google.cloud.bigquery import job + import json config = self._make_one() config.compression = job.Compression.SNAPPY.value @@ -45,9 +46,8 @@ def test_to_api_repr(self): config.print_header = False config._properties["extract"]["someNewField"] = "some-value" config.use_avro_logical_types = True - resource = config.to_api_repr() - self.assertEqual( - resource, + resource = json.dumps(config.to_api_repr(), sort_keys=True) + expected = json.dumps( { "extract": { "compression": "SNAPPY", @@ -58,6 +58,12 @@ def test_to_api_repr(self): "useAvroLogicalTypes": True, } }, + sort_keys=True, + ) + + self.assertEqual( + resource, + expected, ) def test_from_api_repr(self): From 594a2a9d47de819cc6743cfa6817abf144f843a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Thu, 11 Apr 2024 13:07:41 -0500 Subject: [PATCH 09/11] Update tests/unit/job/test_extract.py --- tests/unit/job/test_extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/job/test_extract.py b/tests/unit/job/test_extract.py index 38b87d801..f13233181 100644 --- a/tests/unit/job/test_extract.py +++ b/tests/unit/job/test_extract.py @@ -40,7 +40,7 @@ def test_to_api_repr(self): import json config = self._make_one() - config.compression = job.Compression.SNAPPY.value + config.compression = job.Compression.SNAPPY config.destination_format = job.DestinationFormat.AVRO config.field_delimiter = "ignored for avro" config.print_header = False From e685c2db0ffce158890a825b13b81861d0802def Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Apr 2024 19:05:53 +0000 Subject: [PATCH 10/11] moved json import statement --- tests/unit/job/test_extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/job/test_extract.py b/tests/unit/job/test_extract.py index f13233181..ee0d67d68 100644 --- a/tests/unit/job/test_extract.py +++ b/tests/unit/job/test_extract.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json from unittest import mock from ..helpers import make_connection @@ -37,7 +38,6 @@ def test_ctor_w_properties(self): def test_to_api_repr(self): from google.cloud.bigquery import job - import json config = self._make_one() config.compression = job.Compression.SNAPPY From 37e7f538529d8da25b731ae4222dc649ce53596e Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 11 Apr 2024 19:07:35 +0000 Subject: [PATCH 11/11] removed enums test and file --- tests/unit/test_enums.py | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 tests/unit/test_enums.py diff --git a/tests/unit/test_enums.py b/tests/unit/test_enums.py deleted file mode 100644 index ddf040cea..000000000 --- a/tests/unit/test_enums.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def test_compression_enums(): - """Ensures that the compression enum has all the allowed types of - compression algorithms present. - """ - - from google.cloud.bigquery.enums import Compression - - expected_comps_sorted = ["DEFLATE", "GZIP", "NONE", "SNAPPY", "ZSTD"] - - result_comps_sorted = sorted(comp.value for comp in Compression) - - assert result_comps_sorted == expected_comps_sorted