feat: add support for NUMERIC type (#86)

* feat: add support for NUMERIC type

* add tests

* fix test name

* remove unused import

* add NUMERIC to param_types

* add system tests

* test: update tests to work for emulator

* style: fix lint

Co-authored-by: larkee <[email protected]>

Author: larkee

google/cloud/spanner_v1/_helpers.py

@@ -15,6 +15,7 @@
 """Helper functions for Cloud Spanner."""
 
 import datetime
+import decimal
 import math
 
 import six
@@ -127,6 +128,8 @@ def _make_value_pb(value):
         return Value(string_value=value)
     if isinstance(value, ListValue):
         return Value(list_value=value)
+    if isinstance(value, decimal.Decimal):
+        return Value(string_value=str(value))
     raise ValueError("Unknown type: %s" % (value,))
 
 
@@ -201,6 +204,8 @@ def _parse_value_pb(value_pb, field_type):
             _parse_value_pb(item_pb, field_type.struct_type.fields[i].type)
             for (i, item_pb) in enumerate(value_pb.list_value.values)
         ]
+    elif field_type.code == type_pb2.NUMERIC:
+        result = decimal.Decimal(value_pb.string_value)
     else:
         raise ValueError("Unknown type: %s" % (field_type,))
     return result

google/cloud/spanner_v1/param_types.py

@@ -25,6 +25,7 @@
 FLOAT64 = type_pb2.Type(code=type_pb2.FLOAT64)
 DATE = type_pb2.Type(code=type_pb2.DATE)
 TIMESTAMP = type_pb2.Type(code=type_pb2.TIMESTAMP)
+NUMERIC = type_pb2.Type(code=type_pb2.NUMERIC)
 
 
 def Array(element_type):  # pylint: disable=invalid-name

tests/_fixtures.py

@@ -16,6 +16,58 @@
 
 
 DDL = """\
+CREATE TABLE contacts (
+    contact_id INT64,
+    first_name STRING(1024),
+    last_name STRING(1024),
+    email STRING(1024) )
+    PRIMARY KEY (contact_id);
+CREATE TABLE contact_phones (
+    contact_id INT64,
+    phone_type STRING(1024),
+    phone_number STRING(1024) )
+    PRIMARY KEY (contact_id, phone_type),
+    INTERLEAVE IN PARENT contacts ON DELETE CASCADE;
+CREATE TABLE all_types (
+    pkey INT64 NOT NULL,
+    int_value INT64,
+    int_array ARRAY<INT64>,
+    bool_value BOOL,
+    bool_array ARRAY<BOOL>,
+    bytes_value BYTES(16),
+    bytes_array ARRAY<BYTES(16)>,
+    date_value DATE,
+    date_array ARRAY<DATE>,
+    float_value FLOAT64,
+    float_array ARRAY<FLOAT64>,
+    string_value STRING(16),
+    string_array ARRAY<STRING(16)>,
+    timestamp_value TIMESTAMP,
+    timestamp_array ARRAY<TIMESTAMP>,
+    numeric_value NUMERIC,
+    numeric_array ARRAY<NUMERIC>)
+    PRIMARY KEY (pkey);
+CREATE TABLE counters (
+    name STRING(1024),
+    value INT64 )
+    PRIMARY KEY (name);
+CREATE TABLE string_plus_array_of_string (
+    id INT64,
+    name STRING(16),
+    tags ARRAY<STRING(16)> )
+    PRIMARY KEY (id);
+CREATE INDEX name ON contacts(first_name, last_name);
+CREATE TABLE users_history (
+     id INT64 NOT NULL,
+     commit_ts TIMESTAMP NOT NULL OPTIONS
+        (allow_commit_timestamp=true),
+     name STRING(MAX) NOT NULL,
+     email STRING(MAX),
+     deleted BOOL NOT NULL )
+     PRIMARY KEY(id, commit_ts DESC);
+"""
+
+EMULATOR_DDL = """\
 CREATE TABLE contacts (
     contact_id INT64,
     first_name STRING(1024),
@@ -66,3 +118,6 @@
 """
 
 DDL_STATEMENTS = [stmt.strip() for stmt in DDL.split(";") if stmt.strip()]
+EMULATOR_DDL_STATEMENTS = [
+    stmt.strip() for stmt in EMULATOR_DDL.split(";") if stmt.strip()
+]

tests/system/test_system.py

@@ -14,6 +14,7 @@
 
 import collections
 import datetime
+import decimal
 import math
 import operator
 import os
@@ -38,6 +39,7 @@
 from google.cloud.spanner_v1.proto.type_pb2 import INT64
 from google.cloud.spanner_v1.proto.type_pb2 import STRING
 from google.cloud.spanner_v1.proto.type_pb2 import TIMESTAMP
+from google.cloud.spanner_v1.proto.type_pb2 import NUMERIC
 from google.cloud.spanner_v1.proto.type_pb2 import Type
 
 from google.cloud._helpers import UTC
@@ -52,11 +54,13 @@
 from test_utils.retry import RetryResult
 from test_utils.system import unique_resource_id
 from tests._fixtures import DDL_STATEMENTS
+from tests._fixtures import EMULATOR_DDL_STATEMENTS
 from tests._helpers import OpenTelemetryBase, HAS_OPENTELEMETRY_INSTALLED
 
 
 CREATE_INSTANCE = os.getenv("GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE") is not None
 USE_EMULATOR = os.getenv("SPANNER_EMULATOR_HOST") is not None
+SKIP_BACKUP_TESTS = os.getenv("SKIP_BACKUP_TESTS") is not None
 
 if CREATE_INSTANCE:
     INSTANCE_ID = "google-cloud" + unique_resource_id("-")
@@ -92,7 +96,8 @@ class Config(object):
 
 
 def _has_all_ddl(database):
-    return len(database.ddl_statements) == len(DDL_STATEMENTS)
+    ddl_statements = EMULATOR_DDL_STATEMENTS if USE_EMULATOR else DDL_STATEMENTS
+    return len(database.ddl_statements) == len(ddl_statements)
 
 
 def _list_instances():
@@ -284,8 +289,9 @@ class TestDatabaseAPI(unittest.TestCase, _TestData):
     @classmethod
     def setUpClass(cls):
         pool = BurstyPool(labels={"testcase": "database_api"})
+        ddl_statements = EMULATOR_DDL_STATEMENTS if USE_EMULATOR else DDL_STATEMENTS
         cls._db = Config.INSTANCE.database(
-            cls.DATABASE_NAME, ddl_statements=DDL_STATEMENTS, pool=pool
+            cls.DATABASE_NAME, ddl_statements=ddl_statements, pool=pool
         )
         operation = cls._db.create()
         operation.result(30)  # raises on failure / timeout.
@@ -359,12 +365,13 @@ def test_update_database_ddl_with_operation_id(self):
         temp_db = Config.INSTANCE.database(temp_db_id, pool=pool)
         create_op = temp_db.create()
         self.to_delete.append(temp_db)
+        ddl_statements = EMULATOR_DDL_STATEMENTS if USE_EMULATOR else DDL_STATEMENTS
 
         # We want to make sure the operation completes.
         create_op.result(240)  # raises on failure / timeout.
         # random but shortish always start with letter
         operation_id = "a" + str(uuid.uuid4())[:8]
-        operation = temp_db.update_ddl(DDL_STATEMENTS, operation_id=operation_id)
+        operation = temp_db.update_ddl(ddl_statements, operation_id=operation_id)
 
         self.assertEqual(operation_id, operation.operation.name.split("/")[-1])
 
@@ -373,7 +380,7 @@ def test_update_database_ddl_with_operation_id(self):
 
         temp_db.reload()
 
-        self.assertEqual(len(temp_db.ddl_statements), len(DDL_STATEMENTS))
+        self.assertEqual(len(temp_db.ddl_statements), len(ddl_statements))
 
     def test_db_batch_insert_then_db_snapshot_read(self):
         retry = RetryInstanceState(_has_all_ddl)
@@ -447,15 +454,17 @@ def _unit_of_work(transaction, name):
 
 
 @unittest.skipIf(USE_EMULATOR, "Skipping backup tests")
+@unittest.skipIf(SKIP_BACKUP_TESTS, "Skipping backup tests")
 class TestBackupAPI(unittest.TestCase, _TestData):
     DATABASE_NAME = "test_database" + unique_resource_id("_")
     DATABASE_NAME_2 = "test_database2" + unique_resource_id("_")
 
     @classmethod
     def setUpClass(cls):
         pool = BurstyPool(labels={"testcase": "database_api"})
+        ddl_statements = EMULATOR_DDL_STATEMENTS if USE_EMULATOR else DDL_STATEMENTS
         db1 = Config.INSTANCE.database(
-            cls.DATABASE_NAME, ddl_statements=DDL_STATEMENTS, pool=pool
+            cls.DATABASE_NAME, ddl_statements=ddl_statements, pool=pool
         )
         db2 = Config.INSTANCE.database(cls.DATABASE_NAME_2, pool=pool)
         cls._db = db1
@@ -736,6 +745,8 @@ def test_list_backups(self):
 (OTHER_NAN,) = struct.unpack("<d", b"\x01\x00\x01\x00\x00\x00\xf8\xff")
 BYTES_1 = b"Ymlu"
 BYTES_2 = b"Ym9vdHM="
+NUMERIC_1 = decimal.Decimal("0.123456789")
+NUMERIC_2 = decimal.Decimal("1234567890")
 ALL_TYPES_TABLE = "all_types"
 ALL_TYPES_COLUMNS = (
     "pkey",
@@ -753,9 +764,18 @@ def test_list_backups(self):
     "string_array",
     "timestamp_value",
     "timestamp_array",
+    "numeric_value",
+    "numeric_array",
 )
+EMULATOR_ALL_TYPES_COLUMNS = ALL_TYPES_COLUMNS[:-2]
 AllTypesRowData = collections.namedtuple("AllTypesRowData", ALL_TYPES_COLUMNS)
 AllTypesRowData.__new__.__defaults__ = tuple([None for colum in ALL_TYPES_COLUMNS])
+EmulatorAllTypesRowData = collections.namedtuple(
+    "EmulatorAllTypesRowData", EMULATOR_ALL_TYPES_COLUMNS
+)
+EmulatorAllTypesRowData.__new__.__defaults__ = tuple(
+    [None for colum in EMULATOR_ALL_TYPES_COLUMNS]
+)
 
 ALL_TYPES_ROWDATA = (
     # all nulls
@@ -769,6 +789,7 @@ def test_list_backups(self):
     AllTypesRowData(pkey=106, string_value=u"VALUE"),
     AllTypesRowData(pkey=107, timestamp_value=SOME_TIME),
     AllTypesRowData(pkey=108, timestamp_value=NANO_TIME),
+    AllTypesRowData(pkey=109, numeric_value=NUMERIC_1),
     # empty array values
     AllTypesRowData(pkey=201, int_array=[]),
     AllTypesRowData(pkey=202, bool_array=[]),
@@ -777,6 +798,7 @@ def test_list_backups(self):
     AllTypesRowData(pkey=205, float_array=[]),
     AllTypesRowData(pkey=206, string_array=[]),
     AllTypesRowData(pkey=207, timestamp_array=[]),
+    AllTypesRowData(pkey=208, numeric_array=[]),
     # non-empty array values, including nulls
     AllTypesRowData(pkey=301, int_array=[123, 456, None]),
     AllTypesRowData(pkey=302, bool_array=[True, False, None]),
@@ -785,6 +807,36 @@ def test_list_backups(self):
     AllTypesRowData(pkey=305, float_array=[3.1415926, 2.71828, None]),
     AllTypesRowData(pkey=306, string_array=[u"One", u"Two", None]),
     AllTypesRowData(pkey=307, timestamp_array=[SOME_TIME, NANO_TIME, None]),
+    AllTypesRowData(pkey=308, numeric_array=[NUMERIC_1, NUMERIC_2, None]),
+)
+EMULATOR_ALL_TYPES_ROWDATA = (
+    # all nulls
+    EmulatorAllTypesRowData(pkey=0),
+    # Non-null values
+    EmulatorAllTypesRowData(pkey=101, int_value=123),
+    EmulatorAllTypesRowData(pkey=102, bool_value=False),
+    EmulatorAllTypesRowData(pkey=103, bytes_value=BYTES_1),
+    EmulatorAllTypesRowData(pkey=104, date_value=SOME_DATE),
+    EmulatorAllTypesRowData(pkey=105, float_value=1.4142136),
+    EmulatorAllTypesRowData(pkey=106, string_value=u"VALUE"),
+    EmulatorAllTypesRowData(pkey=107, timestamp_value=SOME_TIME),
+    EmulatorAllTypesRowData(pkey=108, timestamp_value=NANO_TIME),
+    # empty array values
+    EmulatorAllTypesRowData(pkey=201, int_array=[]),
+    EmulatorAllTypesRowData(pkey=202, bool_array=[]),
+    EmulatorAllTypesRowData(pkey=203, bytes_array=[]),
+    EmulatorAllTypesRowData(pkey=204, date_array=[]),
+    EmulatorAllTypesRowData(pkey=205, float_array=[]),
+    EmulatorAllTypesRowData(pkey=206, string_array=[]),
+    EmulatorAllTypesRowData(pkey=207, timestamp_array=[]),
+    # non-empty array values, including nulls
+    EmulatorAllTypesRowData(pkey=301, int_array=[123, 456, None]),
+    EmulatorAllTypesRowData(pkey=302, bool_array=[True, False, None]),
+    EmulatorAllTypesRowData(pkey=303, bytes_array=[BYTES_1, BYTES_2, None]),
+    EmulatorAllTypesRowData(pkey=304, date_array=[SOME_DATE, None]),
+    EmulatorAllTypesRowData(pkey=305, float_array=[3.1415926, 2.71828, None]),
+    EmulatorAllTypesRowData(pkey=306, string_array=[u"One", u"Two", None]),
+    EmulatorAllTypesRowData(pkey=307, timestamp_array=[SOME_TIME, NANO_TIME, None]),
 )
 
 
@@ -794,8 +846,9 @@ class TestSessionAPI(OpenTelemetryBase, _TestData):
     @classmethod
     def setUpClass(cls):
         pool = BurstyPool(labels={"testcase": "session_api"})
+        ddl_statements = EMULATOR_DDL_STATEMENTS if USE_EMULATOR else DDL_STATEMENTS
         cls._db = Config.INSTANCE.database(
-            cls.DATABASE_NAME, ddl_statements=DDL_STATEMENTS, pool=pool
+            cls.DATABASE_NAME, ddl_statements=ddl_statements, pool=pool
         )
         operation = cls._db.create()
         operation.result(30)  # raises on failure / timeout.
@@ -899,13 +952,19 @@ def test_batch_insert_then_read_all_datatypes(self):
         retry = RetryInstanceState(_has_all_ddl)
         retry(self._db.reload)()
 
+        if USE_EMULATOR:
+            all_types_columns = EMULATOR_ALL_TYPES_COLUMNS
+            all_types_rowdata = EMULATOR_ALL_TYPES_ROWDATA
+        else:
+            all_types_columns = ALL_TYPES_COLUMNS
+            all_types_rowdata = ALL_TYPES_ROWDATA
         with self._db.batch() as batch:
             batch.delete(ALL_TYPES_TABLE, self.ALL)
-            batch.insert(ALL_TYPES_TABLE, ALL_TYPES_COLUMNS, ALL_TYPES_ROWDATA)
+            batch.insert(ALL_TYPES_TABLE, all_types_columns, all_types_rowdata)
 
         with self._db.snapshot(read_timestamp=batch.committed) as snapshot:
-            rows = list(snapshot.read(ALL_TYPES_TABLE, ALL_TYPES_COLUMNS, self.ALL))
-        self._check_rows_data(rows, expected=ALL_TYPES_ROWDATA)
+            rows = list(snapshot.read(ALL_TYPES_TABLE, all_types_columns, self.ALL))
+        self._check_rows_data(rows, expected=all_types_rowdata)
 
     def test_batch_insert_or_update_then_query(self):
         retry = RetryInstanceState(_has_all_ddl)
@@ -1704,9 +1763,10 @@ def test_read_w_index(self):
         MY_COLUMNS = self.COLUMNS[0], self.COLUMNS[2]
         EXTRA_DDL = ["CREATE INDEX contacts_by_last_name ON contacts(last_name)"]
         pool = BurstyPool(labels={"testcase": "read_w_index"})
+        ddl_statements = EMULATOR_DDL_STATEMENTS if USE_EMULATOR else DDL_STATEMENTS
         temp_db = Config.INSTANCE.database(
             "test_read" + unique_resource_id("_"),
-            ddl_statements=DDL_STATEMENTS + EXTRA_DDL,
+            ddl_statements=ddl_statements + EXTRA_DDL,
             pool=pool,
         )
         operation = temp_db.create()
@@ -2282,6 +2342,10 @@ def test_execute_sql_w_date_bindings(self):
         dates = [SOME_DATE, SOME_DATE + datetime.timedelta(days=1)]
         self._bind_test_helper(DATE, SOME_DATE, dates)
 
+    @unittest.skipIf(USE_EMULATOR, "Skipping NUMERIC")
+    def test_execute_sql_w_numeric_bindings(self):
+        self._bind_test_helper(NUMERIC, NUMERIC_1, [NUMERIC_1, NUMERIC_2])
+
     def test_execute_sql_w_query_param_struct(self):
         NAME = "Phred"
         COUNT = 123

tests/unit/test__helpers.py

@@ -208,6 +208,15 @@ def test_w_datetime(self):
         self.assertIsInstance(value_pb, Value)
         self.assertEqual(value_pb.string_value, datetime_helpers.to_rfc3339(now))
 
+    def test_w_numeric(self):
+        import decimal
+        from google.protobuf.struct_pb2 import Value
+
+        value = decimal.Decimal("9999999999999999999999999999.999999999")
+        value_pb = self._callFUT(value)
+        self.assertIsInstance(value_pb, Value)
+        self.assertEqual(value_pb.string_value, str(value))
+
     def test_w_unknown_type(self):
         with self.assertRaises(ValueError):
             self._callFUT(object())
@@ -431,6 +440,17 @@ def test_w_struct(self):
 
         self.assertEqual(self._callFUT(value_pb, field_type), VALUES)
 
+    def test_w_numeric(self):
+        import decimal
+        from google.protobuf.struct_pb2 import Value
+        from google.cloud.spanner_v1.proto.type_pb2 import Type, NUMERIC
+
+        VALUE = decimal.Decimal("99999999999999999999999999999.999999999")
+        field_type = Type(code=NUMERIC)
+        value_pb = Value(string_value=str(VALUE))
+
+        self.assertEqual(self._callFUT(value_pb, field_type), VALUE)
+
     def test_w_unknown_type(self):
         from google.protobuf.struct_pb2 import Value
         from google.cloud.spanner_v1.proto.type_pb2 import Type