ENH: bool sparse now supports logical op

Author: sinhrks <[email protected]>

Closes #14000 from sinhrks/sparse_bool and squashes the following commits:

6db3096 [sinhrks] ENH: bool sparse now supports logical op

Author: sinhrks

doc/source/whatsnew/v0.19.0.txt

@@ -762,6 +762,7 @@ Note that the limitation is applied to ``fill_value`` which default is ``np.nan`
    ValueError: unable to coerce current fill_value nan to int64 dtype
 
 - Subclassed ``SparseDataFrame`` and ``SparseSeries`` now preserve class types when slicing or transposing. (:issue:`13787`)
+- ``SparseArray`` with ``bool`` dtype now supports logical (bool) operators (:issue:`14000`)
 - Bug in ``SparseSeries`` with ``MultiIndex`` ``[]`` indexing may raise ``IndexError`` (:issue:`13144`)
 - Bug in ``SparseSeries`` with ``MultiIndex`` ``[]`` indexing result may have normal ``Index`` (:issue:`13144`)
 - Bug in ``SparseDataFrame`` in which ``axis=None`` did not default to ``axis=0`` (:issue:`13048`)

pandas/sparse/array.py

@@ -98,6 +98,7 @@ def _sparse_array_op(left, right, op, name, series=False):
             right = right.astype(np.float64)
 
     dtype = _maybe_match_dtype(left, right)
+    result_dtype = None
 
     if left.sp_index.ngaps == 0 or right.sp_index.ngaps == 0:
         result = op(left.get_values(), right.get_values())
@@ -116,13 +117,26 @@ def _sparse_array_op(left, right, op, name, series=False):
             left, right = right, left
             name = name[1:]
 
-        opname = 'sparse_{name}_{dtype}'.format(name=name, dtype=dtype)
-        sparse_op = getattr(splib, opname)
+        if name in ('and', 'or') and dtype == 'bool':
+            opname = 'sparse_{name}_uint8'.format(name=name, dtype=dtype)
+            # to make template simple, cast here
+            left_sp_values = left.sp_values.view(np.uint8)
+            right_sp_values = right.sp_values.view(np.uint8)
+            result_dtype = np.bool
+        else:
+            opname = 'sparse_{name}_{dtype}'.format(name=name, dtype=dtype)
+            left_sp_values = left.sp_values
+            right_sp_values = right.sp_values
 
-        result, index, fill = sparse_op(left.sp_values, left.sp_index,
-                                        left.fill_value, right.sp_values,
+        sparse_op = getattr(splib, opname)
+        result, index, fill = sparse_op(left_sp_values, left.sp_index,
+                                        left.fill_value, right_sp_values,
                                         right.sp_index, right.fill_value)
-    return _wrap_result(name, result, index, fill, dtype=result.dtype)
+
+    if result_dtype is None:
+        result_dtype = result.dtype
+
+    return _wrap_result(name, result, index, fill, dtype=result_dtype)
 
 
 def _wrap_result(name, data, sparse_index, fill_value, dtype=None):
@@ -750,4 +764,5 @@ def _make_index(length, indices, kind):
 
 ops.add_special_arithmetic_methods(SparseArray, arith_method=_arith_method,
                                    comp_method=_arith_method,
+                                   bool_method=_arith_method,
                                    use_numexpr=False)

pandas/sparse/tests/test_arithmetics.py

@@ -108,6 +108,20 @@ def _check_comparison_ops(self, a, b, a_dense, b_dense):
         self._check_bool_result(a < b_dense)
         self._assert((a < b_dense).to_dense(), a_dense < b_dense)
 
+    def _check_logical_ops(self, a, b, a_dense, b_dense):
+        # sparse & sparse
+        self._check_bool_result(a & b)
+        self._assert((a & b).to_dense(), a_dense & b_dense)
+
+        self._check_bool_result(a | b)
+        self._assert((a | b).to_dense(), a_dense | b_dense)
+        # sparse & dense
+        self._check_bool_result(a & b_dense)
+        self._assert((a & b_dense).to_dense(), a_dense & b_dense)
+
+        self._check_bool_result(a | b_dense)
+        self._assert((a | b_dense).to_dense(), a_dense | b_dense)
+
     def test_float_scalar(self):
         values = self._base([np.nan, 1, 2, 0, np.nan, 0, 1, 2, 1, np.nan])
 
@@ -305,6 +319,36 @@ def test_int_array_comparison(self):
                 b = self._klass(rvalues, dtype=dtype, kind=kind, fill_value=2)
                 self._check_comparison_ops(a, b, values, rvalues)
 
+    def test_bool_same_index(self):
+        # GH 14000
+        # when sp_index are the same
+        for kind in ['integer', 'block']:
+            values = self._base([True, False, True, True], dtype=np.bool)
+            rvalues = self._base([True, False, True, True], dtype=np.bool)
+
+            for fill_value in [True, False, np.nan]:
+                a = self._klass(values, kind=kind, dtype=np.bool,
+                                fill_value=fill_value)
+                b = self._klass(rvalues, kind=kind, dtype=np.bool,
+                                fill_value=fill_value)
+                self._check_logical_ops(a, b, values, rvalues)
+
+    def test_bool_array_logical(self):
+        # GH 14000
+        # when sp_index are the same
+        for kind in ['integer', 'block']:
+            values = self._base([True, False, True, False, True, True],
+                                dtype=np.bool)
+            rvalues = self._base([True, False, False, True, False, True],
+                                 dtype=np.bool)
+
+            for fill_value in [True, False, np.nan]:
+                a = self._klass(values, kind=kind, dtype=np.bool,
+                                fill_value=fill_value)
+                b = self._klass(rvalues, kind=kind, dtype=np.bool,
+                                fill_value=fill_value)
+                self._check_logical_ops(a, b, values, rvalues)
+
 
 class TestSparseSeriesArithmetic(TestSparseArrayArithmetics):
 

pandas/src/sparse.pyx

@@ -758,6 +758,7 @@ cdef class BlockUnion(BlockMerge):
 include "sparse_op_helper.pxi"
 
 
+
 #-------------------------------------------------------------------------------
 # Indexing operations