Deprecate SparseDataFrame and SparseSeries (#26137)

Author: TomAugspurger

doc/source/user_guide/sparse.rst

@@ -299,14 +299,39 @@ Other API Changes
 Deprecations
 ~~~~~~~~~~~~
 
+Sparse Subclasses
+^^^^^^^^^^^^^^^^^
+
+The ``SparseSeries`` and ``SparseDataFrame`` subclasses are deprecated. Their functionality is better-provided
+by a ``Series`` or ``DataFrame`` with sparse values.
+
+**Previous Way**
+
+.. ipython:: python
+   :okwarning:
+
+   df = pd.SparseDataFrame({"A": [0, 0, 1, 2]})
+   df.dtypes
+
+**New Way**
+
+.. ipython:: python
+
+   df = pd.DataFrame({"A": pd.SparseArray([0, 0, 1, 2])})
+   df.dtypes
+
+The memory usage of the two approaches is identical. See :ref:`sparse.migration` for more (:issue:`19239`).
+
+Other Deprecations
+^^^^^^^^^^^^^^^^^^
+
 - The deprecated ``.ix[]`` indexer now raises a more visible FutureWarning instead of DeprecationWarning (:issue:`26438`).
 - Deprecated the ``units=M`` (months) and ``units=Y`` (year) parameters for ``units`` of :func:`pandas.to_timedelta`, :func:`pandas.Timedelta` and :func:`pandas.TimedeltaIndex` (:issue:`16344`)
 - The :attr:`SparseArray.values` attribute is deprecated. You can use ``np.asarray(...)`` or
   the :meth:`SparseArray.to_dense` method instead (:issue:`26421`).
 - The functions :func:`pandas.to_datetime` and :func:`pandas.to_timedelta` have deprecated the ``box`` keyword. Instead, use :meth:`to_numpy` or :meth:`Timestamp.to_datetime64` or :meth:`Timedelta.to_timedelta64`. (:issue:`24416`)
 - The :meth:`DataFrame.compound` and :meth:`Series.compound` methods are deprecated and will be removed in a future version (:issue:`26405`).
 
-
 .. _whatsnew_0250.prior_deprecations:
 
 Removal of prior version deprecations/changes

doc/source/whatsnew/v0.25.0.rst

@@ -2014,9 +2014,9 @@ def from_coo(cls, A, dense_index=False):
         from pandas.core.sparse.scipy_sparse import _coo_to_sparse_series
         from pandas import Series
 
-        result = _coo_to_sparse_series(A, dense_index=dense_index)
-        # SparseSeries -> Series[sparse]
-        result = Series(result.values, index=result.index, copy=False)
+        result = _coo_to_sparse_series(A, dense_index=dense_index,
+                                       sparse_series=False)
+        result = Series(result.array, index=result.index, copy=False)
 
         return result
 

pandas/core/arrays/sparse.py

@@ -1930,13 +1930,13 @@ def to_sparse(self, fill_value=None, kind='block'):
         >>> type(df)
         <class 'pandas.core.frame.DataFrame'>
 
-        >>> sdf = df.to_sparse()
-        >>> sdf
+        >>> sdf = df.to_sparse()  # doctest: +SKIP
+        >>> sdf  # doctest: +SKIP
              0    1
         0  NaN  NaN
         1  1.0  NaN
         2  NaN  1.0
-        >>> type(sdf)
+        >>> type(sdf)  # doctest: +SKIP
         <class 'pandas.core.sparse.frame.SparseDataFrame'>
         """
         from pandas.core.sparse.api import SparseDataFrame

pandas/core/frame.py

@@ -5589,7 +5589,7 @@ def ftypes(self):
         3    float64:dense
         dtype: object
 
-        >>> pd.SparseDataFrame(arr).ftypes
+        >>> pd.SparseDataFrame(arr).ftypes  # doctest: +SKIP
         0    float64:sparse
         1    float64:sparse
         2    float64:sparse

pandas/core/generic.py

@@ -1586,7 +1586,6 @@ def to_sparse(self, kind='block', fill_value=None):
         SparseSeries
             Sparse representation of the Series.
         """
-        # TODO: deprecate
         from pandas.core.sparse.series import SparseSeries
 
         values = SparseArray(self, kind=kind, fill_value=fill_value)

pandas/core/series.py

@@ -28,13 +28,24 @@
 from pandas.core.sparse.series import SparseSeries
 
 _shared_doc_kwargs = dict(klass='SparseDataFrame')
+depr_msg = """\
+SparseDataFrame is deprecated and will be removed in a future version.
+Use a regular DataFrame whose columns are SparseArrays instead.
+
+See http://pandas.pydata.org/pandas-docs/stable/\
+user_guide/sparse.html#migrating for more.
+"""
 
 
 class SparseDataFrame(DataFrame):
     """
     DataFrame containing sparse floating point data in the form of SparseSeries
     objects
 
+    .. deprectaed:: 0.25.0
+
+       Use a DataFrame with sparse values instead.
+
     Parameters
     ----------
     data : same types as can be passed to DataFrame or scipy.sparse.spmatrix
@@ -56,6 +67,7 @@ class SparseDataFrame(DataFrame):
     def __init__(self, data=None, index=None, columns=None, default_kind=None,
                  default_fill_value=None, dtype=None, copy=False):
 
+        warnings.warn(depr_msg, FutureWarning, stacklevel=2)
         # pick up the defaults from the Sparse structures
         if isinstance(data, SparseDataFrame):
             if index is None:

pandas/core/sparse/frame.py

@@ -116,14 +116,32 @@ def _sparse_series_to_coo(ss, row_levels=(0, ), column_levels=(1, ),
     return sparse_matrix, rows, columns
 
 
-def _coo_to_sparse_series(A, dense_index=False):
+def _coo_to_sparse_series(A, dense_index: bool = False,
+                          sparse_series: bool = True):
     """
     Convert a scipy.sparse.coo_matrix to a SparseSeries.
-    Use the defaults given in the SparseSeries constructor.
+
+    Parameters
+    ----------
+    A : scipy.sparse.coo.coo_matrix
+    dense_index : bool, default False
+    sparse_series : bool, default True
+
+    Returns
+    -------
+    Series or SparseSeries
     """
+    from pandas import SparseDtype
+
     s = Series(A.data, MultiIndex.from_arrays((A.row, A.col)))
     s = s.sort_index()
-    s = s.to_sparse()  # TODO: specify kind?
+    if sparse_series:
+        # TODO(SparseSeries): remove this and the sparse_series keyword.
+        # This is just here to avoid a DeprecationWarning when
+        # _coo_to_sparse_series is called via Series.sparse.from_coo
+        s = s.to_sparse()  # TODO: specify kind?
+    else:
+        s = s.astype(SparseDtype(s.dtype))
     if dense_index:
         # is there a better constructor method to use here?
         i = range(A.shape[0])

pandas/core/sparse/scipy_sparse.py

@@ -32,9 +32,24 @@
                           optional_labels='', optional_axis='')
 
 
+depr_msg = """\
+SparseSeries is deprecated and will be removed in a future version.
+Use a Series with sparse values instead.
+
+    >>> series = pd.Series(pd.SparseArray(...))
+
+See http://pandas.pydata.org/pandas-docs/stable/\
+user_guide/sparse.html#migrating for more.
+"""
+
+
 class SparseSeries(Series):
     """Data structure for labeled, sparse floating point data
 
+    .. deprectaed:: 0.25.0
+
+       Use a Series with sparse values instead.
+
     Parameters
     ----------
     data : {array-like, Series, SparseSeries, dict}
@@ -60,6 +75,7 @@ class SparseSeries(Series):
     def __init__(self, data=None, index=None, sparse_index=None, kind='block',
                  fill_value=None, name=None, dtype=None, copy=False,
                  fastpath=False):
+        warnings.warn(depr_msg, FutureWarning, stacklevel=2)
         # TODO: Most of this should be refactored and shared with Series
         # 1. BlockManager -> array
         # 2. Series.index, Series.name, index, name reconciliation

pandas/core/sparse/series.py

@@ -101,3 +101,21 @@ def test_density(self):
         res = df.sparse.density
         expected = 0.75
         assert res == expected
+
+    @pytest.mark.parametrize("dtype", ['int64', 'float64'])
+    @pytest.mark.parametrize("dense_index", [True, False])
+    @td.skip_if_no_scipy
+    def test_series_from_coo(self, dtype, dense_index):
+        import scipy.sparse
+
+        A = scipy.sparse.eye(3, format='coo', dtype=dtype)
+        result = pd.Series.sparse.from_coo(A, dense_index=dense_index)
+        index = pd.MultiIndex.from_tuples([(0, 0), (1, 1), (2, 2)])
+        expected = pd.Series(pd.SparseArray(np.array([1, 1, 1], dtype=dtype)),
+                             index=index)
+        if dense_index:
+            expected = expected.reindex(
+                pd.MultiIndex.from_product(index.levels)
+            )
+
+        tm.assert_series_equal(result, expected)