ENH: more refactoring for LongPanel removal, Index._join_level method, DataFrame.reindex_axis, #108

Author: wesm

TODO.rst

@@ -1,3 +1,11 @@
+LongPanel removal
+=================
+
+- level to flex methods
+- level to reindex
+- fast take for items
+
+
 DONE
 ----
 - SparseSeries name integration + tests

pandas/core/frame.py

@@ -1295,12 +1295,13 @@ def _align_frame(self, other, join='outer', axis=None, level=None,
         if axis is None or axis == 0:
             if not self.index.equals(other.index):
                 join_index, ilidx, iridx = \
-                    self.index.join(other.index, how=join, return_indexers=True)
+                    self.index.join(other.index, how=join, level=level,
+                                    return_indexers=True)
 
         if axis is None or axis == 1:
             if not self.columns.equals(other.columns):
                 join_columns, clidx, cridx = \
-                    self.columns.join(other.columns, how=join,
+                    self.columns.join(other.columns, how=join, level=level,
                                       return_indexers=True)
 
         def _align(frame, row_idx, col_idx):
@@ -1310,7 +1311,8 @@ def _align(frame, row_idx, col_idx):
 
             if col_idx is not None:
                 # TODO: speed up on homogeneous DataFrame objects
-                new_data = new_data.reindex_items(join_columns)
+                new_data = new_data.reindex_indexer(join_columns, col_idx,
+                                                    axis=0)
 
             if copy and new_data is frame._data:
                 new_data = new_data.copy()
@@ -1321,6 +1323,15 @@ def _align(frame, row_idx, col_idx):
         right = _align(other, iridx, cridx)
         return left, right
 
+    def _align_level(self, multi_index, level, axis=0, copy=True):
+        levnum = multi_index._get_level_number(level)
+        data = self.reindex_axis(multi_index.levels[levnum], axis=axis,
+                                 copy=False)._data
+        mgr_axis = 0 if axis == 1 else 1
+        new_data = data.reindex_indexer(multi_index, multi_index.labels[levnum],
+                                        axis=mgr_axis)
+        return DataFrame(new_data)
+
     def _align_series(self, other, join='outer', axis=None, level=None,
                       copy=True):
         fdata = self._data
@@ -1337,11 +1348,12 @@ def _align_series(self, other, join='outer', axis=None, level=None,
             join_index = self.columns
             lidx, ridx = None, None
             if not self.columns.equals(other.index):
-                join_index, lidx, ridx = self.columns.join(other.index, how=join,
-                                                           return_indexers=True)
+                join_index, lidx, ridx = \
+                    self.columns.join(other.index, how=join,
+                                      return_indexers=True)
 
             if lidx is not None:
-                fdata = fdata.reindex_items(join_index)
+                fdata = fdata.reindex_indexer(join_index, lidx, axis=0)
         else:
             raise ValueError('Must specify axis=0 or 1')
 
@@ -1354,7 +1366,7 @@ def _align_series(self, other, join='outer', axis=None, level=None,
 
     def reindex(self, index=None, columns=None, method=None, level=None,
                 copy=True):
-        """Conform Series to new index with optional filling logic, placing
+        """Conform DataFrame to new index with optional filling logic, placing
         NA/NaN in locations having no value in the previous index. A new object
         is produced unless the new index is equivalent to the current one and
         copy=False
@@ -1385,33 +1397,65 @@ def reindex(self, index=None, columns=None, method=None, level=None,
         frame = self
 
         if index is not None:
-            index = _ensure_index(index)
             frame = frame._reindex_index(index, method, copy, level)
 
         if columns is not None:
-            columns = _ensure_index(columns)
             frame = frame._reindex_columns(columns, copy, level)
 
         return frame
 
+    def reindex_axis(self, labels, axis=0, method=None, level=None, copy=True):
+        """Conform DataFrame to new index with optional filling logic, placing
+        NA/NaN in locations having no value in the previous index. A new object
+        is produced unless the new index is equivalent to the current one and
+        copy=False
+
+        Parameters
+        ----------
+        index : array-like, optional
+            New labels / index to conform to. Preferably an Index object to
+            avoid duplicating data
+        axis : {0, 1}
+            0 -> index (rows)
+            1 -> columns
+        method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
+            Method to use for filling holes in reindexed DataFrame
+            pad / ffill: propagate last valid observation forward to next valid
+            backfill / bfill: use NEXT valid observation to fill gap
+        copy : boolean, default True
+            Return a new object, even if the passed indexes are the same
+
+        Examples
+        --------
+        >>> df.reindex(['A', 'B', 'C'], axis=1)
+
+        See also
+        --------
+        DataFrame.reindex, DataFrame.reindex_like
+
+        Returns
+        -------
+        reindexed : same type as calling instance
+        """
+        self._consolidate_inplace()
+        if axis == 0:
+            return self._reindex_index(labels, method, copy, level)
+        elif axis == 1:
+            return self._reindex_columns(labels, copy, level)
+        else:  # pragma: no cover
+            raise ValueError('Must specify axis=0 or 1')
+
     def _reindex_index(self, new_index, method, copy, level):
-        if new_index.equals(self.index):
-            if copy:
-                result = self.copy()
-                result.index = new_index
-                return result
-            else:
-                return self
-        new_data = self._data.reindex_axis(new_index, method, axis=1)
+        if level is not None:
+            return self._align_level(new_index, level, axis=0, copy=copy)
+        new_data = self._data.reindex_axis(new_index, method, axis=1,
+                                           copy=copy)
         return self._constructor(new_data)
 
-    def _reindex_columns(self, new_columns, copy):
-        if new_columns.equals(self.columns):
-            if copy:
-                return self.copy()
-            else:
-                return self
-        new_data = self._data.reindex_axis(new_columns, axis=0)
+    def _reindex_columns(self, new_columns, copy, level):
+        if level is not None:
+            return self._align_level(new_columns, level, axis=1, copy=copy)
+        new_data = self._data.reindex_axis(new_columns, axis=0, copy=copy)
         return self._constructor(new_data)
 
     def reindex_like(self, other, method=None, copy=True):
@@ -1948,7 +1992,8 @@ def _indexed_same(self, other):
         same_columns = self.columns.equals(other.columns)
         return same_index and same_columns
 
-    def _combine_series(self, other, func, fill_value=None, axis=None):
+    def _combine_series(self, other, func, fill_value=None, axis=None,
+                        level=None):
         if axis is not None:
             axis = self._get_axis_name(axis)
             if axis == 'index':
@@ -3659,16 +3704,6 @@ def _is_sequence(x):
     except Exception:
         return False
 
-def _align_level(frame, multi_index, level, axis=0):
-    levnum = multi_index._get_level_number(level)
-
-    data = frame.reindex(multi_index.levels[levnum], copy=False)._data
-
-    mgr_axis = 0 if axis == 1 else 1
-    new_data = data.reindex_indexer(multi_index, multi_index.labels[levnum],
-                                    axis=mgr_axis)
-    return DataFrame(new_data)
-
 def install_ipython_completers():  # pragma: no cover
     """Register the DataFrame type with IPython's tab completion machinery, so
     that it knows about accessing column names as attributes."""

pandas/core/generic.py

@@ -370,17 +370,13 @@ def _is_mixed_type(self):
         return len(self._data.blocks) > 1
 
     def _reindex_axis(self, new_index, fill_method, axis, copy):
-        new_index = _ensure_index(new_index)
-        cur_axis = self._data.axes[axis]
-        if cur_axis.equals(new_index) and not copy:
-            return self
+        new_data = self._data.reindex_axis(new_index, axis=axis,
+                                           method=fill_method, copy=copy)
 
-        if axis == 0:
-            new_data = self._data.reindex_items(new_index)
+        if new_data is self._data and not copy:
+            return self
         else:
-            new_data = self._data.reindex_axis(new_index, axis=axis,
-                                               method=fill_method)
-        return self._constructor(new_data)
+            return self._constructor(new_data)
 
     def cumsum(self, axis=None, skipna=True):
         """
@@ -586,7 +582,7 @@ def take(self, indices, axis=0):
         if axis == 0:
             labels = self._get_axis(axis)
             new_items = labels.take(indices)
-            new_data = self._data.reindex_items(new_items)
+            new_data = self._data.reindex_axis(new_items, axis=0)
         else:
             new_data = self._data.take(indices, axis=axis)
         return self._constructor(new_data)

pandas/core/index.py

@@ -559,11 +559,32 @@ def reindex(self, target, method=None):
         indexer = self.get_indexer(target, method=method)
         return target, indexer
 
-    def join(self, other, how='left', return_indexers=False):
+    def join(self, other, how='left', level=None, return_indexers=False):
+        """
+        Internal API method. Compute join_index and indexers to conform data
+        structures to the new index.
+
+        Parameters
+        ----------
+        other : Index
+        how : {'left', 'right', 'inner', 'outer'}
+        level :
+        return_indexers : boolean, default False
+
+        Returns
+        -------
+        join_index, (left_indexer, right_indexer)
+        """
+        if (level is not None and (isinstance(self, MultiIndex) or
+                                   isinstance(other, MultiIndex))):
+            return self._join_level(other, level, how=how,
+                                    return_indexers=return_indexers)
+
         if self.dtype != other.dtype:
             this = self.astype('O')
             other = other.astype('O')
-            return this.join(other, how=how, return_indexers=return_indexers)
+            return this.join(other, how=how,
+                             return_indexers=return_indexers)
 
         if self.is_monotonic and other.is_monotonic:
             return self._join_monotonic(other, how=how,
@@ -593,6 +614,61 @@ def join(self, other, how='left', return_indexers=False):
         else:
             return join_index
 
+    def _join_level(self, other, level, how='left', return_indexers=False):
+        """
+        The join method *only* affects the level of the resulting
+        MultiIndex. Otherwise it just exactly aligns the Index data to the
+        labels of the level in the MultiIndex. The order of the data indexed by
+        the MultiIndex will not be changed (currently)
+        """
+
+        if isinstance(self, MultiIndex) and isinstance(other, MultiIndex):
+            raise Exception('Join on level between two MultiIndex objects '
+                            'is ambiguous')
+
+        left, right = self, other
+
+        flip_order = not isinstance(self, MultiIndex)
+        if flip_order:
+            left, right = right, left
+
+        level = left._get_level_number(level)
+
+        old_level = left.levels[level]
+
+        new_level, left_lev_indexer, right_lev_indexer = \
+            old_level.join(right, how=how, return_indexers=True)
+
+        if left_lev_indexer is not None:
+            rev_indexer = lib.get_reverse_indexer(left_lev_indexer,
+                                                  len(old_level))
+
+            new_labels = list(left.labels)
+            new_labels[level] = rev_indexer.take(left.labels[level])
+
+            new_levels = list(left.levels)
+            new_levels[level] = new_level
+
+            join_index = MultiIndex(levels=new_levels, labels=new_labels,
+                                    names=left.names)
+        else:
+            join_index = left
+
+        left_indexer = None
+
+        if right_lev_indexer is not None:
+            right_indexer = right_lev_indexer.take(join_index.labels[level])
+        else:
+            right_indexer = join_index.labels[level]
+
+        if flip_order:
+            left_indexer, right_indexer = right_indexer, left_indexer
+
+        if return_indexers:
+            return join_index, left_indexer, right_indexer
+        else:
+            return join_index
+
     def _join_monotonic(self, other, how='left', return_indexers=False):
         if how == 'left':
             join_index = self