Fix assignment to multiple columns when some column do not exist

Author: Si Wei How

doc/source/whatsnew/v0.25.0.rst

@@ -425,6 +425,7 @@ Indexing
 - Bug in which :meth:`DataFrame.to_csv` caused a segfault for a reindexed data frame, when the indices were single-level :class:`MultiIndex` (:issue:`26303`).
 - Fixed bug where assigning a :class:`arrays.PandasArray` to a :class:`pandas.core.frame.DataFrame` would raise error (:issue:`26390`)
 - Allow keyword arguments for callable local reference used in the :method:`DataFrame.query` string (:issue:`26426`)
+- Bug in assignment to multiple columns of a `DataFrame` when some of the columns do not exist (:issue:`13658`)
 
 
 Missing

pandas/core/frame.py

@@ -3373,7 +3373,8 @@ def _setitem_array(self, key, value):
                 for k1, k2 in zip(key, value.columns):
                     self[k1] = value[k2]
             else:
-                indexer = self.loc._convert_to_indexer(key, axis=1)
+                indexer = self.loc._convert_to_indexer(key, axis=1,
+                                                       is_setter=True)
                 self._check_setitem_copy()
                 self.loc._setitem_with_indexer((slice(None), indexer), value)
 

pandas/core/indexing.py

@@ -304,6 +304,14 @@ def _setitem_with_indexer(self, indexer, value):
         # maybe partial set
         take_split_path = self.obj._is_mixed_type
 
+        has_missing_columns = (
+            isinstance(indexer, tuple) and
+            is_list_like_indexer(indexer[info_axis]) and
+            any(isinstance(k, dict) for k in indexer[1]))
+
+        if has_missing_columns:
+            take_split_path = True
+
         # if there is only one block/type, still have to take split path
         # unless the block is one-dimensional or it can hold the value
         if not take_split_path and self.obj._data.blocks:
@@ -466,14 +474,19 @@ def _setitem_with_indexer(self, indexer, value):
             if isinstance(value, ABCSeries):
                 value = self._align_series(indexer, value)
 
-            info_idx = indexer[info_axis]
-            if is_integer(info_idx):
-                info_idx = [info_idx]
-            labels = item_labels[info_idx]
+            if has_missing_columns:
+                labels = [item_labels[idx] if is_integer(idx) else idx['key']
+                          for idx in indexer[info_axis]]
+            else:
+                info_idx = indexer[info_axis]
+                if is_integer(info_idx):
+                    info_idx = [info_idx]
+                labels = item_labels[info_idx]
 
             # if we have a partial multiindex, then need to adjust the plane
             # indexer here
             if (len(labels) == 1 and
+                    labels[0] in self.obj.columns and
                     isinstance(self.obj[labels[0]].axes[0], MultiIndex)):
                 item = labels[0]
                 obj = self.obj[item]
@@ -517,7 +530,9 @@ def _setitem_with_indexer(self, indexer, value):
                     lplane_indexer = 0
 
             def setter(item, v):
-                s = self.obj[item]
+                is_missing_column = item not in self.obj.columns
+                if not is_missing_column:
+                    s = self.obj[item]
                 pi = plane_indexer[0] if lplane_indexer == 1 else plane_indexer
 
                 # perform the equivalent of a setitem on the info axis
@@ -529,8 +544,14 @@ def setter(item, v):
                         all(com.is_null_slice(idx) or
                             com.is_full_slice(idx, len(self.obj))
                             for idx in pi)):
+                    if is_missing_column:
+                        self.obj[item] = v
+                        return
                     s = v
                 else:
+                    if is_missing_column:
+                        self[plane_indexer[0], item] = v
+                        return
                     # set the item, possibly having a dtype change
                     s._consolidate_inplace()
                     s = s.copy()
@@ -1352,6 +1373,13 @@ def _convert_to_indexer(self, obj, axis=None, is_setter=False,
                 inds, = obj.nonzero()
                 return inds
             else:
+                # allow missing columns when setting
+                if is_setter and axis == 1:
+                    return [
+                        self._convert_to_indexer(k, axis=axis,
+                                                 is_setter=is_setter,
+                                                 raise_missing=raise_missing)
+                        for k in obj]
                 # When setting, missing keys are not allowed, even with .loc:
                 kwargs = {'raise_missing': True if is_setter else
                           raise_missing}

pandas/tests/frame/test_indexing.py

@@ -18,7 +18,8 @@
 from pandas.tests.frame.common import TestData
 import pandas.util.testing as tm
 from pandas.util.testing import (
-    assert_almost_equal, assert_frame_equal, assert_series_equal)
+    assert_almost_equal, assert_frame_equal, assert_index_equal,
+    assert_numpy_array_equal, assert_series_equal)
 
 from pandas.tseries.offsets import BDay
 
@@ -185,6 +186,51 @@ def test_setitem_list_of_tuples(self):
         expected = Series(tuples, index=self.frame.index, name='tuples')
         assert_series_equal(result, expected)
 
+    def test_setitem_list_all_missing_columns_scalar(self):
+        df = self.frame.copy()
+        df[['E', 'F']] = 1
+
+        result = df.columns
+        expected = self.frame.columns.union(['E', 'F'])
+        assert_index_equal(result, expected)
+
+        result = df[self.frame.columns]
+        expected = self.frame
+        assert_frame_equal(result, expected)
+
+        assert (df[['E', 'F']] == 1).all().all()
+
+    def test_setitem_list_some_missing_columns_list(self):
+        df = self.frame.copy()
+        df[['A', 'E']] = [1, 2]
+
+        result = df.columns
+        expected = self.frame.columns.union(['E'])
+        assert_index_equal(result, expected)
+
+        result = df[self.frame.columns.drop('A')]
+        expected = self.frame.drop('A', axis=1)
+        assert_frame_equal(result, expected)
+
+        assert (df['A'] == 1).all()
+        assert (df['E'] == 2).all()
+
+    def test_setitem_list_some_missing_columns_dataframe(self):
+        df = self.frame.copy()
+        df[['A', 'E']] = self.frame[['B', 'C']]
+
+        result = df.columns
+        expected = self.frame.columns.union(['E'])
+        assert_index_equal(result, expected)
+
+        result = df[self.frame.columns.drop('A')]
+        expected = self.frame.drop('A', axis=1)
+        assert_frame_equal(result, expected)
+
+        result = df[['A', 'E']]
+        expected = self.frame[['B', 'C']]
+        assert_numpy_array_equal(result.to_numpy(), expected.to_numpy())
+
     def test_setitem_mulit_index(self):
         # GH7655, test that assigning to a sub-frame of a frame
         # with multi-index columns aligns both rows and columns
@@ -454,9 +500,6 @@ def test_setitem(self):
         self.frame['col6'] = series
         tm.assert_series_equal(series, self.frame['col6'], check_names=False)
 
-        with pytest.raises(KeyError):
-            self.frame[np.random.randn(len(self.frame) + 1)] = 1
-
         # set ndarray
         arr = np.random.randn(len(self.frame))
         self.frame['col9'] = arr
@@ -1093,10 +1136,11 @@ def test_fancy_index_int_labels_exceptions(self):
                    r" dtype='object'\)\] are in the \[index\]")
             with pytest.raises(KeyError, match=msg):
                 self.frame.ix[['foo', 'bar', 'baz']] = 1
-            msg = (r"None of \[Index\(\['E'\], dtype='object'\)\] are in the"
-                   r" \[columns\]")
-            with pytest.raises(KeyError, match=msg):
-                self.frame.ix[:, ['E']] = 1
+            # partial setting now allows this GH13658
+            # msg = (r"None of \[Index\(\['E'\], dtype='object'\)\] are in the"
+            #        r" \[columns\]")
+            # with pytest.raises(KeyError, match=msg):
+            #     self.frame.ix[:, ['E']] = 1
 
             # partial setting now allows this GH2578
             # pytest.raises(KeyError, self.frame.ix.__setitem__,