Fix assignment to multiple columns when some column do not exist

Author: Si Wei How

doc/source/whatsnew/v0.25.0.rst

@@ -425,6 +425,7 @@ Indexing
 - Bug in which :meth:`DataFrame.to_csv` caused a segfault for a reindexed data frame, when the indices were single-level :class:`MultiIndex` (:issue:`26303`).
 - Fixed bug where assigning a :class:`arrays.PandasArray` to a :class:`pandas.core.frame.DataFrame` would raise error (:issue:`26390`)
 - Allow keyword arguments for callable local reference used in the :method:`DataFrame.query` string (:issue:`26426`)
+- Bug in assignment to multiple columns of a `DataFrame` when some of the columns do not exist (:issue:`13658`)
 
 
 Missing

pandas/core/frame.py

@@ -3371,11 +3371,13 @@ def _setitem_array(self, key, value):
                 if len(value.columns) != len(key):
                     raise ValueError('Columns must be same length as key')
                 for k1, k2 in zip(key, value.columns):
-                    self[k1] = value[k2]
+                    self._set_item(k1, value[k2])
             else:
-                indexer = self.loc._convert_to_indexer(key, axis=1)
+                indexer = self.loc._convert_to_indexer(key, axis=1,
+                                                       is_setter=True)
                 self._check_setitem_copy()
-                self.loc._setitem_with_indexer((slice(None), indexer), value)
+                self.loc._setitem_with_indexer((slice(None), indexer), value,
+                                               columns=key)
 
     def _setitem_frame(self, key, value):
         # support boolean setting with DataFrame input, e.g.

pandas/core/indexing.py

@@ -294,7 +294,7 @@ def _has_valid_positional_setitem_indexer(self, indexer):
 
         return True
 
-    def _setitem_with_indexer(self, indexer, value):
+    def _setitem_with_indexer(self, indexer, value, columns=None):
         self._has_valid_setitem_indexer(indexer)
 
         # also has the side effect of consolidating in-place
@@ -304,6 +304,11 @@ def _setitem_with_indexer(self, indexer, value):
         # maybe partial set
         take_split_path = self.obj._is_mixed_type
 
+        # if not all columns are present
+        if (columns is not None and
+                any(k not in self.obj.columns for k in columns)):
+            take_split_path = True
+
         # if there is only one block/type, still have to take split path
         # unless the block is one-dimensional or it can hold the value
         if not take_split_path and self.obj._data.blocks:
@@ -466,10 +471,13 @@ def _setitem_with_indexer(self, indexer, value):
             if isinstance(value, ABCSeries):
                 value = self._align_series(indexer, value)
 
-            info_idx = indexer[info_axis]
-            if is_integer(info_idx):
-                info_idx = [info_idx]
-            labels = item_labels[info_idx]
+            if columns is None:
+                info_idx = indexer[info_axis]
+                if is_integer(info_idx):
+                    info_idx = [info_idx]
+                labels = item_labels[info_idx]
+            else:
+                labels = columns
 
             # if we have a partial multiindex, then need to adjust the plane
             # indexer here
@@ -517,6 +525,9 @@ def _setitem_with_indexer(self, indexer, value):
                     lplane_indexer = 0
 
             def setter(item, v):
+                if item not in self.obj.columns:
+                    self.obj[item] = v
+                    return
                 s = self.obj[item]
                 pi = plane_indexer[0] if lplane_indexer == 1 else plane_indexer
 
@@ -1109,7 +1120,8 @@ def _getitem_axis(self, key, axis=None):
 
             return self._get_label(key, axis=axis)
 
-    def _get_listlike_indexer(self, key, axis, raise_missing=False):
+    def _get_listlike_indexer(self, key, axis, is_setter=False,
+                              raise_missing=False):
         """
         Transform a list-like of keys into a new index and an indexer.
 
@@ -1162,6 +1174,7 @@ def _get_listlike_indexer(self, key, axis, raise_missing=False):
 
         self._validate_read_indexer(keyarr, indexer,
                                     o._get_axis_number(axis),
+                                    is_setter=is_setter,
                                     raise_missing=raise_missing)
         return keyarr, indexer
 
@@ -1210,7 +1223,8 @@ def _getitem_iterable(self, key, axis=None):
             return self.obj._reindex_with_indexers({axis: [keyarr, indexer]},
                                                    copy=True, allow_dups=True)
 
-    def _validate_read_indexer(self, key, indexer, axis, raise_missing=False):
+    def _validate_read_indexer(self, key, indexer, axis, is_setter=False,
+                               raise_missing=False):
         """
         Check that indexer can be used to return a result (e.g. at least one
         element was found, unless the list of keys was actually empty).
@@ -1244,7 +1258,8 @@ def _validate_read_indexer(self, key, indexer, axis, raise_missing=False):
         missing = (indexer < 0).sum()
 
         if missing:
-            if missing == len(indexer):
+            if (not(self.name == 'loc' and is_setter) and
+                    missing == len(indexer)):
                 raise KeyError(
                     "None of [{key}] are in the [{axis}]".format(
                         key=key, axis=self.obj._get_axis_name(axis)))
@@ -1268,7 +1283,7 @@ def _validate_read_indexer(self, key, indexer, axis, raise_missing=False):
             See the documentation here:
             https://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike""")  # noqa
 
-            if not (ax.is_categorical() or ax.is_interval()):
+            if not is_setter and not (ax.is_categorical() or ax.is_interval()):
                 warnings.warn(_missing_key_warning,
                               FutureWarning, stacklevel=6)
 
@@ -1353,8 +1368,8 @@ def _convert_to_indexer(self, obj, axis=None, is_setter=False,
                 return inds
             else:
                 # When setting, missing keys are not allowed, even with .loc:
-                kwargs = {'raise_missing': True if is_setter else
-                          raise_missing}
+                kwargs = {'is_setter': is_setter,
+                          'raise_missing': raise_missing}
                 return self._get_listlike_indexer(obj, axis, **kwargs)[1]
         else:
             try:

pandas/tests/frame/test_indexing.py

@@ -18,7 +18,8 @@
 from pandas.tests.frame.common import TestData
 import pandas.util.testing as tm
 from pandas.util.testing import (
-    assert_almost_equal, assert_frame_equal, assert_series_equal)
+    assert_almost_equal, assert_frame_equal, assert_index_equal,
+    assert_numpy_array_equal, assert_series_equal)
 
 from pandas.tseries.offsets import BDay
 
@@ -185,6 +186,49 @@ def test_setitem_list_of_tuples(self):
         expected = Series(tuples, index=self.frame.index, name='tuples')
         assert_series_equal(result, expected)
 
+    def test_setitem_list_missing_columns(self):
+        df = self.frame.copy()
+        df[['E', 'F']] = 1
+
+        result = df.columns
+        expected = self.frame.columns.union(['E', 'F'])
+        assert_index_equal(result, expected)
+
+        result = df[self.frame.columns]
+        expected = self.frame
+        assert_frame_equal(result, expected)
+
+        assert (df[['E', 'F']] == 1).all().all()
+
+        df = self.frame.copy()
+        df[['A', 'E']] = [1, 2]
+
+        result = df.columns
+        expected = self.frame.columns.union(['E'])
+        assert_index_equal(result, expected)
+
+        result = df[self.frame.columns.drop('A')]
+        expected = self.frame.drop('A', axis=1)
+        assert_frame_equal(result, expected)
+
+        assert (df['A'] == 1).all()
+        assert (df['E'] == 2).all()
+
+        df = self.frame.copy()
+        df[['A', 'E']] = self.frame[['B', 'C']]
+
+        result = df.columns
+        expected = self.frame.columns.union(['E'])
+        assert_index_equal(result, expected)
+
+        result = df[self.frame.columns.drop('A')]
+        expected = self.frame.drop('A', axis=1)
+        assert_frame_equal(result, expected)
+
+        result = df[['A', 'E']]
+        expected = self.frame[['B', 'C']]
+        assert_numpy_array_equal(result.to_numpy(), expected.to_numpy())
+
     def test_setitem_mulit_index(self):
         # GH7655, test that assigning to a sub-frame of a frame
         # with multi-index columns aligns both rows and columns
@@ -454,9 +498,6 @@ def test_setitem(self):
         self.frame['col6'] = series
         tm.assert_series_equal(series, self.frame['col6'], check_names=False)
 
-        with pytest.raises(KeyError):
-            self.frame[np.random.randn(len(self.frame) + 1)] = 1
-
         # set ndarray
         arr = np.random.randn(len(self.frame))
         self.frame['col9'] = arr