ENH: DTA to_pydatetime, time, timetz, date, iter support non-nano (#47307)

* ENH: Timestamp +- timedeltalike scalar support non-nano

* ENH: Timestamp.__sub__(datetime) with non-nano

* better exception message

* BUG: concat not sorting mixed column names when None is included (#47331)

* REGR: concat not sorting columns for mixed column names

* Fix none in columns

* BUG: concat not sorting column names when None is included

* Update doc/source/whatsnew/v1.5.0.rst

Co-authored-by: Matthew Roeschke <[email protected]>

* Add gh reference

Co-authored-by: Matthew Roeschke <[email protected]>

* Add run-tests action (#47292)

* Add run-tests action

* Fix

* Fix

* Fix

* Update macos-windows.yml

* Update posix.yml

* Update python-dev.yml

* Update action.yml

* Update macos-windows.yml

* Update posix.yml

* Update python-dev.yml

* Update python-dev.yml

* Update python-dev.yml

* Update python-dev.yml

* Update python-dev.yml

* Update python-dev.yml

* Update python-dev.yml

* Update python-dev.yml

* Update python-dev.yml

* ENH: Timestamp pickle support non-nano tzaware (#47340)

* ENH: DTA to_pydatetime, time, timetz, date, iter support non-nano

* cast in liboffsets

* mypy fixup

Co-authored-by: Patrick Hoefler <[email protected]>
Co-authored-by: Matthew Roeschke <[email protected]>
Co-authored-by: Jonas Haag <[email protected]>

Author: 4 people

pandas/_libs/tslibs/offsets.pyx

@@ -3115,7 +3115,7 @@ cdef class FY5253Quarter(FY5253Mixin):
             for qlen in qtr_lens:
                 if qlen * 7 <= tdelta.days:
                     num_qtrs += 1
-                    tdelta -= Timedelta(days=qlen * 7)
+                    tdelta -= (<_Timedelta>Timedelta(days=qlen * 7))._as_reso(norm._reso)
                 else:
                     break
         else:

pandas/_libs/tslibs/vectorized.pyi

@@ -37,6 +37,7 @@ def ints_to_pydatetime(
     freq: BaseOffset | None = ...,
     fold: bool = ...,
     box: str = ...,
+    reso: int = ...,  # NPY_DATETIMEUNIT
 ) -> npt.NDArray[np.object_]: ...
 def tz_convert_from_utc(
     stamps: npt.NDArray[np.int64],

pandas/_libs/tslibs/vectorized.pyx

@@ -100,7 +100,8 @@ def ints_to_pydatetime(
     tzinfo tz=None,
     BaseOffset freq=None,
     bint fold=False,
-    str box="datetime"
+    str box="datetime",
+    NPY_DATETIMEUNIT reso=NPY_FR_ns,
 ) -> np.ndarray:
     # stamps is int64, arbitrary ndim
     """
@@ -126,12 +127,14 @@ def ints_to_pydatetime(
         * If time, convert to datetime.time
         * If Timestamp, convert to pandas.Timestamp
 
+    reso : NPY_DATETIMEUNIT, default NPY_FR_ns
+
     Returns
     -------
     ndarray[object] of type specified by box
     """
     cdef:
-        Localizer info = Localizer(tz, reso=NPY_FR_ns)
+        Localizer info = Localizer(tz, reso=reso)
         int64_t utc_val, local_val
         Py_ssize_t i, n = stamps.size
         Py_ssize_t pos = -1  # unused, avoid not-initialized warning
@@ -179,10 +182,12 @@ def ints_to_pydatetime(
                 # find right representation of dst etc in pytz timezone
                 new_tz = tz._tzinfos[tz._transition_info[pos]]
 
-            dt64_to_dtstruct(local_val, &dts)
+            pandas_datetime_to_datetimestruct(local_val, reso, &dts)
 
             if use_ts:
-                res_val = create_timestamp_from_ts(utc_val, dts, new_tz, freq, fold)
+                res_val = create_timestamp_from_ts(
+                    utc_val, dts, new_tz, freq, fold, reso=reso
+                )
             elif use_pydt:
                 res_val = datetime(
                     dts.year, dts.month, dts.day, dts.hour, dts.min, dts.sec, dts.us,

pandas/core/arrays/datetimelike.py

@@ -424,17 +424,18 @@ def astype(self, dtype, copy: bool = True):
 
         if is_object_dtype(dtype):
             if self.dtype.kind == "M":
+                self = cast("DatetimeArray", self)
                 # *much* faster than self._box_values
                 #  for e.g. test_get_loc_tuple_monotonic_above_size_cutoff
-                i8data = self.asi8.ravel()
+                i8data = self.asi8
                 converted = ints_to_pydatetime(
                     i8data,
-                    # error: "DatetimeLikeArrayMixin" has no attribute "tz"
-                    tz=self.tz,  # type: ignore[attr-defined]
+                    tz=self.tz,
                     freq=self.freq,
                     box="timestamp",
+                    reso=self._reso,
                 )
-                return converted.reshape(self.shape)
+                return converted
 
             elif self.dtype.kind == "m":
                 return ints_to_pytimedelta(self._ndarray, box=True)

pandas/core/arrays/datetimes.py

@@ -653,7 +653,11 @@ def __iter__(self):
                 start_i = i * chunksize
                 end_i = min((i + 1) * chunksize, length)
                 converted = ints_to_pydatetime(
-                    data[start_i:end_i], tz=self.tz, freq=self.freq, box="timestamp"
+                    data[start_i:end_i],
+                    tz=self.tz,
+                    freq=self.freq,
+                    box="timestamp",
+                    reso=self._reso,
                 )
                 yield from converted
 
@@ -1044,7 +1048,7 @@ def to_pydatetime(self) -> npt.NDArray[np.object_]:
         -------
         datetimes : ndarray[object]
         """
-        return ints_to_pydatetime(self.asi8, tz=self.tz)
+        return ints_to_pydatetime(self.asi8, tz=self.tz, reso=self._reso)
 
     def normalize(self) -> DatetimeArray:
         """
@@ -1301,7 +1305,7 @@ def time(self) -> npt.NDArray[np.object_]:
         # keeping their timezone and not using UTC
         timestamps = self._local_timestamps()
 
-        return ints_to_pydatetime(timestamps, box="time")
+        return ints_to_pydatetime(timestamps, box="time", reso=self._reso)
 
     @property
     def timetz(self) -> npt.NDArray[np.object_]:
@@ -1311,7 +1315,7 @@ def timetz(self) -> npt.NDArray[np.object_]:
 
         The time part of the Timestamps.
         """
-        return ints_to_pydatetime(self.asi8, self.tz, box="time")
+        return ints_to_pydatetime(self.asi8, self.tz, box="time", reso=self._reso)
 
     @property
     def date(self) -> npt.NDArray[np.object_]:
@@ -1326,7 +1330,7 @@ def date(self) -> npt.NDArray[np.object_]:
         # keeping their timezone and not using UTC
         timestamps = self._local_timestamps()
 
-        return ints_to_pydatetime(timestamps, box="date")
+        return ints_to_pydatetime(timestamps, box="date", reso=self._reso)
 
     def isocalendar(self) -> DataFrame:
         """

pandas/tests/arrays/test_datetimes.py

@@ -126,6 +126,35 @@ def test_to_period(self, dta_dti):
 
         tm.assert_extension_array_equal(result, expected)
 
+    def test_iter(self, dta):
+        res = next(iter(dta))
+        expected = dta[0]
+
+        assert type(res) is pd.Timestamp
+        assert res.value == expected.value
+        assert res._reso == expected._reso
+        assert res == expected
+
+    def test_astype_object(self, dta):
+        result = dta.astype(object)
+        assert all(x._reso == dta._reso for x in result)
+        assert all(x == y for x, y in zip(result, dta))
+
+    def test_to_pydatetime(self, dta_dti):
+        dta, dti = dta_dti
+
+        result = dta.to_pydatetime()
+        expected = dti.to_pydatetime()
+        tm.assert_numpy_array_equal(result, expected)
+
+    @pytest.mark.parametrize("meth", ["time", "timetz", "date"])
+    def test_time_date(self, dta_dti, meth):
+        dta, dti = dta_dti
+
+        result = getattr(dta, meth)
+        expected = getattr(dti, meth)
+        tm.assert_numpy_array_equal(result, expected)
+
 
 class TestDatetimeArrayComparisons:
     # TODO: merge this into tests/arithmetic/test_datetime64 once it is