Skip to content

REF: support reso in remaining tslibs helpers #48661

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 21, 2022
Merged

REF: support reso in remaining tslibs helpers #48661

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
747f65c
REF: support reso in remaining tslibs helpers
jbrockmendel Sep 20, 2022
ef213e6
update setup.py
jbrockmendel Sep 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions pandas/_libs/tslib.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ from pandas._libs.tslibs.np_datetime cimport (
NPY_DATETIMEUNIT,
NPY_FR_ns,
check_dts_bounds,
dtstruct_to_dt64,
get_datetime64_value,
npy_datetimestruct,
npy_datetimestruct_to_datetime,
pandas_datetime_to_datetimestruct,
pydate_to_dt64,
pydatetime_to_dt64,
Expand Down Expand Up @@ -95,7 +95,7 @@ def _test_parse_iso8601(ts: str):
return Timestamp.now().normalize()

string_to_dts(ts, &obj.dts, &out_bestunit, &out_local, &out_tzoffset, True)
obj.value = dtstruct_to_dt64(&obj.dts)
obj.value = npy_datetimestruct_to_datetime(NPY_FR_ns, &obj.dts)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW: It looks like this entire function is just unit tested and not used anywhere else so could be removed?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yah, i always figured it was really a test for string_to_dts, would be OK with declaring we already have enough indirect tests of that

check_dts_bounds(&obj.dts)
if out_local == 1:
obj.tzinfo = pytz.FixedOffset(out_tzoffset)
Expand Down Expand Up @@ -547,7 +547,7 @@ cpdef array_to_datetime(

elif is_datetime64_object(val):
seen_datetime = True
iresult[i] = get_datetime64_nanos(val)
iresult[i] = get_datetime64_nanos(val, NPY_FR_ns)

elif is_integer_object(val) or is_float_object(val):
# these must be ns unit by-definition
Expand Down Expand Up @@ -628,7 +628,7 @@ cpdef array_to_datetime(
if not string_to_dts_failed:
# No error reported by string_to_dts, pick back up
# where we left off
value = dtstruct_to_dt64(&dts)
value = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)
if out_local == 1:
seen_datetime_offset = True
# Store the out_tzoffset in seconds
Expand Down
2 changes: 1 addition & 1 deletion pandas/_libs/tslibs/conversion.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ cdef _TSObject convert_datetime_to_tsobject(datetime ts, tzinfo tz,
int32_t nanos=*,
NPY_DATETIMEUNIT reso=*)

cdef int64_t get_datetime64_nanos(object val) except? -1
cdef int64_t get_datetime64_nanos(object val, NPY_DATETIMEUNIT reso) except? -1

cpdef datetime localize_pydatetime(datetime dt, tzinfo tz)
cdef int64_t cast_from_unit(object ts, str unit) except? -1
Expand Down
17 changes: 8 additions & 9 deletions pandas/_libs/tslibs/conversion.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ from pandas._libs.tslibs.np_datetime cimport (
NPY_DATETIMEUNIT,
NPY_FR_ns,
check_dts_bounds,
dtstruct_to_dt64,
get_datetime64_unit,
get_datetime64_value,
get_implementation_bounds,
Expand Down Expand Up @@ -171,7 +170,7 @@ cpdef inline (int64_t, int) precision_from_unit(str unit):
return m, p


cdef inline int64_t get_datetime64_nanos(object val) except? -1:
cdef inline int64_t get_datetime64_nanos(object val, NPY_DATETIMEUNIT reso) except? -1:
"""
Extract the value and unit from a np.datetime64 object, then convert the
value to nanoseconds if necessary.
Expand All @@ -187,10 +186,10 @@ cdef inline int64_t get_datetime64_nanos(object val) except? -1:

unit = get_datetime64_unit(val)

if unit != NPY_FR_ns:
if unit != reso:
pandas_datetime_to_datetimestruct(ival, unit, &dts)
check_dts_bounds(&dts)
ival = dtstruct_to_dt64(&dts)
check_dts_bounds(&dts, reso)
ival = npy_datetimestruct_to_datetime(reso, &dts)

return ival

Expand Down Expand Up @@ -238,7 +237,7 @@ cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
if ts is None or ts is NaT:
obj.value = NPY_NAT
elif is_datetime64_object(ts):
obj.value = get_datetime64_nanos(ts)
obj.value = get_datetime64_nanos(ts, NPY_FR_ns)
if obj.value != NPY_NAT:
pandas_datetime_to_datetimestruct(obj.value, NPY_FR_ns, &obj.dts)
elif is_integer_object(ts):
Expand Down Expand Up @@ -403,7 +402,7 @@ cdef _TSObject _create_tsobject_tz_using_offset(npy_datetimestruct dts,
datetime dt
Py_ssize_t pos

value = dtstruct_to_dt64(&dts)
value = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)
obj.dts = dts
obj.tzinfo = pytz.FixedOffset(tzoffset)
obj.value = tz_localize_to_utc_single(value, obj.tzinfo)
Expand Down Expand Up @@ -490,12 +489,12 @@ cdef _TSObject _convert_str_to_tsobject(object ts, tzinfo tz, str unit,
)
if not string_to_dts_failed:
try:
check_dts_bounds(&dts)
check_dts_bounds(&dts, NPY_FR_ns)
if out_local == 1:
return _create_tsobject_tz_using_offset(dts,
out_tzoffset, tz)
else:
ival = dtstruct_to_dt64(&dts)
ival = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)
if tz is not None:
# shift for _localize_tso
ival = tz_localize_to_utc_single(ival, tz,
Expand Down
10 changes: 6 additions & 4 deletions pandas/_libs/tslibs/np_datetime.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,13 @@ cdef bint cmp_scalar(int64_t lhs, int64_t rhs, int op) except -1

cdef check_dts_bounds(npy_datetimestruct *dts, NPY_DATETIMEUNIT unit=?)

cdef int64_t dtstruct_to_dt64(npy_datetimestruct* dts) nogil

cdef int64_t pydatetime_to_dt64(datetime val, npy_datetimestruct *dts)
cdef int64_t pydatetime_to_dt64(
datetime val, npy_datetimestruct *dts, NPY_DATETIMEUNIT reso=?
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No defaulting reso to NPY_DATETIMEUNIT.NPY_FR_ns?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we'll soon need to be explicit everywhere this is called anyway

)
cdef void pydatetime_to_dtstruct(datetime dt, npy_datetimestruct *dts)
cdef int64_t pydate_to_dt64(date val, npy_datetimestruct *dts)
cdef int64_t pydate_to_dt64(
date val, npy_datetimestruct *dts, NPY_DATETIMEUNIT reso=?
)
cdef void pydate_to_dtstruct(date val, npy_datetimestruct *dts)

cdef npy_datetime get_datetime64_value(object obj) nogil
Expand Down
16 changes: 7 additions & 9 deletions pandas/_libs/tslibs/np_datetime.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,11 +215,6 @@ cdef check_dts_bounds(npy_datetimestruct *dts, NPY_DATETIMEUNIT unit=NPY_FR_ns):
# ----------------------------------------------------------------------
# Conversion

cdef inline int64_t dtstruct_to_dt64(npy_datetimestruct* dts) nogil:
"""Convenience function to call npy_datetimestruct_to_datetime
with the by-far-most-common frequency NPY_FR_ns"""
return npy_datetimestruct_to_datetime(NPY_FR_ns, dts)


# just exposed for testing at the moment
def py_td64_to_tdstruct(int64_t td64, NPY_DATETIMEUNIT unit):
Expand Down Expand Up @@ -247,12 +242,13 @@ cdef inline void pydatetime_to_dtstruct(datetime dt, npy_datetimestruct *dts):


cdef inline int64_t pydatetime_to_dt64(datetime val,
npy_datetimestruct *dts):
npy_datetimestruct *dts,
NPY_DATETIMEUNIT reso=NPY_FR_ns):
"""
Note we are assuming that the datetime object is timezone-naive.
"""
pydatetime_to_dtstruct(val, dts)
return dtstruct_to_dt64(dts)
return npy_datetimestruct_to_datetime(reso, dts)


cdef inline void pydate_to_dtstruct(date val, npy_datetimestruct *dts):
Expand All @@ -263,9 +259,11 @@ cdef inline void pydate_to_dtstruct(date val, npy_datetimestruct *dts):
dts.ps = dts.as = 0
return

cdef inline int64_t pydate_to_dt64(date val, npy_datetimestruct *dts):
cdef inline int64_t pydate_to_dt64(
date val, npy_datetimestruct *dts, NPY_DATETIMEUNIT reso=NPY_FR_ns
):
pydate_to_dtstruct(val, dts)
return dtstruct_to_dt64(dts)
return npy_datetimestruct_to_datetime(reso, dts)


cdef inline int string_to_dts(
Expand Down
5 changes: 3 additions & 2 deletions pandas/_libs/tslibs/strptime.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@ from pandas._libs.tslibs.nattype cimport (
c_nat_strings as nat_strings,
)
from pandas._libs.tslibs.np_datetime cimport (
NPY_FR_ns,
check_dts_bounds,
dtstruct_to_dt64,
npy_datetimestruct,
npy_datetimestruct_to_datetime,
)


Expand Down Expand Up @@ -329,7 +330,7 @@ def array_strptime(ndarray[object] values, str fmt, bint exact=True, errors='rai
dts.us = us
dts.ps = ns * 1000

iresult[i] = dtstruct_to_dt64(&dts)
iresult[i] = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)
try:
check_dts_bounds(&dts)
except ValueError:
Expand Down
1 change: 1 addition & 0 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,7 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
"_libs.tslibs.strptime": {
"pyxfile": "_libs/tslibs/strptime",
"depends": tseries_depends,
"sources": ["pandas/_libs/tslibs/src/datetime/np_datetime.c"],
},
"_libs.tslibs.timedeltas": {
"pyxfile": "_libs/tslibs/timedeltas",
Expand Down