Skip to content

Commit a5ff922

Browse files
committed
[SPARK-28308][CORE] CalendarInterval sub-second part should be padded before parsing
## What changes were proposed in this pull request? The sub-second part of the interval should be padded before parsing. Currently, Spark gives a correct value only when there is 9 digits below `.`. ``` spark-sql> select interval '0 0:0:0.123456789' day to second; interval 123 milliseconds 456 microseconds spark-sql> select interval '0 0:0:0.12345678' day to second; interval 12 milliseconds 345 microseconds spark-sql> select interval '0 0:0:0.1234' day to second; interval 1 microseconds ``` ## How was this patch tested? Pass the Jenkins with the fixed test cases. Closes #25079 from dongjoon-hyun/SPARK-28308. Authored-by: Dongjoon Hyun <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent 28ea445 commit a5ff922

File tree

3 files changed

+9
-5
lines changed

3 files changed

+9
-5
lines changed

common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,8 @@ public static CalendarInterval fromDayTimeString(String s) throws IllegalArgumen
178178
long minutes = toLongWithRange("minute", m.group(5), 0, 59);
179179
long seconds = toLongWithRange("second", m.group(6), 0, 59);
180180
// Hive allow nanosecond precision interval
181-
long nanos = toLongWithRange("nanosecond", m.group(8), 0L, 999999999L);
181+
String nanoStr = m.group(8) == null ? null : (m.group(8) + "000000000").substring(0, 9);
182+
long nanos = toLongWithRange("nanosecond", nanoStr, 0L, 999999999L);
182183
result = new CalendarInterval(0, sign * (
183184
days * MICROS_PER_DAY + hours * MICROS_PER_HOUR + minutes * MICROS_PER_MINUTE +
184185
seconds * MICROS_PER_SECOND + nanos / 1000L));

common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,8 @@ public void fromDayTimeStringTest() {
162162
assertEquals(fromDayTimeString(input), i);
163163

164164
input = "10 0:12:0.888";
165-
i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE);
165+
i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE +
166+
888 * MICROS_PER_MILLI);
166167
assertEquals(fromDayTimeString(input), i);
167168

168169
input = "-3 0:0:0";

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1179,14 +1179,16 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
11791179
}
11801180

11811181
test("Convert hive interval term into Literal of CalendarIntervalType") {
1182+
checkAnswer(sql("select interval '0 0:0:0.1' day to second"),
1183+
Row(CalendarInterval.fromString("interval 100 milliseconds")))
11821184
checkAnswer(sql("select interval '10-9' year to month"),
11831185
Row(CalendarInterval.fromString("interval 10 years 9 months")))
11841186
checkAnswer(sql("select interval '20 15:40:32.99899999' day to second"),
11851187
Row(CalendarInterval.fromString("interval 2 weeks 6 days 15 hours 40 minutes " +
1186-
"32 seconds 99 milliseconds 899 microseconds")))
1188+
"32 seconds 998 milliseconds 999 microseconds")))
11871189
checkAnswer(sql("select interval '15:40:32.99899999' hour to second"),
1188-
Row(CalendarInterval.fromString("interval 15 hours 40 minutes 32 seconds 99 milliseconds " +
1189-
"899 microseconds")))
1190+
Row(CalendarInterval.fromString("interval 15 hours 40 minutes 32 seconds 998 milliseconds " +
1191+
"999 microseconds")))
11901192
checkAnswer(sql("select interval '30' year"),
11911193
Row(CalendarInterval.fromString("interval 30 years")))
11921194
checkAnswer(sql("select interval '25' month"),

0 commit comments

Comments
 (0)