cover all timezone

Author: Davies Liu

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala

@@ -854,14 +854,18 @@ object DateTimeUtils {
    * Lookup the offset for given millis seconds since 1970-01-01 00:00:00 in a timezone.
    */
   private def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone): Long = {
-    val rawOffset = tz.getRawOffset
+    var guess = tz.getRawOffset
     // the actual offset should be calculated based on milliseconds in UTC
-    var offset = tz.getOffset(millisLocal - rawOffset)
-    if (offset != rawOffset) {
-      // Could be in DST, try with best effort
-      offset = tz.getOffset(millisLocal - offset)
+    var actual = tz.getOffset(millisLocal - guess)
+    // At the start of DST, the local time is forwarded by an hour, so it's OK to get the
+    // local time bigger than current one after an round trip (local -> UTC -> local).
+    // At the end of DST, the local time is backwarded by an hour, actual offset will be
+    // less than guess, we should decrease the guess and try again.
+    while (actual < guess) {
+      guess = actual
+      actual = tz.getOffset(millisLocal - guess)
     }
-    offset
+    guess
   }
 
   /**
@@ -883,4 +887,14 @@ object DateTimeUtils {
     val offset = getOffsetFromLocalMillis(time / 1000L, tz)
     time - offset * 1000L
   }
+
+  /**
+   * Re-initialize the current thread's thread locals. Exposed for testing.
+   */
+  private[util] def resetThreadLocals(): Unit = {
+    threadLocalGmtCalendar.remove()
+    threadLocalLocalTimeZone.remove()
+    threadLocalTimestampFormat.remove()
+    threadLocalDateFormat.remove()
+  }
 }

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.util.TimeZone
+
+/**
+ * Helper functions for testing date and time functionality.
+ */
+object DateTimeTestUtils {
+
+  val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone)
+
+  def withDefaultTimeZone[T](newDefaultTimeZone: TimeZone)(block: => T): T = {
+    val originalDefaultTimeZone = TimeZone.getDefault
+    try {
+      DateTimeUtils.resetThreadLocals()
+      TimeZone.setDefault(newDefaultTimeZone)
+      block
+    } finally {
+      TimeZone.setDefault(originalDefaultTimeZone)
+      DateTimeUtils.resetThreadLocals()
+    }
+  }
+}

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala

@@ -524,11 +524,12 @@ class DateTimeUtilsSuite extends SparkFunSuite {
   }
 
   test("daysToMillis and millisToDays") {
-    (-1001 to 2000).foreach { d =>
-      assert(millisToDays(daysToMillis(d)) === d)
-      val date = toJavaTimestamp(daysToMillis(d) * 1000L)
-      assert(date.getHours === 0)
-      assert(date.getMinutes === 0)
+     for (tz <- DateTimeTestUtils.ALL_TIMEZONES) {
+      DateTimeTestUtils.withDefaultTimeZone(tz) {
+        (-2000 to 2000).foreach { d =>
+          assert(millisToDays(daysToMillis(d)) === d)
+        }
+      }
     }
   }
 }