Skip to content

Commit a0e332d

Browse files
HyukjinKwonHyukjinKwon
committed
Use XXX for ISO timezone instead of ZZ which is FastDateFormat specific in CSV/JSON time related options
1 parent c734fc5 commit a0e332d

File tree

8 files changed

+15
-15
lines changed

8 files changed

+15
-15
lines changed

‎python/pyspark/sql/readwriter.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
223223
:param timestampFormat: sets the string that indicates a timestamp format. Custom date
224224
formats follow the formats at ``java.text.SimpleDateFormat``.
225225
This applies to timestamp type. If None is set, it uses the
226-
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSZZ``.
226+
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
227227
:param wholeFile: parse one record, which may span multiple lines, per file. If None is
228228
set, it uses the default value, ``false``.
229229
@@ -363,7 +363,7 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
363363
:param timestampFormat: sets the string that indicates a timestamp format. Custom date
364364
formats follow the formats at ``java.text.SimpleDateFormat``.
365365
This applies to timestamp type. If None is set, it uses the
366-
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSZZ``.
366+
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
367367
:param maxColumns: defines a hard limit of how many columns a record can have. If None is
368368
set, it uses the default value, ``20480``.
369369
:param maxCharsPerColumn: defines the maximum number of characters allowed for any given
@@ -653,7 +653,7 @@ def json(self, path, mode=None, compression=None, dateFormat=None, timestampForm
653653
:param timestampFormat: sets the string that indicates a timestamp format. Custom date
654654
formats follow the formats at ``java.text.SimpleDateFormat``.
655655
This applies to timestamp type. If None is set, it uses the
656-
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSZZ``.
656+
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
657657
658658
>>> df.write.json(os.path.join(tempfile.mkdtemp(), 'data'))
659659
"""
@@ -745,7 +745,7 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
745745
:param timestampFormat: sets the string that indicates a timestamp format. Custom date
746746
formats follow the formats at ``java.text.SimpleDateFormat``.
747747
This applies to timestamp type. If None is set, it uses the
748-
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSZZ``.
748+
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
749749
:param ignoreLeadingWhiteSpace: a flag indicating whether or not leading whitespaces from
750750
values being written should be skipped. If None is set, it
751751
uses the default value, ``true``.

‎python/pyspark/sql/streaming.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,7 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
457457
:param timestampFormat: sets the string that indicates a timestamp format. Custom date
458458
formats follow the formats at ``java.text.SimpleDateFormat``.
459459
This applies to timestamp type. If None is set, it uses the
460-
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSZZ``.
460+
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
461461
:param wholeFile: parse one record, which may span multiple lines, per file. If None is
462462
set, it uses the default value, ``false``.
463463
@@ -581,7 +581,7 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
581581
:param timestampFormat: sets the string that indicates a timestamp format. Custom date
582582
formats follow the formats at ``java.text.SimpleDateFormat``.
583583
This applies to timestamp type. If None is set, it uses the
584-
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSZZ``.
584+
default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
585585
:param maxColumns: defines a hard limit of how many columns a record can have. If None is
586586
set, it uses the default value, ``20480``.
587587
:param maxCharsPerColumn: defines the maximum number of characters allowed for any given

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ private[sql] class JSONOptions(
7979

8080
val timestampFormat: FastDateFormat =
8181
FastDateFormat.getInstance(
82-
parameters.getOrElse("timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSZZ"), timeZone, Locale.US)
82+
parameters.getOrElse("timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), timeZone, Locale.US)
8383

8484
val wholeFile = parameters.get("wholeFile").map(_.toBoolean).getOrElse(false)
8585

‎sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
320320
* <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
321321
* Custom date formats follow the formats at `java.text.SimpleDateFormat`. This applies to
322322
* date type.</li>
323-
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSZZ`): sets the string that
323+
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
324324
* indicates a timestamp format. Custom date formats follow the formats at
325325
* `java.text.SimpleDateFormat`. This applies to timestamp type.</li>
326326
* <li>`wholeFile` (default `false`): parse one record, which may span multiple lines,
@@ -502,7 +502,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
502502
* <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
503503
* Custom date formats follow the formats at `java.text.SimpleDateFormat`. This applies to
504504
* date type.</li>
505-
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSZZ`): sets the string that
505+
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
506506
* indicates a timestamp format. Custom date formats follow the formats at
507507
* `java.text.SimpleDateFormat`. This applies to timestamp type.</li>
508508
* <li>`maxColumns` (default `20480`): defines a hard limit of how many columns

‎sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
477477
* <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
478478
* Custom date formats follow the formats at `java.text.SimpleDateFormat`. This applies to
479479
* date type.</li>
480-
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSZZ`): sets the string that
480+
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
481481
* indicates a timestamp format. Custom date formats follow the formats at
482482
* `java.text.SimpleDateFormat`. This applies to timestamp type.</li>
483483
* </ul>
@@ -583,7 +583,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
583583
* <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
584584
* Custom date formats follow the formats at `java.text.SimpleDateFormat`. This applies to
585585
* date type.</li>
586-
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSZZ`): sets the string that
586+
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
587587
* indicates a timestamp format. Custom date formats follow the formats at
588588
* `java.text.SimpleDateFormat`. This applies to timestamp type.</li>
589589
* <li>`ignoreLeadingWhiteSpace` (default `true`): a flag indicating whether or not leading

‎sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVOptions.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ class CSVOptions(
126126

127127
val timestampFormat: FastDateFormat =
128128
FastDateFormat.getInstance(
129-
parameters.getOrElse("timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSZZ"), timeZone, Locale.US)
129+
parameters.getOrElse("timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), timeZone, Locale.US)
130130

131131
val wholeFile = parameters.get("wholeFile").map(_.toBoolean).getOrElse(false)
132132

‎sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
201201
* <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
202202
* Custom date formats follow the formats at `java.text.SimpleDateFormat`. This applies to
203203
* date type.</li>
204-
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSZZ`): sets the string that
204+
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
205205
* indicates a timestamp format. Custom date formats follow the formats at
206206
* `java.text.SimpleDateFormat`. This applies to timestamp type.</li>
207207
* <li>`wholeFile` (default `false`): parse one record, which may span multiple lines,
@@ -252,7 +252,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
252252
* <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
253253
* Custom date formats follow the formats at `java.text.SimpleDateFormat`. This applies to
254254
* date type.</li>
255-
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSZZ`): sets the string that
255+
* <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
256256
* indicates a timestamp format. Custom date formats follow the formats at
257257
* `java.text.SimpleDateFormat`. This applies to timestamp type.</li>
258258
* <li>`maxColumns` (default `20480`): defines a hard limit of how many columns

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -766,7 +766,7 @@ class CSVSuite extends QueryTest with SharedSQLContext with SQLTestUtils {
766766
.option("header", "true")
767767
.load(iso8601timestampsPath)
768768

769-
val iso8501 = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSSZZ", Locale.US)
769+
val iso8501 = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.US)
770770
val expectedTimestamps = timestamps.collect().map { r =>
771771
// This should be ISO8601 formatted string.
772772
Row(iso8501.format(r.toSeq.head))

0 commit comments

Comments
 (0)