Skip to content

Commit 7d3a568

Browse files
committed
address comments
1 parent b530e5f commit 7d3a568

File tree

2 files changed

+5
-5
lines changed

2 files changed

+5
-5
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -176,16 +176,16 @@ case class ParquetPartitionReaderFactory(
176176
reader
177177
}
178178

179-
private def createRowBaseReader(file: PartitionedFile): ParquetRecordReader[UnsafeRow] = {
180-
buildReaderBase(file, createRowBaseReader0).asInstanceOf[ParquetRecordReader[UnsafeRow]]
179+
private def createRowBaseReader(file: PartitionedFile): RecordReader[Void, UnsafeRow] = {
180+
buildReaderBase(file, createRowBaseReader0)
181181
}
182182

183183
private def createRowBaseReader0(
184184
split: ParquetInputSplit,
185185
partitionValues: InternalRow,
186186
hadoopAttemptContext: TaskAttemptContextImpl,
187187
pushed: Option[FilterPredicate],
188-
convertTz: Option[TimeZone]): ParquetRecordReader[UnsafeRow] = {
188+
convertTz: Option[TimeZone]): RecordReader[Void, UnsafeRow] = {
189189
logDebug(s"Falling back to parquet-mr")
190190
val taskContext = Option(TaskContext.get())
191191
// ParquetRecordReader returns UnsafeRow

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetScanBuilder.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,9 @@ case class ParquetScanBuilder(
6363
this.filters
6464
}
6565

66-
// The actual filter push down happens in [[ParquetPartitionReaderFactory]].
66+
// Note: for Parquet, the actual filter push down happens in [[ParquetPartitionReaderFactory]].
6767
// It requires the Parquet physical schema to determine whether a filter is convertible.
68-
// So here we simply mark that all the filters are pushed down.
68+
// All filters that can be converted to Parquet are pushed down.
6969
override def pushedFilters(): Array[Filter] = _pushedFilters
7070

7171
override def build(): Scan = {

0 commit comments

Comments
 (0)