Merge branch 'customizations/24.8.14' into backports/24.8/71539_enable_merge_filters_optimization

Author: Enmk

.github/create_combined_ci_report.py

@@ -54,7 +54,7 @@ def get_checks_known_fails(client: Client, job_url: str, known_fails: dict):
         len(df.columns) - 1,
         "reason",
         df["test_name"]
-        .cat.remove_unused_categories()
+        .astype(str)
         .apply(
             lambda test_name: known_fails[test_name].get("reason", "No reason given")
         ),

.github/workflows/release_branches.yml

@@ -539,7 +539,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression
-      commit: 53d73ed32155a8a17ee0d0cdb15aee96c98010a2
+      commit: 11dcb1ad771e6afebcb06bcc7bf1c1d8b184d838
       arch: release
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300
@@ -550,7 +550,7 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression
-      commit: 53d73ed32155a8a17ee0d0cdb15aee96c98010a2
+      commit: 11dcb1ad771e6afebcb06bcc7bf1c1d8b184d838
       arch: aarch64
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout_minutes: 300

src/Processors/Formats/Impl/Parquet/ParquetDataBuffer.h

@@ -48,6 +48,22 @@ class ParquetDataBuffer
         consume(bytes);
     }
 
+    template <typename TValue, typename ParquetType>
+    void ALWAYS_INLINE readValuesOfDifferentSize(TValue * dst, size_t count)
+    {
+        auto necessary_bytes = count * sizeof(ParquetType);
+        checkAvaible(necessary_bytes);
+
+        const ParquetType* src = reinterpret_cast<const ParquetType*>(data);
+
+        for (std::size_t i = 0; i < count; i++)
+        {
+            dst[i] = static_cast<TValue>(src[i]);
+        }
+
+        consume(necessary_bytes);
+    }
+
     void ALWAYS_INLINE readDateTime64FromInt96(DateTime64 & dst)
     {
         static const int max_scale_num = 9;

src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.cpp

@@ -240,8 +240,8 @@ TValue * getResizedPrimitiveData(TColumn & column, size_t size)
 } // anoynomous namespace
 
 
-template <>
-void ParquetPlainValuesReader<ColumnString>::readBatch(
+template <typename TColumn>
+void ParquetPlainByteArrayValuesReader<TColumn>::readBatch(
     MutableColumnPtr & col_ptr, LazyNullMap & null_map, UInt32 num_values)
 {
     auto & column = *assert_cast<ColumnString *>(col_ptr.get());
@@ -322,8 +322,8 @@ void ParquetBitPlainReader<TColumn>::readBatch(
 }
 
 
-template <>
-void ParquetPlainValuesReader<ColumnDecimal<DateTime64>, ParquetReaderTypes::TimestampInt96>::readBatch(
+template <typename TColumn>
+void ParquetPlainInt96ValuesReader<TColumn>::readBatch(
     MutableColumnPtr & col_ptr, LazyNullMap & null_map, UInt32 num_values)
 {
     auto cursor = col_ptr->size();
@@ -350,8 +350,8 @@ void ParquetPlainValuesReader<ColumnDecimal<DateTime64>, ParquetReaderTypes::Tim
     );
 }
 
-template <typename TColumn, ParquetReaderTypes reader_type>
-void ParquetPlainValuesReader<TColumn, reader_type>::readBatch(
+template <typename TColumn, typename ParquetType>
+void ParquetPlainValuesReader<TColumn, ParquetType>::readBatch(
     MutableColumnPtr & col_ptr, LazyNullMap & null_map, UInt32 num_values)
 {
     auto cursor = col_ptr->size();
@@ -365,11 +365,11 @@ void ParquetPlainValuesReader<TColumn, reader_type>::readBatch(
         null_map,
         /* individual_visitor */ [&](size_t nest_cursor)
         {
-            plain_data_buffer.readValue(column_data[nest_cursor]);
+            plain_data_buffer.readValuesOfDifferentSize<TValue, ParquetType>(column_data + nest_cursor, 1);
         },
         /* repeated_visitor */ [&](size_t nest_cursor, UInt32 count)
         {
-            plain_data_buffer.readBytes(column_data + nest_cursor, count * sizeof(TValue));
+            plain_data_buffer.readValuesOfDifferentSize<TValue, ParquetType>(column_data + nest_cursor, count);
         }
     );
 }
@@ -576,17 +576,19 @@ void ParquetRleDictReader<TColumnVector>::readBatch(
 }
 
 
-template class ParquetPlainValuesReader<ColumnInt32>;
-template class ParquetPlainValuesReader<ColumnUInt32>;
-template class ParquetPlainValuesReader<ColumnInt64>;
-template class ParquetPlainValuesReader<ColumnUInt64>;
-template class ParquetPlainValuesReader<ColumnFloat32>;
-template class ParquetPlainValuesReader<ColumnFloat64>;
-template class ParquetPlainValuesReader<ColumnDecimal<Decimal32>>;
-template class ParquetPlainValuesReader<ColumnDecimal<Decimal64>>;
-template class ParquetPlainValuesReader<ColumnDecimal<DateTime64>>;
-template class ParquetPlainValuesReader<ColumnString>;
-template class ParquetPlainValuesReader<ColumnUInt8>;
+template class ParquetPlainValuesReader<ColumnUInt8, int32_t>;
+template class ParquetPlainValuesReader<ColumnInt8, int32_t>;
+template class ParquetPlainValuesReader<ColumnUInt16, int32_t>;
+template class ParquetPlainValuesReader<ColumnInt16, int32_t>;
+template class ParquetPlainValuesReader<ColumnUInt32, int32_t>;
+template class ParquetPlainValuesReader<ColumnInt32, int32_t>;
+template class ParquetPlainValuesReader<ColumnUInt64, int64_t>;
+template class ParquetPlainValuesReader<ColumnInt64, int64_t>;
+template class ParquetPlainValuesReader<ColumnFloat32, float>;
+template class ParquetPlainValuesReader<ColumnFloat64, double>;
+template class ParquetPlainValuesReader<ColumnDecimal<Decimal32>, int32_t>;
+template class ParquetPlainValuesReader<ColumnDecimal<Decimal64>, int64_t>;
+template class ParquetPlainValuesReader<ColumnDecimal<DateTime64>, int64_t>;
 
 template class ParquetBitPlainReader<ColumnUInt8>;
 
@@ -597,7 +599,6 @@ template class ParquetRleLCReader<ColumnUInt8>;
 template class ParquetRleLCReader<ColumnUInt16>;
 template class ParquetRleLCReader<ColumnUInt32>;
 
-template class ParquetRleDictReader<ColumnUInt8>;
 template class ParquetRleDictReader<ColumnInt32>;
 template class ParquetRleDictReader<ColumnUInt32>;
 template class ParquetRleDictReader<ColumnInt64>;
@@ -611,4 +612,8 @@ template class ParquetRleDictReader<ColumnDecimal<Decimal256>>;
 template class ParquetRleDictReader<ColumnDecimal<DateTime64>>;
 template class ParquetRleDictReader<ColumnString>;
 
+template class ParquetPlainByteArrayValuesReader<ColumnString>;
+
+template class ParquetPlainInt96ValuesReader<ColumnDecimal<DateTime64>>;
+
 }

src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.h

@@ -150,7 +150,7 @@ enum class ParquetReaderTypes
 /**
  * The definition level is RLE or BitPacked encoding, while data is read directly
  */
-template <typename TColumn, ParquetReaderTypes reader_type = ParquetReaderTypes::Normal>
+template <typename TColumn, typename ParquetType>
 class ParquetPlainValuesReader : public ParquetDataValuesReader
 {
 public:
@@ -172,6 +172,50 @@ class ParquetPlainValuesReader : public ParquetDataValuesReader
     ParquetDataBuffer plain_data_buffer;
 };
 
+template <typename TColumn>
+class ParquetPlainInt96ValuesReader : public ParquetDataValuesReader
+{
+public:
+
+    ParquetPlainInt96ValuesReader(
+        Int32 max_def_level_,
+        std::unique_ptr<RleValuesReader> def_level_reader_,
+        ParquetDataBuffer data_buffer_)
+        : max_def_level(max_def_level_)
+        , def_level_reader(std::move(def_level_reader_))
+        , plain_data_buffer(std::move(data_buffer_))
+    {}
+
+    void readBatch(MutableColumnPtr & col_ptr, LazyNullMap & null_map, UInt32 num_values) override;
+
+private:
+    Int32 max_def_level;
+    std::unique_ptr<RleValuesReader> def_level_reader;
+    ParquetDataBuffer plain_data_buffer;
+};
+
+template <typename TColumn>
+class ParquetPlainByteArrayValuesReader : public ParquetDataValuesReader
+{
+public:
+
+    ParquetPlainByteArrayValuesReader(
+        Int32 max_def_level_,
+        std::unique_ptr<RleValuesReader> def_level_reader_,
+        ParquetDataBuffer data_buffer_)
+        : max_def_level(max_def_level_)
+        , def_level_reader(std::move(def_level_reader_))
+        , plain_data_buffer(std::move(data_buffer_))
+    {}
+
+    void readBatch(MutableColumnPtr & col_ptr, LazyNullMap & null_map, UInt32 num_values) override;
+
+private:
+    Int32 max_def_level;
+    std::unique_ptr<RleValuesReader> def_level_reader;
+    ParquetDataBuffer plain_data_buffer;
+};
+
 template <typename TColumn>
 class ParquetBitPlainReader : public ParquetDataValuesReader
 {