Update arrow 48.0.0 (#7854)

* Update arrow 48.0.0

* Fix for pyo3

* Update json.slt

* Update pin and fix clippy

* More clippy

Author: tustvold

Cargo.toml

@@ -48,12 +48,12 @@ rust-version = "1.70"
 version = "32.0.0"
 
 [workspace.dependencies]
-arrow = { version = "47.0.0", features = ["prettyprint"] }
-arrow-array = { version = "47.0.0", default-features = false, features = ["chrono-tz"] }
-arrow-buffer = { version = "47.0.0", default-features = false }
-arrow-flight = { version = "47.0.0", features = ["flight-sql-experimental"] }
-arrow-schema = { version = "47.0.0", default-features = false }
-parquet = { version = "47.0.0", features = ["arrow", "async", "object_store"] }
+arrow = { version = "48.0.0", features = ["prettyprint"] }
+arrow-array = { version = "48.0.0", default-features = false, features = ["chrono-tz"] }
+arrow-buffer = { version = "48.0.0", default-features = false }
+arrow-flight = { version = "48.0.0", features = ["flight-sql-experimental"] }
+arrow-schema = { version = "48.0.0", default-features = false }
+parquet = { version = "48.0.0", features = ["arrow", "async", "object_store"] }
 sqlparser = { version = "0.38.0", features = ["visitor"] }
 chrono = { version = "0.4.31", default-features = false }
 

datafusion-cli/Cargo.lock

@@ -29,7 +29,7 @@ rust-version = "1.70"
 readme = "README.md"
 
 [dependencies]
-arrow = "47.0.0"
+arrow = "48.0.0"
 async-trait = "0.1.41"
 aws-config = "0.55"
 aws-credential-types = "0.55"

datafusion-cli/Cargo.toml

@@ -59,7 +59,7 @@ fn print_batches_with_sep(batches: &[RecordBatch], delimiter: u8) -> Result<Stri
     let mut bytes = vec![];
     {
         let builder = WriterBuilder::new()
-            .has_headers(true)
+            .with_header(true)
             .with_delimiter(delimiter);
         let mut writer = builder.build(&mut bytes);
         for batch in batches {

datafusion-cli/src/print_format.rs

@@ -50,7 +50,7 @@ half = { version = "2.1", default-features = false }
 num_cpus = "1.13.0"
 object_store = { version = "0.7.0", default-features = false, optional = true }
 parquet = { workspace = true, optional = true }
-pyo3 = { version = "0.19.0", optional = true }
+pyo3 = { version = "0.20.0", optional = true }
 sqlparser = { workspace = true }
 
 [dev-dependencies]

datafusion/common/Cargo.toml

@@ -37,13 +37,6 @@ pub struct CsvWriterOptions {
     /// Compression to apply after ArrowWriter serializes RecordBatches.
     /// This compression is applied by DataFusion not the ArrowWriter itself.
     pub compression: CompressionTypeVariant,
-    /// Indicates whether WriterBuilder.has_header() is set to true.
-    /// This is duplicative as WriterBuilder also stores this information.
-    /// However, WriterBuilder does not allow public read access to the
-    /// has_header parameter.
-    pub has_header: bool,
-    // TODO: expose a way to read has_header in arrow create
-    // https://github.com/apache/arrow-rs/issues/4735
 }
 
 impl CsvWriterOptions {
@@ -54,7 +47,6 @@ impl CsvWriterOptions {
         Self {
             writer_options,
             compression,
-            has_header: true,
         }
     }
 }
@@ -65,29 +57,20 @@ impl TryFrom<(&ConfigOptions, &StatementOptions)> for CsvWriterOptions {
     fn try_from(value: (&ConfigOptions, &StatementOptions)) -> Result<Self> {
         let _configs = value.0;
         let statement_options = value.1;
-        let mut has_header = true;
         let mut builder = WriterBuilder::default();
         let mut compression = CompressionTypeVariant::UNCOMPRESSED;
         for (option, value) in &statement_options.options {
             builder = match option.to_lowercase().as_str(){
                 "header" => {
-                    has_header = value.parse()
+                    let has_header = value.parse()
                         .map_err(|_| DataFusionError::Configuration(format!("Unable to parse {value} as bool as required for {option}!")))?;
-                    builder.has_headers(has_header)
+                    builder.with_header(has_header)
                 },
                 "date_format" => builder.with_date_format(value.to_owned()),
                 "datetime_format" => builder.with_datetime_format(value.to_owned()),
                 "timestamp_format" => builder.with_timestamp_format(value.to_owned()),
                 "time_format" => builder.with_time_format(value.to_owned()),
-                "rfc3339" => {
-                    let value_bool = value.parse()
-                        .map_err(|_| DataFusionError::Configuration(format!("Unable to parse {value} as bool as required for {option}!")))?;
-                    if value_bool{
-                        builder.with_rfc3339()
-                    } else{
-                        builder
-                    }
-                },
+                "rfc3339" => builder, // No-op
                 "null_value" => builder.with_null(value.to_owned()),
                 "compression" => {
                     compression = CompressionTypeVariant::from_str(value.replace('\'', "").as_str())?;
@@ -112,7 +95,6 @@ impl TryFrom<(&ConfigOptions, &StatementOptions)> for CsvWriterOptions {
             }
         }
         Ok(CsvWriterOptions {
-            has_header,
             writer_options: builder,
             compression,
         })

datafusion/common/src/file_options/csv_writer.rs

@@ -523,9 +523,9 @@ mod tests {
 
         let csv_options = CsvWriterOptions::try_from((&config, &options))?;
         let builder = csv_options.writer_options;
+        assert!(builder.header());
         let buff = Vec::new();
         let _properties = builder.build(buff);
-        assert!(csv_options.has_header);
         assert_eq!(csv_options.compression, CompressionTypeVariant::GZIP);
         // TODO expand unit test if csv::WriterBuilder allows public read access to properties