Skip to content

Commit dae8678

Browse files
author
Brent Gardner
committed
Upgrade arrow
fix decimal (#4) Fix human error Patch crates io to fix build (#5) * fix decimal * patch crate versions Patch objectstore Test in CI Undo override? Fix more errors Fix last error? Formatting Clippy
1 parent 92e98df commit dae8678

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+442
-360
lines changed

‎benchmarks/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ simd = ["datafusion/simd"]
3232
snmalloc = ["snmalloc-rs"]
3333

3434
[dependencies]
35-
datafusion = { path = "../datafusion/core" }
35+
datafusion = { path = "../datafusion/core", features = [], optional = false }
3636
env_logger = "0.9"
3737
futures = "0.3"
3838
mimalloc = { version = "0.1", optional = true, default-features = false }

‎datafusion-cli/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@ rust-version = "1.59"
2929
readme = "README.md"
3030

3131
[dependencies]
32-
arrow = { version = "19.0.0" }
32+
arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "6bb4b5ee16488c2a6427a5897bb6fbe334cc280e", features = [], optional = false }
3333
clap = { version = "3", features = ["derive", "cargo"] }
34-
datafusion = { path = "../datafusion/core", version = "10.0.0" }
34+
datafusion = { path = "../datafusion/core", features = [], optional = false }
3535
dirs = "4.0.0"
3636
env_logger = "0.9"
3737
mimalloc = { version = "0.1", default-features = false }

‎datafusion-examples/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@ path = "examples/avro_sql.rs"
3434
required-features = ["datafusion/avro"]
3535

3636
[dev-dependencies]
37-
arrow-flight = { version = "19.0.0" }
37+
arrow-flight = { git = "https://github.com/apache/arrow-rs.git", rev = "6bb4b5ee16488c2a6427a5897bb6fbe334cc280e", features = [], optional = false }
3838
async-trait = "0.1.41"
39-
datafusion = { path = "../datafusion/core" }
39+
datafusion = { path = "../datafusion/core", features = [], optional = false }
4040
futures = "0.3"
4141
num_cpus = "1.13.0"
4242
prost = "0.10"

‎datafusion/common/Cargo.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,11 @@ jit = ["cranelift-module"]
3838
pyarrow = ["pyo3"]
3939

4040
[dependencies]
41-
arrow = { version = "19.0.0", features = ["prettyprint"] }
41+
arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "6bb4b5ee16488c2a6427a5897bb6fbe334cc280e", features = ["prettyprint"], optional = false }
4242
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
4343
cranelift-module = { version = "0.86.1", optional = true }
44-
object_store = { version = "0.3", optional = true }
44+
object_store = { git = "https://github.com/apache/arrow-rs.git", rev = "6bb4b5ee16488c2a6427a5897bb6fbe334cc280e", features = [], optional = true }
4545
ordered-float = "3.0"
46-
parquet = { version = "19.0.0", features = ["arrow"], optional = true }
46+
parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "6bb4b5ee16488c2a6427a5897bb6fbe334cc280e", features = ["arrow"], optional = true }
4747
pyo3 = { version = "0.16", optional = true }
4848
sqlparser = "0.19"

‎datafusion/common/src/scalar.rs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ use arrow::{
2727
IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, TimeUnit,
2828
TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
2929
TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type,
30-
DECIMAL_MAX_PRECISION,
30+
DECIMAL128_MAX_PRECISION,
3131
},
3232
util::decimal::{BasicDecimal, Decimal128},
3333
};
@@ -611,7 +611,7 @@ impl ScalarValue {
611611
scale: usize,
612612
) -> Result<Self> {
613613
// make sure the precision and scale is valid
614-
if precision <= DECIMAL_MAX_PRECISION && scale <= precision {
614+
if precision <= DECIMAL128_MAX_PRECISION && scale <= precision {
615615
return Ok(ScalarValue::Decimal128(Some(value), precision, scale));
616616
}
617617
Err(DataFusionError::Internal(format!(
@@ -654,7 +654,7 @@ impl ScalarValue {
654654
ScalarValue::Int32(_) => DataType::Int32,
655655
ScalarValue::Int64(_) => DataType::Int64,
656656
ScalarValue::Decimal128(_, precision, scale) => {
657-
DataType::Decimal(*precision, *scale)
657+
DataType::Decimal128(*precision, *scale)
658658
}
659659
ScalarValue::TimestampSecond(_, tz_opt) => {
660660
DataType::Timestamp(TimeUnit::Second, tz_opt.clone())
@@ -935,7 +935,7 @@ impl ScalarValue {
935935
}
936936

937937
let array: ArrayRef = match &data_type {
938-
DataType::Decimal(precision, scale) => {
938+
DataType::Decimal128(precision, scale) => {
939939
let decimal_array =
940940
ScalarValue::iter_to_decimal_array(scalars, precision, scale)?;
941941
Arc::new(decimal_array)
@@ -1448,7 +1448,7 @@ impl ScalarValue {
14481448

14491449
Ok(match array.data_type() {
14501450
DataType::Null => ScalarValue::Null,
1451-
DataType::Decimal(precision, scale) => {
1451+
DataType::Decimal128(precision, scale) => {
14521452
ScalarValue::get_decimal_value_from_array(array, index, precision, scale)
14531453
}
14541454
DataType::Boolean => typed_cast!(array, index, BooleanArray, Boolean),
@@ -1899,7 +1899,7 @@ impl TryFrom<&DataType> for ScalarValue {
18991899
DataType::UInt16 => ScalarValue::UInt16(None),
19001900
DataType::UInt32 => ScalarValue::UInt32(None),
19011901
DataType::UInt64 => ScalarValue::UInt64(None),
1902-
DataType::Decimal(precision, scale) => {
1902+
DataType::Decimal128(precision, scale) => {
19031903
ScalarValue::Decimal128(None, *precision, *scale)
19041904
}
19051905
DataType::Utf8 => ScalarValue::Utf8(None),
@@ -2145,7 +2145,7 @@ mod tests {
21452145
#[test]
21462146
fn scalar_decimal_test() {
21472147
let decimal_value = ScalarValue::Decimal128(Some(123), 10, 1);
2148-
assert_eq!(DataType::Decimal(10, 1), decimal_value.get_datatype());
2148+
assert_eq!(DataType::Decimal128(10, 1), decimal_value.get_datatype());
21492149
let try_into_value: i128 = decimal_value.clone().try_into().unwrap();
21502150
assert_eq!(123_i128, try_into_value);
21512151
assert!(!decimal_value.is_null());
@@ -2163,14 +2163,14 @@ mod tests {
21632163
let array = decimal_value.to_array();
21642164
let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
21652165
assert_eq!(1, array.len());
2166-
assert_eq!(DataType::Decimal(10, 1), array.data_type().clone());
2166+
assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone());
21672167
assert_eq!(123i128, array.value(0).as_i128());
21682168

21692169
// decimal scalar to array with size
21702170
let array = decimal_value.to_array_of_size(10);
21712171
let array_decimal = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
21722172
assert_eq!(10, array.len());
2173-
assert_eq!(DataType::Decimal(10, 1), array.data_type().clone());
2173+
assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone());
21742174
assert_eq!(123i128, array_decimal.value(0).as_i128());
21752175
assert_eq!(123i128, array_decimal.value(9).as_i128());
21762176
// test eq array
@@ -2208,7 +2208,7 @@ mod tests {
22082208
// convert the vec to decimal array and check the result
22092209
let array = ScalarValue::iter_to_array(decimal_vec.into_iter()).unwrap();
22102210
assert_eq!(3, array.len());
2211-
assert_eq!(DataType::Decimal(10, 2), array.data_type().clone());
2211+
assert_eq!(DataType::Decimal128(10, 2), array.data_type().clone());
22122212

22132213
let decimal_vec = vec![
22142214
ScalarValue::Decimal128(Some(1), 10, 2),
@@ -2218,7 +2218,7 @@ mod tests {
22182218
];
22192219
let array = ScalarValue::iter_to_array(decimal_vec.into_iter()).unwrap();
22202220
assert_eq!(4, array.len());
2221-
assert_eq!(DataType::Decimal(10, 2), array.data_type().clone());
2221+
assert_eq!(DataType::Decimal128(10, 2), array.data_type().clone());
22222222

22232223
assert!(ScalarValue::try_new_decimal128(1, 10, 2)
22242224
.unwrap()

‎datafusion/core/Cargo.toml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -55,18 +55,18 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion
5555

5656
[dependencies]
5757
ahash = { version = "0.7", default-features = false }
58-
arrow = { version = "19.0.0", features = ["prettyprint"] }
58+
arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "6bb4b5ee16488c2a6427a5897bb6fbe334cc280e", features = ["prettyprint"], optional = false }
5959
async-trait = "0.1.41"
6060
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
6161
bytes = "1.1"
6262
chrono = { version = "0.4", default-features = false }
63-
datafusion-common = { path = "../common", version = "10.0.0", features = ["parquet", "object_store"] }
64-
datafusion-expr = { path = "../expr", version = "10.0.0" }
65-
datafusion-jit = { path = "../jit", version = "10.0.0", optional = true }
66-
datafusion-optimizer = { path = "../optimizer", version = "10.0.0" }
67-
datafusion-physical-expr = { path = "../physical-expr", version = "10.0.0" }
68-
datafusion-row = { path = "../row", version = "10.0.0" }
69-
datafusion-sql = { path = "../sql", version = "10.0.0" }
63+
datafusion-common = { path = "../common", features = ["parquet", "object_store"], optional = false }
64+
datafusion-expr = { path = "../expr", features = [], optional = false }
65+
datafusion-jit = { path = "../jit", features = [], optional = true }
66+
datafusion-optimizer = { path = "../optimizer", features = [], optional = false }
67+
datafusion-physical-expr = { path = "../physical-expr", features = [], optional = false }
68+
datafusion-row = { path = "../row", features = [], optional = false }
69+
datafusion-sql = { path = "../sql", features = [], optional = false }
7070
futures = "0.3"
7171
glob = "0.3.0"
7272
hashbrown = { version = "0.12", features = ["raw"] }
@@ -75,10 +75,10 @@ lazy_static = { version = "^1.4.0" }
7575
log = "^0.4"
7676
num-traits = { version = "0.2", optional = true }
7777
num_cpus = "1.13.0"
78-
object_store = "0.3.0"
78+
object_store = { git = "https://github.com/apache/arrow-rs.git", rev = "6bb4b5ee16488c2a6427a5897bb6fbe334cc280e", features = [], optional = false }
7979
ordered-float = "3.0"
8080
parking_lot = "0.12"
81-
parquet = { version = "19.0.0", features = ["arrow", "async"] }
81+
parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "6bb4b5ee16488c2a6427a5897bb6fbe334cc280e", features = ["arrow", "async"], optional = false }
8282
paste = "^1.0"
8383
pin-project-lite = "^0.2.7"
8484
pyo3 = { version = "0.16", optional = true }
@@ -98,7 +98,7 @@ csv = "1.1.6"
9898
ctor = "0.1.22"
9999
doc-comment = "0.3"
100100
env_logger = "0.9"
101-
fuzz-utils = { path = "fuzz-utils" }
101+
fuzz-utils = { path = "fuzz-utils", features = [], optional = false }
102102

103103
[[bench]]
104104
harness = false

‎datafusion/core/fuzz-utils/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,6 @@ edition = "2021"
2323
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
2424

2525
[dependencies]
26-
arrow = { version = "19.0.0", features = ["prettyprint"] }
26+
arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "6bb4b5ee16488c2a6427a5897bb6fbe334cc280e", features = ["prettyprint"], optional = false }
2727
env_logger = "0.9.0"
2828
rand = "0.8"

‎datafusion/core/src/avro_to_arrow/arrow_array_reader.rs‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -101,12 +101,10 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
101101
"Failed to parse avro value: {:?}",
102102
e
103103
))),
104-
other => {
105-
return Err(ArrowError::ParseError(format!(
106-
"Row needs to be of type object, got: {:?}",
107-
other
108-
)))
109-
}
104+
other => Err(ArrowError::ParseError(format!(
105+
"Row needs to be of type object, got: {:?}",
106+
other
107+
))),
110108
})
111109
.collect::<ArrowResult<Vec<Vec<(String, Value)>>>>()?;
112110
if rows.is_empty() {

‎datafusion/core/src/avro_to_arrow/schema.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ fn schema_to_field_with_props(
140140
AvroSchema::Fixed { size, .. } => DataType::FixedSizeBinary(*size as i32),
141141
AvroSchema::Decimal {
142142
precision, scale, ..
143-
} => DataType::Decimal(*precision, *scale),
143+
} => DataType::Decimal128(*precision, *scale),
144144
AvroSchema::Uuid => DataType::FixedSizeBinary(16),
145145
AvroSchema::Date => DataType::Date32,
146146
AvroSchema::TimeMillis => DataType::Time32(TimeUnit::Millisecond),
@@ -216,7 +216,7 @@ fn default_field_name(dt: &DataType) -> &str {
216216
DataType::Union(_, _, _) => "union",
217217
DataType::Dictionary(_, _) => "map",
218218
DataType::Map(_, _) => unimplemented!("Map support not implemented"),
219-
DataType::Decimal(_, _) => "decimal",
219+
DataType::Decimal128(_, _) => "decimal",
220220
DataType::Decimal256(_, _) => "decimal",
221221
}
222222
}

‎datafusion/core/src/catalog/information_schema.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -508,7 +508,7 @@ impl InformationSchemaColumnsBuilder {
508508
Float32 => (Some(24), Some(2), None),
509509
// Numbers from postgres `double` type
510510
Float64 => (Some(24), Some(2), None),
511-
Decimal(precision, scale) => {
511+
Decimal128(precision, scale) => {
512512
(Some(*precision as u64), Some(10), Some(*scale as u64))
513513
}
514514
_ => (None, None, None),

0 commit comments

Comments
 (0)