Skip to content

Commit cb6f7fe

Browse files
compheadcomphead
authored
Extend backtrace coverage for DatafusionError::Plan errors errors (#7803)
* improve backtrace coverag for plan errors * fix cli * cli fmt * fix tests * docs * doc fmt
1 parent fa2bb6c commit cb6f7fe

File tree

29 files changed

+236
-218
lines changed

29 files changed

+236
-218
lines changed

‎datafusion-cli/src/exec.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use crate::{
2626
},
2727
print_options::{MaxRows, PrintOptions},
2828
};
29+
use datafusion::common::plan_datafusion_err;
2930
use datafusion::sql::{parser::DFParser, sqlparser::dialect::dialect_from_str};
3031
use datafusion::{
3132
datasource::listing::ListingTableUrl,
@@ -202,11 +203,11 @@ async fn exec_and_print(
202203
let task_ctx = ctx.task_ctx();
203204
let dialect = &task_ctx.session_config().options().sql_parser.dialect;
204205
let dialect = dialect_from_str(dialect).ok_or_else(|| {
205-
DataFusionError::Plan(format!(
206+
plan_datafusion_err!(
206207
"Unsupported SQL dialect: {dialect}. Available dialects: \
207208
Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, \
208209
MsSQL, ClickHouse, BigQuery, Ansi."
209-
))
210+
)
210211
})?;
211212
let statements = DFParser::parse_sql_with_dialect(&sql, dialect.as_ref())?;
212213
for statement in statements {

‎datafusion/common/src/dfschema.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ use std::fmt::{Display, Formatter};
2424
use std::hash::Hash;
2525
use std::sync::Arc;
2626

27-
use crate::error::{unqualified_field_not_found, DataFusionError, Result, SchemaError};
27+
use crate::error::{
28+
unqualified_field_not_found, DataFusionError, Result, SchemaError, _plan_err,
29+
};
2830
use crate::{
2931
field_not_found, Column, FunctionalDependencies, OwnedTableReference, TableReference,
3032
};
@@ -187,10 +189,10 @@ impl DFSchema {
187189
match &self.fields[i].qualifier {
188190
Some(qualifier) => {
189191
if (qualifier.to_string() + "." + self.fields[i].name()) == name {
190-
return Err(DataFusionError::Plan(format!(
192+
return _plan_err!(
191193
"Fully qualified field name '{name}' was supplied to `index_of` \
192194
which is deprecated. Please use `index_of_column_by_name` instead"
193-
)));
195+
);
194196
}
195197
}
196198
None => (),
@@ -378,12 +380,11 @@ impl DFSchema {
378380
.zip(arrow_schema.fields().iter())
379381
.try_for_each(|(l_field, r_field)| {
380382
if !can_cast_types(r_field.data_type(), l_field.data_type()) {
381-
Err(DataFusionError::Plan(
382-
format!("Column {} (type: {}) is not compatible with column {} (type: {})",
383+
_plan_err!("Column {} (type: {}) is not compatible with column {} (type: {})",
383384
r_field.name(),
384385
r_field.data_type(),
385386
l_field.name(),
386-
l_field.data_type())))
387+
l_field.data_type())
387388
} else {
388389
Ok(())
389390
}

‎datafusion/common/src/error.rs‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -477,12 +477,25 @@ macro_rules! with_dollar_sign {
477477
/// plan_err!("Error {:?}", val)
478478
/// plan_err!("Error {val}")
479479
/// plan_err!("Error {val:?}")
480+
///
481+
/// `NAME_ERR` - macro name for wrapping Err(DataFusionError::*)
482+
/// `NAME_DF_ERR` - macro name for wrapping DataFusionError::*. Needed to keep backtrace opportunity
483+
/// in construction where DataFusionError::* used directly, like `map_err`, `ok_or_else`, etc
480484
macro_rules! make_error {
481-
($NAME:ident, $ERR:ident) => {
485+
($NAME_ERR:ident, $NAME_DF_ERR: ident, $ERR:ident) => {
482486
with_dollar_sign! {
483487
($d:tt) => {
488+
/// Macro wraps `$ERR` to add backtrace feature
489+
#[macro_export]
490+
macro_rules! $NAME_DF_ERR {
491+
($d($d args:expr),*) => {
492+
DataFusionError::$ERR(format!("{}{}", format!($d($d args),*), DataFusionError::get_back_trace()).into())
493+
}
494+
}
495+
496+
/// Macro wraps Err(`$ERR`) to add backtrace feature
484497
#[macro_export]
485-
macro_rules! $NAME {
498+
macro_rules! $NAME_ERR {
486499
($d($d args:expr),*) => {
487500
Err(DataFusionError::$ERR(format!("{}{}", format!($d($d args),*), DataFusionError::get_back_trace()).into()))
488501
}
@@ -493,16 +506,16 @@ macro_rules! make_error {
493506
}
494507

495508
// Exposes a macro to create `DataFusionError::Plan`
496-
make_error!(plan_err, Plan);
509+
make_error!(plan_err, plan_datafusion_err, Plan);
497510

498511
// Exposes a macro to create `DataFusionError::Internal`
499-
make_error!(internal_err, Internal);
512+
make_error!(internal_err, internal_datafusion_err, Internal);
500513

501514
// Exposes a macro to create `DataFusionError::NotImplemented`
502-
make_error!(not_impl_err, NotImplemented);
515+
make_error!(not_impl_err, not_impl_datafusion_err, NotImplemented);
503516

504517
// Exposes a macro to create `DataFusionError::Execution`
505-
make_error!(exec_err, Execution);
518+
make_error!(exec_err, exec_datafusion_err, Execution);
506519

507520
// Exposes a macro to create `DataFusionError::SQL`
508521
#[macro_export]
@@ -517,6 +530,7 @@ macro_rules! sql_err {
517530
pub use exec_err as _exec_err;
518531
pub use internal_err as _internal_err;
519532
pub use not_impl_err as _not_impl_err;
533+
pub use plan_err as _plan_err;
520534

521535
#[cfg(test)]
522536
mod test {

‎datafusion/common/src/functional_dependencies.rs‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use std::fmt::{Display, Formatter};
2323
use std::ops::Deref;
2424
use std::vec::IntoIter;
2525

26+
use crate::error::_plan_err;
2627
use crate::{DFSchema, DFSchemaRef, DataFusionError, JoinType, Result};
2728

2829
use sqlparser::ast::TableConstraint;
@@ -95,18 +96,18 @@ impl Constraints {
9596
Constraint::Unique(indices)
9697
})
9798
}
98-
TableConstraint::ForeignKey { .. } => Err(DataFusionError::Plan(
99-
"Foreign key constraints are not currently supported".to_string(),
100-
)),
101-
TableConstraint::Check { .. } => Err(DataFusionError::Plan(
102-
"Check constraints are not currently supported".to_string(),
103-
)),
104-
TableConstraint::Index { .. } => Err(DataFusionError::Plan(
105-
"Indexes are not currently supported".to_string(),
106-
)),
107-
TableConstraint::FulltextOrSpatial { .. } => Err(DataFusionError::Plan(
108-
"Indexes are not currently supported".to_string(),
109-
)),
99+
TableConstraint::ForeignKey { .. } => {
100+
_plan_err!("Foreign key constraints are not currently supported")
101+
}
102+
TableConstraint::Check { .. } => {
103+
_plan_err!("Check constraints are not currently supported")
104+
}
105+
TableConstraint::Index { .. } => {
106+
_plan_err!("Indexes are not currently supported")
107+
}
108+
TableConstraint::FulltextOrSpatial { .. } => {
109+
_plan_err!("Indexes are not currently supported")
110+
}
110111
})
111112
.collect::<Result<Vec<_>>>()?;
112113
Ok(Constraints::new_unverified(constraints))

‎datafusion/core/src/datasource/listing/table.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -233,10 +233,10 @@ impl FromStr for ListingTableInsertMode {
233233
"append_to_file" => Ok(ListingTableInsertMode::AppendToFile),
234234
"append_new_files" => Ok(ListingTableInsertMode::AppendNewFiles),
235235
"error" => Ok(ListingTableInsertMode::Error),
236-
_ => Err(DataFusionError::Plan(format!(
236+
_ => plan_err!(
237237
"Unknown or unsupported insert mode {s}. Supported options are \
238238
append_to_file, append_new_files, and error."
239-
))),
239+
),
240240
}
241241
}
242242
}
@@ -865,10 +865,10 @@ impl TableProvider for ListingTable {
865865
let writer_mode = match self.options.insert_mode {
866866
ListingTableInsertMode::AppendToFile => {
867867
if input_partitions > file_groups.len() {
868-
return Err(DataFusionError::Plan(format!(
868+
return plan_err!(
869869
"Cannot append {input_partitions} partitions to {} files!",
870870
file_groups.len()
871-
)));
871+
);
872872
}
873873

874874
crate::datasource::file_format::write::FileWriterMode::Append
@@ -919,9 +919,9 @@ impl TableProvider for ListingTable {
919919
self.options().insert_mode,
920920
ListingTableInsertMode::AppendToFile
921921
) {
922-
return Err(DataFusionError::Plan(
923-
"Cannot insert into a sorted ListingTable with mode append!".into(),
924-
));
922+
return plan_err!(
923+
"Cannot insert into a sorted ListingTable with mode append!"
924+
);
925925
}
926926
// Multiple sort orders in outer vec are equivalent, so we pass only the first one
927927
let ordering = self

‎datafusion/core/src/execution/context.rs‎

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ use crate::{
3030
};
3131
use datafusion_common::{
3232
alias::AliasGenerator,
33-
exec_err, not_impl_err, plan_err,
33+
exec_err, not_impl_err, plan_datafusion_err, plan_err,
3434
tree_node::{TreeNode, TreeNodeVisitor, VisitRecursion},
3535
};
3636
use datafusion_execution::registry::SerializerRegistry;
@@ -1577,17 +1577,14 @@ impl SessionState {
15771577
self.catalog_list
15781578
.catalog(&resolved_ref.catalog)
15791579
.ok_or_else(|| {
1580-
DataFusionError::Plan(format!(
1580+
plan_datafusion_err!(
15811581
"failed to resolve catalog: {}",
15821582
resolved_ref.catalog
1583-
))
1583+
)
15841584
})?
15851585
.schema(&resolved_ref.schema)
15861586
.ok_or_else(|| {
1587-
DataFusionError::Plan(format!(
1588-
"failed to resolve schema: {}",
1589-
resolved_ref.schema
1590-
))
1587+
plan_datafusion_err!("failed to resolve schema: {}", resolved_ref.schema)
15911588
})
15921589
}
15931590

@@ -1689,11 +1686,11 @@ impl SessionState {
16891686
dialect: &str,
16901687
) -> Result<datafusion_sql::parser::Statement> {
16911688
let dialect = dialect_from_str(dialect).ok_or_else(|| {
1692-
DataFusionError::Plan(format!(
1689+
plan_datafusion_err!(
16931690
"Unsupported SQL dialect: {dialect}. Available dialects: \
16941691
Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, \
16951692
MsSQL, ClickHouse, BigQuery, Ansi."
1696-
))
1693+
)
16971694
})?;
16981695
let mut statements = DFParser::parse_sql_with_dialect(sql, dialect.as_ref())?;
16991696
if statements.len() > 1 {
@@ -2022,7 +2019,7 @@ impl<'a> ContextProvider for SessionContextProvider<'a> {
20222019
self.tables
20232020
.get(&name)
20242021
.cloned()
2025-
.ok_or_else(|| DataFusionError::Plan(format!("table '{name}' not found")))
2022+
.ok_or_else(|| plan_datafusion_err!("table '{name}' not found"))
20262023
}
20272024

20282025
fn get_function_meta(&self, name: &str) -> Option<Arc<ScalarUDF>> {
@@ -2069,29 +2066,23 @@ impl FunctionRegistry for SessionState {
20692066
let result = self.scalar_functions.get(name);
20702067

20712068
result.cloned().ok_or_else(|| {
2072-
DataFusionError::Plan(format!(
2073-
"There is no UDF named \"{name}\" in the registry"
2074-
))
2069+
plan_datafusion_err!("There is no UDF named \"{name}\" in the registry")
20752070
})
20762071
}
20772072

20782073
fn udaf(&self, name: &str) -> Result<Arc<AggregateUDF>> {
20792074
let result = self.aggregate_functions.get(name);
20802075

20812076
result.cloned().ok_or_else(|| {
2082-
DataFusionError::Plan(format!(
2083-
"There is no UDAF named \"{name}\" in the registry"
2084-
))
2077+
plan_datafusion_err!("There is no UDAF named \"{name}\" in the registry")
20852078
})
20862079
}
20872080

20882081
fn udwf(&self, name: &str) -> Result<Arc<WindowUDF>> {
20892082
let result = self.window_functions.get(name);
20902083

20912084
result.cloned().ok_or_else(|| {
2092-
DataFusionError::Plan(format!(
2093-
"There is no UDWF named \"{name}\" in the registry"
2094-
))
2085+
plan_datafusion_err!("There is no UDWF named \"{name}\" in the registry")
20952086
})
20962087
}
20972088
}

‎datafusion/core/src/physical_optimizer/pruning.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ use arrow::{
3535
datatypes::{DataType, Field, Schema, SchemaRef},
3636
record_batch::RecordBatch,
3737
};
38-
use datafusion_common::{downcast_value, ScalarValue};
38+
use datafusion_common::{downcast_value, plan_datafusion_err, ScalarValue};
3939
use datafusion_common::{
4040
internal_err, plan_err,
4141
tree_node::{Transformed, TreeNode},
@@ -451,7 +451,7 @@ fn build_statistics_record_batch<S: PruningStatistics>(
451451
);
452452

453453
RecordBatch::try_new_with_options(schema, arrays, &options).map_err(|err| {
454-
DataFusionError::Plan(format!("Can not create statistics record batch: {err}"))
454+
plan_datafusion_err!("Can not create statistics record batch: {err}")
455455
})
456456
}
457457

‎datafusion/core/src/physical_optimizer/sort_pushdown.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ use crate::physical_plan::sorts::sort::SortExec;
2929
use crate::physical_plan::{with_new_children_if_necessary, ExecutionPlan};
3030

3131
use datafusion_common::tree_node::{Transformed, TreeNode, VisitRecursion};
32-
use datafusion_common::{plan_err, DataFusionError, Result};
32+
use datafusion_common::{plan_datafusion_err, plan_err, DataFusionError, Result};
3333
use datafusion_expr::JoinType;
3434
use datafusion_physical_expr::expressions::Column;
3535
use datafusion_physical_expr::utils::{
@@ -127,7 +127,7 @@ pub(crate) fn pushdown_sorts(
127127
let plan = &requirements.plan;
128128
let parent_required = requirements.required_ordering.as_deref();
129129
const ERR_MSG: &str = "Expects parent requirement to contain something";
130-
let err = || DataFusionError::Plan(ERR_MSG.to_string());
130+
let err = || plan_datafusion_err!("{}", ERR_MSG);
131131
if let Some(sort_exec) = plan.as_any().downcast_ref::<SortExec>() {
132132
let mut new_plan = plan.clone();
133133
if !ordering_satisfy_requirement(
@@ -199,7 +199,7 @@ fn pushdown_requirement_to_children(
199199
parent_required: Option<&[PhysicalSortRequirement]>,
200200
) -> Result<Option<Vec<Option<Vec<PhysicalSortRequirement>>>>> {
201201
const ERR_MSG: &str = "Expects parent requirement to contain something";
202-
let err = || DataFusionError::Plan(ERR_MSG.to_string());
202+
let err = || plan_datafusion_err!("{}", ERR_MSG);
203203
let maintains_input_order = plan.maintains_input_order();
204204
if is_window(plan) {
205205
let required_input_ordering = plan.required_input_ordering();

‎datafusion/execution/src/task.rs‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use std::{
2222

2323
use datafusion_common::{
2424
config::{ConfigOptions, Extensions},
25-
DataFusionError, Result,
25+
plan_datafusion_err, DataFusionError, Result,
2626
};
2727
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
2828

@@ -182,19 +182,15 @@ impl FunctionRegistry for TaskContext {
182182
let result = self.scalar_functions.get(name);
183183

184184
result.cloned().ok_or_else(|| {
185-
DataFusionError::Plan(format!(
186-
"There is no UDF named \"{name}\" in the TaskContext"
187-
))
185+
plan_datafusion_err!("There is no UDF named \"{name}\" in the TaskContext")
188186
})
189187
}
190188

191189
fn udaf(&self, name: &str) -> Result<Arc<AggregateUDF>> {
192190
let result = self.aggregate_functions.get(name);
193191

194192
result.cloned().ok_or_else(|| {
195-
DataFusionError::Plan(format!(
196-
"There is no UDAF named \"{name}\" in the TaskContext"
197-
))
193+
plan_datafusion_err!("There is no UDAF named \"{name}\" in the TaskContext")
198194
})
199195
}
200196

‎datafusion/expr/src/aggregate_function.rs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
use crate::utils;
2121
use crate::{type_coercion::aggregates::*, Signature, TypeSignature, Volatility};
2222
use arrow::datatypes::{DataType, Field};
23-
use datafusion_common::{plan_err, DataFusionError, Result};
23+
use datafusion_common::{plan_datafusion_err, plan_err, DataFusionError, Result};
2424
use std::sync::Arc;
2525
use std::{fmt, str::FromStr};
2626
use strum_macros::EnumIter;
@@ -232,11 +232,14 @@ impl AggregateFunction {
232232
// original errors are all related to wrong function signature
233233
// aggregate them for better error message
234234
.map_err(|_| {
235-
DataFusionError::Plan(utils::generate_signature_error_msg(
236-
&format!("{self}"),
237-
self.signature(),
238-
input_expr_types,
239-
))
235+
plan_datafusion_err!(
236+
"{}",
237+
utils::generate_signature_error_msg(
238+
&format!("{self}"),
239+
self.signature(),
240+
input_expr_types,
241+
)
242+
)
240243
})?;
241244

242245
match self {

0 commit comments

Comments
 (0)