diff --git a/datafusion/catalog-listing/src/helpers.rs b/datafusion/catalog-listing/src/helpers.rs index 00e9c71df348..bfafac931df8 100644 --- a/datafusion/catalog-listing/src/helpers.rs +++ b/datafusion/catalog-listing/src/helpers.rs @@ -63,7 +63,7 @@ pub fn expr_applicable_for_cols(col_names: &[&str], expr: &Expr) -> bool { } Expr::Literal(_, _) | Expr::Alias(_) - | Expr::OuterReferenceColumn(_, _) + | Expr::OuterReferenceColumn(_) | Expr::ScalarVariable(_, _) | Expr::Not(_) | Expr::IsNotNull(_) diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs index a19e6f558162..c0a784df148b 100644 --- a/datafusion/core/src/dataframe/mod.rs +++ b/datafusion/core/src/dataframe/mod.rs @@ -2291,7 +2291,7 @@ impl DataFrame { if cols.contains(field) { // Try to cast fill value to column type. If the cast fails, fallback to the original column. match value.clone().cast_to(field.data_type()) { - Ok(fill_value) => Expr::Alias(Alias { + Ok(fill_value) => Expr::Alias(Box::new(Alias { expr: Box::new(Expr::ScalarFunction(ScalarFunction { func: coalesce(), args: vec![col(field.name()), lit(fill_value)], @@ -2299,7 +2299,7 @@ impl DataFrame { relation: None, name: field.name().to_string(), metadata: None, - }), + })), Err(_) => col(field.name()), } } else { diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 1021abc9e47b..8fd92a3fd5e6 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -72,7 +72,7 @@ use datafusion_datasource::file_groups::FileGroup; use datafusion_datasource::memory::MemorySourceConfig; use datafusion_expr::dml::{CopyTo, InsertOp}; use datafusion_expr::expr::{ - physical_name, AggregateFunction, AggregateFunctionParams, Alias, GroupingSet, + physical_name, AggregateFunction, AggregateFunctionParams, GroupingSet, WindowFunction, WindowFunctionParams, }; use datafusion_expr::expr_rewriter::unnormalize_cols; @@ -602,9 +602,9 @@ impl DefaultPhysicalPlanner { } = &window_fun.as_ref().params; generate_sort_key(partition_by, order_by) } - Expr::Alias(Alias { expr, .. }) => { + Expr::Alias(boxed_alias) => { // Convert &Box to &T - match &**expr { + match boxed_alias.expr.as_ref() { Expr::WindowFunction(window_fun) => { let WindowFunctionParams { ref partition_by, @@ -1693,7 +1693,7 @@ pub fn create_window_expr( ) -> Result> { // unpack aliased logical expressions, e.g. "sum(col) over () as total" let (name, e) = match e { - Expr::Alias(Alias { expr, name, .. }) => (name.clone(), expr.as_ref()), + Expr::Alias(boxed_alias) => (boxed_alias.name.clone(), boxed_alias.expr.as_ref()), _ => (e.schema_name().to_string(), e), }; create_window_expr_with_name(e, name, logical_schema, execution_props) @@ -1784,9 +1784,11 @@ pub fn create_aggregate_expr_and_maybe_filter( ) -> Result { // unpack (nested) aliased logical expressions, e.g. "sum(col) as total" let (name, human_display, e) = match e { - Expr::Alias(Alias { expr, name, .. }) => { - (Some(name.clone()), String::default(), expr.as_ref()) - } + Expr::Alias(boxed_alias) => ( + Some(boxed_alias.name.clone()), + String::default(), + boxed_alias.expr.as_ref(), + ), Expr::AggregateFunction(_) => ( Some(e.schema_name().to_string()), e.human_display().to_string(), diff --git a/datafusion/core/tests/dataframe/dataframe_functions.rs b/datafusion/core/tests/dataframe/dataframe_functions.rs index 40590d74ad91..ee167fafa805 100644 --- a/datafusion/core/tests/dataframe/dataframe_functions.rs +++ b/datafusion/core/tests/dataframe/dataframe_functions.rs @@ -415,11 +415,11 @@ async fn test_fn_approx_percentile_cont() -> Result<()> { "); // the arg2 parameter is a complex expr, but it can be evaluated to the literal value - let alias_expr = Expr::Alias(Alias::new( + let alias_expr = Expr::Alias(Box::new(Alias::new( cast(lit(0.5), DataType::Float32), None::<&str>, "arg_2".to_string(), - )); + ))); let expr = approx_percentile_cont(col("b").sort(true, false), alias_expr, None); let df = create_test_table().await?; let batches = df.aggregate(vec![], vec![expr]).unwrap().collect().await?; @@ -435,11 +435,11 @@ async fn test_fn_approx_percentile_cont() -> Result<()> { " ); - let alias_expr = Expr::Alias(Alias::new( + let alias_expr = Expr::Alias(Box::new(Alias::new( cast(lit(0.1), DataType::Float32), None::<&str>, "arg_2".to_string(), - )); + ))); let expr = approx_percentile_cont(col("b").sort(false, false), alias_expr, None); let df = create_test_table().await?; let batches = df.aggregate(vec![], vec![expr]).unwrap().collect().await?; diff --git a/datafusion/core/tests/user_defined/expr_planner.rs b/datafusion/core/tests/user_defined/expr_planner.rs index 07d289cab06c..8f65be00b5e6 100644 --- a/datafusion/core/tests/user_defined/expr_planner.rs +++ b/datafusion/core/tests/user_defined/expr_planner.rs @@ -55,11 +55,11 @@ impl ExprPlanner for MyCustomPlanner { }))) } BinaryOperator::Question => { - Ok(PlannerResult::Planned(Expr::Alias(Alias::new( + Ok(PlannerResult::Planned(Expr::Alias(Box::new(Alias::new( Expr::Literal(ScalarValue::Boolean(Some(true)), None), None::<&str>, format!("{} ? {}", expr.left, expr.right), - )))) + ))))) } _ => Ok(PlannerResult::Original(expr)), } diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index efe8a639087a..57f9aa3f0bde 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -279,7 +279,7 @@ use sqlparser::ast::{ #[derive(Clone, PartialEq, PartialOrd, Eq, Debug, Hash)] pub enum Expr { /// An expression with a specific name. - Alias(Alias), + Alias(Box), /// A named reference to a qualified field in a schema. Column(Column), /// A named reference to a variable in a registry. @@ -362,7 +362,7 @@ pub enum Expr { Placeholder(Placeholder), /// A placeholder which holds a reference to a qualified field /// in the outer query, used for correlated sub queries. - OuterReferenceColumn(DataType, Column), + OuterReferenceColumn(Box<(DataType, Column)>), /// Unnest expression Unnest(Unnest), } @@ -1424,7 +1424,9 @@ impl Expr { name, spans: _, }) => (relation.clone(), name.clone()), - Expr::Alias(Alias { relation, name, .. }) => (relation.clone(), name.clone()), + Expr::Alias(boxed_alias) => { + (boxed_alias.relation.clone(), boxed_alias.name.clone()) + } _ => (None, self.schema_name().to_string()), } } @@ -1446,7 +1448,7 @@ impl Expr { Expr::Case { .. } => "Case", Expr::Cast { .. } => "Cast", Expr::Column(..) => "Column", - Expr::OuterReferenceColumn(_, _) => "Outer", + Expr::OuterReferenceColumn(_) => "Outer", Expr::Exists { .. } => "Exists", Expr::GroupingSet(..) => "GroupingSet", Expr::InList { .. } => "InList", @@ -1572,7 +1574,7 @@ impl Expr { /// Return `self AS name` alias expression pub fn alias(self, name: impl Into) -> Expr { - Expr::Alias(Alias::new(self, None::<&str>, name.into())) + Expr::Alias(Box::new(Alias::new(self, None::<&str>, name.into()))) } /// Return `self AS name` alias expression with metadata @@ -1595,7 +1597,9 @@ impl Expr { name: impl Into, metadata: Option, ) -> Expr { - Expr::Alias(Alias::new(self, None::<&str>, name.into()).with_metadata(metadata)) + Expr::Alias(Box::new( + Alias::new(self, None::<&str>, name.into()).with_metadata(metadata), + )) } /// Return `self AS name` alias expression with a specific qualifier @@ -1604,7 +1608,7 @@ impl Expr { relation: Option>, name: impl Into, ) -> Expr { - Expr::Alias(Alias::new(self, relation, name.into())) + Expr::Alias(Box::new(Alias::new(self, relation, name.into()))) } /// Return `self AS name` alias expression with a specific qualifier and metadata @@ -1628,7 +1632,9 @@ impl Expr { name: impl Into, metadata: Option, ) -> Expr { - Expr::Alias(Alias::new(self, relation, name.into()).with_metadata(metadata)) + Expr::Alias(Box::new( + Alias::new(self, relation, name.into()).with_metadata(metadata), + )) } /// Remove an alias from an expression if one exists. @@ -2021,7 +2027,7 @@ impl Expr { | Expr::SimilarTo(..) | Expr::Not(..) | Expr::Negative(..) - | Expr::OuterReferenceColumn(_, _) + | Expr::OuterReferenceColumn(_) | Expr::TryCast(..) | Expr::Unnest(..) | Expr::Wildcard { .. } @@ -2109,23 +2115,10 @@ impl NormalizeEq for Expr { && self_right.normalize_eq(other_right) } } - ( - Expr::Alias(Alias { - expr: self_expr, - relation: self_relation, - name: self_name, - .. - }), - Expr::Alias(Alias { - expr: other_expr, - relation: other_relation, - name: other_name, - .. - }), - ) => { - self_name == other_name - && self_relation == other_relation - && self_expr.normalize_eq(other_expr) + (Expr::Alias(boxed_alias), Expr::Alias(boxed_other_alias)) => { + boxed_alias.name == boxed_other_alias.name + && boxed_alias.relation == boxed_other_alias.relation + && boxed_alias.expr.normalize_eq(&*boxed_other_alias.expr) } ( Expr::Like(Like { @@ -2459,14 +2452,9 @@ impl HashNode for Expr { fn hash_node(&self, state: &mut H) { mem::discriminant(self).hash(state); match self { - Expr::Alias(Alias { - expr: _expr, - relation, - name, - .. - }) => { - relation.hash(state); - name.hash(state); + Expr::Alias(boxed_alias) => { + boxed_alias.relation.hash(state); + boxed_alias.name.hash(state); } Expr::Column(column) => { column.hash(state); @@ -2609,7 +2597,8 @@ impl HashNode for Expr { Expr::Placeholder(place_holder) => { place_holder.hash(state); } - Expr::OuterReferenceColumn(data_type, column) => { + Expr::OuterReferenceColumn(boxed_orc) => { + let (data_type, column) = boxed_orc.as_ref(); data_type.hash(state); column.hash(state); } @@ -2673,12 +2662,14 @@ impl Display for SchemaDisplay<'_> { } } // Expr is not shown since it is aliased - Expr::Alias(Alias { - name, - relation: Some(relation), - .. - }) => write!(f, "{relation}.{name}"), - Expr::Alias(Alias { name, .. }) => write!(f, "{name}"), + Expr::Alias(boxed_alias) => { + let alias = boxed_alias.as_ref(); + if let Some(relation) = &alias.relation { + write!(f, "{relation}.{}", alias.name) + } else { + write!(f, "{}", alias.name) + } + } Expr::Between(Between { expr, negated, @@ -2929,7 +2920,10 @@ impl Display for SqlDisplay<'_> { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self.0 { Expr::Literal(scalar, _) => scalar.fmt(f), - Expr::Alias(Alias { name, .. }) => write!(f, "{name}"), + Expr::Alias(boxed_alias) => { + let name = &boxed_alias.as_ref().name; + write!(f, "{name}") + } Expr::Between(Between { expr, negated, @@ -3189,9 +3183,13 @@ pub const UNNEST_COLUMN_PREFIX: &str = "UNNEST"; impl Display for Expr { fn fmt(&self, f: &mut Formatter) -> fmt::Result { match self { - Expr::Alias(Alias { expr, name, .. }) => write!(f, "{expr} AS {name}"), + Expr::Alias(boxed_alias) => { + let Alias { expr, name, .. } = boxed_alias.as_ref(); + write!(f, "{expr} AS {name}") + } Expr::Column(c) => write!(f, "{c}"), - Expr::OuterReferenceColumn(_, c) => { + Expr::OuterReferenceColumn(boxed_orc) => { + let (_, c) = boxed_orc.as_ref(); write!(f, "{OUTER_REFERENCE_COLUMN_PREFIX}({c})") } Expr::ScalarVariable(_, var_names) => write!(f, "{}", var_names.join(".")), @@ -3839,7 +3837,7 @@ mod test { // If this test fails when you change `Expr`, please try // `Box`ing the fields to make `Expr` smaller // See https://github.com/apache/datafusion/issues/16199 for details - assert_eq!(size_of::(), 128); + assert_eq!(size_of::(), 112); assert_eq!(size_of::(), 64); assert_eq!(size_of::(), 24); // 3 ptrs assert_eq!(size_of::>(), 24); diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs index a511d082c0ff..48c39058dcb2 100644 --- a/datafusion/expr/src/expr_fn.rs +++ b/datafusion/expr/src/expr_fn.rs @@ -71,7 +71,7 @@ pub fn col(ident: impl Into) -> Expr { /// Create an out reference column which hold a reference that has been resolved to a field /// outside of the current plan. pub fn out_ref_col(dt: DataType, ident: impl Into) -> Expr { - Expr::OuterReferenceColumn(dt, ident.into()) + Expr::OuterReferenceColumn(Box::new((dt, ident.into()))) } /// Create an unqualified column expression from the provided name, without normalizing diff --git a/datafusion/expr/src/expr_rewriter/mod.rs b/datafusion/expr/src/expr_rewriter/mod.rs index 05a9425452a1..11361273716a 100644 --- a/datafusion/expr/src/expr_rewriter/mod.rs +++ b/datafusion/expr/src/expr_rewriter/mod.rs @@ -174,10 +174,14 @@ pub fn create_col_from_scalar_expr( subqry_alias: String, ) -> Result { match scalar_expr { - Expr::Alias(Alias { name, .. }) => Ok(Column::new( - Some::(subqry_alias.into()), - name, - )), + Expr::Alias(boxed_alias) => { + // boxed_alias: &Box (if you’re matching on &expr) or Box + let name = &boxed_alias.name; + Ok(Column::new( + Some::(subqry_alias.into()), + name, + )) + } Expr::Column(col) => Ok(col.with_relation(subqry_alias.into())), _ => { let scalar_column = scalar_expr.schema_name().to_string(); @@ -200,7 +204,9 @@ pub fn unnormalize_cols(exprs: impl IntoIterator) -> Vec { pub fn strip_outer_reference(expr: Expr) -> Expr { expr.transform(|expr| { Ok({ - if let Expr::OuterReferenceColumn(_, col) = expr { + // Match the boxed (DataType, Column) tuple and extract the Column + if let Expr::OuterReferenceColumn(boxed_pair) = expr { + let (_data_type, col) = *boxed_pair; Transformed::yes(Expr::Column(col)) } else { Transformed::no(expr) @@ -250,7 +256,9 @@ fn coerce_exprs_for_schema( let new_type = dst_schema.field(idx).data_type(); if new_type != &expr.get_type(src_schema)? { match expr { - Expr::Alias(Alias { expr, name, .. }) => { + Expr::Alias(boxed_alias) => { + // boxed_alias: Box + let Alias { expr, name, .. } = *boxed_alias; Ok(expr.cast_to(new_type, src_schema)?.alias(name)) } #[expect(deprecated)] @@ -264,12 +272,13 @@ fn coerce_exprs_for_schema( .collect::>() } -/// Recursively un-alias an expressions +/// Recursively un-alias an expression #[inline] pub fn unalias(expr: Expr) -> Expr { match expr { - Expr::Alias(Alias { expr, .. }) => unalias(*expr), - _ => expr, + // Unbox the Alias, then recurse on the inner Expr + Expr::Alias(boxed_alias) => unalias(*boxed_alias.expr), + other => other, } } diff --git a/datafusion/expr/src/expr_rewriter/order_by.rs b/datafusion/expr/src/expr_rewriter/order_by.rs index 6db95555502d..46412a5dfa26 100644 --- a/datafusion/expr/src/expr_rewriter/order_by.rs +++ b/datafusion/expr/src/expr_rewriter/order_by.rs @@ -137,8 +137,11 @@ fn rewrite_in_terms_of_projection( /// so avg(c) as average will match avgc fn expr_match(needle: &Expr, expr: &Expr) -> bool { // check inside aliases - if let Expr::Alias(Alias { expr, .. }) = &expr { - expr.as_ref() == needle + if let Expr::Alias(boxed_alias) = expr { + // boxed_alias: &Box, so boxed_alias.as_ref() is &Alias + let alias: &Alias = boxed_alias.as_ref(); + // alias.expr: Box, so alias.expr.as_ref() is &Expr + alias.expr.as_ref() == needle } else { expr == needle } diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rs index 8ca479bb6f9b..e9575ea1d5ad 100644 --- a/datafusion/expr/src/expr_schema.rs +++ b/datafusion/expr/src/expr_schema.rs @@ -103,16 +103,18 @@ impl ExprSchemable for Expr { #[cfg_attr(feature = "recursive_protection", recursive::recursive)] fn get_type(&self, schema: &dyn ExprSchema) -> Result { match self { - Expr::Alias(Alias { expr, name, .. }) => match &**expr { + Expr::Alias(boxed_alias) => match boxed_alias.expr.as_ref() { Expr::Placeholder(Placeholder { data_type, .. }) => match &data_type { - None => schema.data_type(&Column::from_name(name)).cloned(), + None => schema + .data_type(&Column::from_name(&boxed_alias.name)) + .cloned(), Some(dt) => Ok(dt.clone()), }, - _ => expr.get_type(schema), + _ => boxed_alias.expr.get_type(schema), }, Expr::Negative(expr) => expr.get_type(schema), Expr::Column(c) => Ok(schema.data_type(c)?.clone()), - Expr::OuterReferenceColumn(ty, _) => Ok(ty.clone()), + Expr::OuterReferenceColumn(boxed_orc) => Ok(boxed_orc.as_ref().0.clone()), Expr::ScalarVariable(ty, _) => Ok(ty.clone()), Expr::Literal(l, _) => Ok(l.data_type()), Expr::Case(case) => { @@ -242,9 +244,11 @@ impl ExprSchemable for Expr { /// column that does not exist in the schema. fn nullable(&self, input_schema: &dyn ExprSchema) -> Result { match self { - Expr::Alias(Alias { expr, .. }) | Expr::Not(expr) | Expr::Negative(expr) => { + Expr::Alias(boxed_alias) => { + let Alias { expr, .. } = boxed_alias.as_ref(); expr.nullable(input_schema) } + Expr::Not(expr) | Expr::Negative(expr) => expr.nullable(input_schema), Expr::InList(InList { expr, list, .. }) => { // Avoid inspecting too many expressions. @@ -276,7 +280,7 @@ impl ExprSchemable for Expr { || high.nullable(input_schema)?), Expr::Column(c) => input_schema.nullable(c), - Expr::OuterReferenceColumn(_, _) => Ok(true), + Expr::OuterReferenceColumn(_) => Ok(true), Expr::Literal(value, _) => Ok(value.is_null()), Expr::Case(case) => { // This expression is nullable if any of the input expressions are nullable @@ -380,30 +384,30 @@ impl ExprSchemable for Expr { let (relation, schema_name) = self.qualified_name(); #[allow(deprecated)] let field = match self { - Expr::Alias(Alias { - expr, - name, - metadata, - .. - }) => { - let field = match &**expr { + Expr::Alias(boxed_alias) => { + let field = match &*boxed_alias.expr { Expr::Placeholder(Placeholder { data_type, .. }) => { match &data_type { None => schema - .data_type_and_nullable(&Column::from_name(name)) + .data_type_and_nullable(&Column::from_name( + &boxed_alias.name, + )) .map(|(d, n)| Field::new(&schema_name, d.clone(), n)), Some(dt) => Ok(Field::new( &schema_name, dt.clone(), - expr.nullable(schema)?, + boxed_alias.expr.nullable(schema)?, )), } } - _ => expr.to_field(schema).map(|(_, f)| f.as_ref().clone()), + _ => boxed_alias + .expr + .to_field(schema) + .map(|(_, f)| f.as_ref().clone()), }?; - let mut combined_metadata = expr.metadata(schema)?; - if let Some(metadata) = metadata { + let mut combined_metadata = boxed_alias.expr.metadata(schema)?; + if let Some(metadata) = &boxed_alias.metadata { combined_metadata.extend(metadata.clone()); } @@ -411,9 +415,11 @@ impl ExprSchemable for Expr { } Expr::Negative(expr) => expr.to_field(schema).map(|(_, f)| f), Expr::Column(c) => schema.field_from_column(c).map(|f| Arc::new(f.clone())), - Expr::OuterReferenceColumn(ty, _) => { - Ok(Arc::new(Field::new(&schema_name, ty.clone(), true))) - } + Expr::OuterReferenceColumn(boxed_alias) => Ok(Arc::new(Field::new( + &schema_name, + boxed_alias.as_ref().0.clone(), + true, + ))), Expr::ScalarVariable(ty, _) => { Ok(Arc::new(Field::new(&schema_name, ty.clone(), true))) } diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index 1ab5ffa75842..9624b4c3a8da 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -24,7 +24,7 @@ use std::iter::once; use std::sync::Arc; use crate::dml::CopyTo; -use crate::expr::{Alias, PlannedReplaceSelectItem, Sort as SortExpr}; +use crate::expr::{PlannedReplaceSelectItem, Sort as SortExpr}; use crate::expr_rewriter::{ coerce_plan_expr_for_schema, normalize_col, normalize_col_with_schemas_and_ambiguity_check, normalize_cols, normalize_sorts, @@ -742,8 +742,8 @@ impl LogicalPlanBuilder { // As described in https://github.com/apache/datafusion/issues/5293 let all_aliases = missing_exprs.iter().all(|e| { projection_exprs.iter().any(|proj_expr| { - if let Expr::Alias(Alias { expr, .. }) = proj_expr { - e == expr.as_ref() + if let Expr::Alias(boxed_alias) = proj_expr { + e == boxed_alias.expr.as_ref() } else { false } diff --git a/datafusion/expr/src/tree_node.rs b/datafusion/expr/src/tree_node.rs index b6f583ca4c74..778da37a997d 100644 --- a/datafusion/expr/src/tree_node.rs +++ b/datafusion/expr/src/tree_node.rs @@ -44,8 +44,10 @@ impl TreeNode for Expr { f: F, ) -> Result { match self { - Expr::Alias(Alias { expr, .. }) - | Expr::Unnest(Unnest { expr }) + Expr::Alias(boxed_alias) => { + boxed_alias.expr.apply_elements(f) + } + Expr::Unnest(Unnest { expr }) | Expr::Not(expr) | Expr::IsNotNull(expr) | Expr::IsTrue(expr) @@ -71,7 +73,7 @@ impl TreeNode for Expr { #[expect(deprecated)] Expr::Column(_) // Treat OuterReferenceColumn as a leaf expression - | Expr::OuterReferenceColumn(_, _) + | Expr::OuterReferenceColumn(_) | Expr::ScalarVariable(_, _) | Expr::Literal(_, _) | Expr::Exists { .. } @@ -122,7 +124,7 @@ impl TreeNode for Expr { Expr::Column(_) | Expr::Wildcard { .. } | Expr::Placeholder(Placeholder { .. }) - | Expr::OuterReferenceColumn(_, _) + | Expr::OuterReferenceColumn(_) | Expr::Exists { .. } | Expr::ScalarSubquery(_) | Expr::ScalarVariable(_, _) @@ -130,14 +132,19 @@ impl TreeNode for Expr { Expr::Unnest(Unnest { expr, .. }) => expr .map_elements(f)? .update_data(|expr| Expr::Unnest(Unnest { expr })), - Expr::Alias(Alias { - expr, - relation, - name, - metadata, - }) => f(*expr)?.update_data(|e| { - e.alias_qualified_with_metadata(relation, name, metadata) - }), + Expr::Alias(boxed_alias) => { + let Alias { + expr, + relation, + name, + metadata, + .. + } = *boxed_alias; + + f(*expr)?.update_data(|e| { + e.alias_qualified_with_metadata(relation, name, metadata) + }) + } Expr::InSubquery(InSubquery { expr, subquery, diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs index e55415232897..bdafea62f442 100644 --- a/datafusion/expr/src/utils.rs +++ b/datafusion/expr/src/utils.rs @@ -22,7 +22,7 @@ use std::collections::{BTreeSet, HashSet}; use std::hash::Hasher; use std::sync::Arc; -use crate::expr::{Alias, Sort, WildcardOptions, WindowFunctionParams}; +use crate::expr::{Sort, WildcardOptions, WindowFunctionParams}; use crate::expr_rewriter::strip_outer_reference; use crate::{ and, BinaryExpr, Expr, ExprSchemable, Filter, GroupingSet, LogicalPlan, Operator, @@ -945,7 +945,9 @@ fn split_conjunction_impl<'a>(expr: &'a Expr, mut exprs: Vec<&'a Expr>) -> Vec<& let exprs = split_conjunction_impl(left, exprs); split_conjunction_impl(right, exprs) } - Expr::Alias(Alias { expr, .. }) => split_conjunction_impl(expr, exprs), + Expr::Alias(boxed_alias) => { + split_conjunction_impl(boxed_alias.expr.as_ref(), exprs) + } other => { exprs.push(other); exprs @@ -969,7 +971,9 @@ pub fn iter_conjunction(expr: &Expr) -> impl Iterator { stack.push(right); stack.push(left); } - Expr::Alias(Alias { expr, .. }) => stack.push(expr), + Expr::Alias(boxed_alias) => { + stack.push(&*boxed_alias.expr); + } other => return Some(other), } } @@ -993,7 +997,7 @@ pub fn iter_conjunction_owned(expr: Expr) -> impl Iterator { stack.push(*right); stack.push(*left); } - Expr::Alias(Alias { expr, .. }) => stack.push(*expr), + Expr::Alias(boxed_alias) => stack.push(*boxed_alias.expr), other => return Some(other), } } @@ -1062,8 +1066,8 @@ fn split_binary_owned_impl( let exprs = split_binary_owned_impl(*left, operator, exprs); split_binary_owned_impl(*right, operator, exprs) } - Expr::Alias(Alias { expr, .. }) => { - split_binary_owned_impl(*expr, operator, exprs) + Expr::Alias(boxed_alias) => { + split_binary_owned_impl(*boxed_alias.expr, operator, exprs) } other => { exprs.push(other); @@ -1089,7 +1093,9 @@ fn split_binary_impl<'a>( let exprs = split_binary_impl(left, operator, exprs); split_binary_impl(right, operator, exprs) } - Expr::Alias(Alias { expr, .. }) => split_binary_impl(expr, operator, exprs), + Expr::Alias(boxed_alias) => { + split_binary_impl(boxed_alias.expr.as_ref(), operator, exprs) + } other => { exprs.push(other); exprs diff --git a/datafusion/optimizer/src/analyzer/resolve_grouping_function.rs b/datafusion/optimizer/src/analyzer/resolve_grouping_function.rs index fa7ff1b8b19d..34926bb5cf51 100644 --- a/datafusion/optimizer/src/analyzer/resolve_grouping_function.rs +++ b/datafusion/optimizer/src/analyzer/resolve_grouping_function.rs @@ -103,11 +103,11 @@ fn replace_grouping_exprs( &group_expr_to_bitmap_index, is_grouping_set, )?; - projection_exprs.push(Expr::Alias(Alias::new( + projection_exprs.push(Expr::Alias(Box::new(Alias::new( grouping_expr, column.relation, column.name, - ))); + )))); } _ => { projection_exprs.push(Expr::Column(column)); diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index e6fc006cb2ff..35a2508aa823 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -33,8 +33,8 @@ use datafusion_common::{ DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, TableReference, }; use datafusion_expr::expr::{ - self, AggregateFunctionParams, Alias, Between, BinaryExpr, Case, Exists, InList, - InSubquery, Like, ScalarFunction, Sort, WindowFunction, + self, AggregateFunctionParams, Between, BinaryExpr, Case, Exists, InList, InSubquery, + Like, ScalarFunction, Sort, WindowFunction, }; use datafusion_expr::expr_rewriter::coerce_plan_expr_for_schema; use datafusion_expr::expr_schema::cast_subquery; @@ -602,7 +602,7 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> { | Expr::Wildcard { .. } | Expr::GroupingSet(_) | Expr::Placeholder(_) - | Expr::OuterReferenceColumn(_, _) => Ok(Transformed::no(expr)), + | Expr::OuterReferenceColumn(_) => Ok(Transformed::no(expr)), } } } @@ -1047,7 +1047,9 @@ fn project_with_column_index( .into_iter() .enumerate() .map(|(i, e)| match e { - Expr::Alias(Alias { ref name, .. }) if name != schema.field(i).name() => { + Expr::Alias(ref boxed_alias) + if &boxed_alias.name != schema.field(i).name() => + { Ok(e.unalias().alias(schema.field(i).name())) } Expr::Column(Column { diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs index 88d51e1adea3..e3cf14bfed87 100644 --- a/datafusion/optimizer/src/common_subexpr_eliminate.rs +++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs @@ -30,7 +30,7 @@ use datafusion_common::alias::AliasGenerator; use datafusion_common::cse::{CSEController, FoundCommonNodes, CSE}; use datafusion_common::tree_node::{Transformed, TreeNode}; use datafusion_common::{qualified_name, Column, DFSchema, DFSchemaRef, Result}; -use datafusion_expr::expr::{Alias, ScalarFunction}; +use datafusion_expr::expr::ScalarFunction; use datafusion_expr::logical_plan::{ Aggregate, Filter, LogicalPlan, Projection, Sort, Window, }; @@ -330,12 +330,12 @@ impl CommonSubexprEliminate { rewritten_aggr_expr.into_iter().zip(new_aggr_expr) { if expr_rewritten == expr_orig { - if let Expr::Alias(Alias { expr, name, .. }) = - expr_rewritten - { - agg_exprs.push(expr.alias(&name)); - proj_exprs - .push(Expr::Column(Column::from_name(name))); + if let Expr::Alias(boxed_alias) = expr_rewritten { + agg_exprs + .push(boxed_alias.expr.alias(&boxed_alias.name)); + proj_exprs.push(Expr::Column(Column::from_name( + boxed_alias.name, + ))); } else { let expr_alias = config.alias_generator().next(CSE_PREFIX); diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs index 63236787743a..52d1e1c7344b 100644 --- a/datafusion/optimizer/src/decorrelate.rs +++ b/datafusion/optimizer/src/decorrelate.rs @@ -27,7 +27,6 @@ use datafusion_common::tree_node::{ Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter, }; use datafusion_common::{plan_err, Column, DFSchemaRef, HashMap, Result, ScalarValue}; -use datafusion_expr::expr::Alias; use datafusion_expr::simplify::SimplifyContext; use datafusion_expr::utils::{ collect_subquery_cols, conjunction, find_join_exprs, split_conjunction, @@ -548,7 +547,7 @@ fn proj_exprs_evaluation_result_on_empty_batch( let simplifier = ExprSimplifier::new(info); let result_expr = simplifier.simplify(result_expr)?; let expr_name = match expr { - Expr::Alias(Alias { name, .. }) => name.to_string(), + Expr::Alias(boxed_alias) => boxed_alias.name.to_string(), Expr::Column(Column { relation: _, name, diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index 7b7be82b70ca..2f7ac426b352 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -509,16 +509,22 @@ fn merge_consecutive_projections(proj: Projection) -> Result rewrite_expr(*expr, &prev_projection).map(|result| { - result.update_data(|expr| { - Expr::Alias(Alias::new(expr, relation, name).with_metadata(metadata)) + Expr::Alias(boxed_alias) => { + let Alias { + expr: inner_expr, + relation, + name, + metadata, + } = *boxed_alias; + + rewrite_expr(*inner_expr, &prev_projection).map(|result| { + result.update_data(|expr| { + Expr::Alias(Box::new( + Alias::new(expr, relation, name).with_metadata(metadata), + )) + }) }) - }), + } e => rewrite_expr(e, &prev_projection), } })?; @@ -637,8 +643,8 @@ fn outer_columns<'a>(expr: &'a Expr, columns: &mut HashSet<&'a Column>) { // inspect_expr_pre doesn't handle subquery references, so find them explicitly expr.apply(|expr| { match expr { - Expr::OuterReferenceColumn(_, col) => { - columns.insert(col); + Expr::OuterReferenceColumn(boxed_orc) => { + columns.insert(&boxed_orc.as_ref().1); } Expr::ScalarSubquery(subquery) => { outer_columns_helper_multi(&subquery.outer_ref_columns, columns); diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 35ec7d074d5f..ef483d575933 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -263,7 +263,7 @@ fn can_evaluate_as_join_condition(predicate: &Expr) -> Result { Expr::Exists { .. } | Expr::InSubquery(_) | Expr::ScalarSubquery(_) - | Expr::OuterReferenceColumn(_, _) + | Expr::OuterReferenceColumn(_) | Expr::Unnest(_) => { is_evaluate = false; Ok(TreeNodeRecursion::Stop) diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 9a3a8bcd23a7..bc50d798080c 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -628,7 +628,7 @@ impl<'a> ConstEvaluator<'a> { Expr::AggregateFunction { .. } | Expr::ScalarVariable(_, _) | Expr::Column(_) - | Expr::OuterReferenceColumn(_, _) + | Expr::OuterReferenceColumn(_) | Expr::Exists { .. } | Expr::InSubquery(_) | Expr::ScalarSubquery(_) diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs index fbc19b1202ee..08a29b6f4380 100644 --- a/datafusion/physical-expr/src/planner.rs +++ b/datafusion/physical-expr/src/planner.rs @@ -28,9 +28,7 @@ use datafusion_common::{ exec_err, not_impl_err, plan_err, DFSchema, Result, ScalarValue, ToDFSchema, }; use datafusion_expr::execution_props::ExecutionProps; -use datafusion_expr::expr::{ - Alias, Cast, FieldMetadata, InList, Placeholder, ScalarFunction, -}; +use datafusion_expr::expr::{Cast, FieldMetadata, InList, Placeholder, ScalarFunction}; use datafusion_expr::var_provider::is_system_variables; use datafusion_expr::var_provider::VarType; use datafusion_expr::{ @@ -113,18 +111,22 @@ pub fn create_physical_expr( let input_schema: &Schema = &input_dfschema.into(); match e { - Expr::Alias(Alias { expr, metadata, .. }) => { - if let Expr::Literal(v, prior_metadata) = expr.as_ref() { + Expr::Alias(boxed_alias) => { + if let Expr::Literal(v, prior_metadata) = boxed_alias.expr.as_ref() { let new_metadata = FieldMetadata::merge_options( prior_metadata.as_ref(), - metadata.as_ref(), + boxed_alias.metadata.as_ref(), ); Ok(Arc::new(Literal::new_with_metadata( v.clone(), new_metadata, ))) } else { - Ok(create_physical_expr(expr, input_dfschema, execution_props)?) + Ok(create_physical_expr( + boxed_alias.expr.as_ref(), + input_dfschema, + execution_props, + )?) } } Expr::Column(c) => { diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index 6c5b348698c7..a7a672082c67 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -343,7 +343,7 @@ pub fn parse_expr( } } } - ExprType::Alias(alias) => Ok(Expr::Alias(Alias::new( + ExprType::Alias(alias) => Ok(Expr::Alias(Box::new(Alias::new( parse_required_expr(alias.expr.as_deref(), registry, "expr", codec)?, alias .relation @@ -351,7 +351,7 @@ pub fn parse_expr( .map(|r| TableReference::try_from(r.clone())) .transpose()?, alias.alias.clone(), - ))), + )))), ExprType::IsNullExpr(is_null) => Ok(Expr::IsNull(Box::new(parse_required_expr( is_null.expr.as_deref(), registry, diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index f59e97df0d46..1ea1edc247f1 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -24,8 +24,8 @@ use std::collections::HashMap; use datafusion_common::{NullEquality, TableReference, UnnestOptions}; use datafusion_expr::dml::InsertOp; use datafusion_expr::expr::{ - self, AggregateFunctionParams, Alias, Between, BinaryExpr, Cast, GroupingSet, InList, - Like, Placeholder, ScalarFunction, Unnest, + self, AggregateFunctionParams, Between, BinaryExpr, Cast, GroupingSet, InList, Like, + Placeholder, ScalarFunction, Unnest, }; use datafusion_expr::WriteOp; use datafusion_expr::{ @@ -198,20 +198,17 @@ pub fn serialize_expr( Expr::Column(c) => protobuf::LogicalExprNode { expr_type: Some(ExprType::Column(c.into())), }, - Expr::Alias(Alias { - expr, - relation, - name, - metadata, - }) => { + Expr::Alias(boxed_alias) => { let alias = Box::new(protobuf::AliasNode { - expr: Some(Box::new(serialize_expr(expr.as_ref(), codec)?)), - relation: relation + expr: Some(Box::new(serialize_expr(boxed_alias.expr.as_ref(), codec)?)), + relation: boxed_alias + .relation .to_owned() .map(|r| vec![r.into()]) .unwrap_or(vec![]), - alias: name.to_owned(), - metadata: metadata + alias: boxed_alias.name.to_owned(), + metadata: boxed_alias + .metadata .as_ref() .map(|m| m.to_hashmap()) .unwrap_or(HashMap::new()), diff --git a/datafusion/sql/src/expr/identifier.rs b/datafusion/sql/src/expr/identifier.rs index 434ac50bce50..d9b77ae72878 100644 --- a/datafusion/sql/src/expr/identifier.rs +++ b/datafusion/sql/src/expr/identifier.rs @@ -74,10 +74,10 @@ impl SqlToRel<'_, S> { outer.qualified_field_with_unqualified_name(normalize_ident.as_str()) { // Found an exact match on a qualified name in the outer plan schema, so this is an outer reference column - return Ok(Expr::OuterReferenceColumn( + return Ok(Expr::OuterReferenceColumn(Box::new(( field.data_type().clone(), Column::from((qualifier, field)), - )); + )))); } } @@ -181,10 +181,10 @@ impl SqlToRel<'_, S> { // Found matching field with no spare identifier(s) Some((field, qualifier, _nested_names)) => { // Found an exact match on a qualified name in the outer plan schema, so this is an outer reference column - Ok(Expr::OuterReferenceColumn( + Ok(Expr::OuterReferenceColumn(Box::new(( field.data_type().clone(), Column::from((qualifier, field)), - )) + )))) } // Found no matching field, will return a default None => { diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index b50fbf68129c..12ce2bd5e1ac 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -31,7 +31,7 @@ use datafusion_common::error::DataFusionErrorBuilder; use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; use datafusion_common::{not_impl_err, plan_err, Result}; use datafusion_common::{RecursionUnnestOption, UnnestOptions}; -use datafusion_expr::expr::{Alias, PlannedReplaceSelectItem, WildcardOptions}; +use datafusion_expr::expr::{PlannedReplaceSelectItem, WildcardOptions}; use datafusion_expr::expr_rewriter::{ normalize_col, normalize_col_with_schemas_and_ambiguity_check, normalize_sorts, }; @@ -189,8 +189,8 @@ impl SqlToRel<'_, S> { .iter() .filter(|select_expr| match select_expr { Expr::AggregateFunction(_) => false, - Expr::Alias(Alias { expr, name: _, .. }) => { - !matches!(**expr, Expr::AggregateFunction(_)) + Expr::Alias(boxed_alias) => { + !matches!(boxed_alias.expr.as_ref(), Expr::AggregateFunction(_)) } _ => true, }) diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index 4c0dc316615c..84df8b9a1b57 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -42,7 +42,7 @@ use datafusion_common::{ ScalarValue, }; use datafusion_expr::{ - expr::{Alias, Exists, InList, ScalarFunction, Sort, WindowFunction}, + expr::{Exists, InList, ScalarFunction, Sort, WindowFunction}, Between, BinaryExpr, Case, Cast, Expr, GroupingSet, Like, Operator, TryCast, }; use sqlparser::ast::helpers::attached_token::AttachedToken; @@ -189,7 +189,7 @@ impl Unparser<'_> { Ok(self.cast_to_sql(expr, data_type)?) } Expr::Literal(value, _) => Ok(self.scalar_to_sql(value)?), - Expr::Alias(Alias { expr, name: _, .. }) => self.expr_to_sql_inner(expr), + Expr::Alias(boxed_alias) => self.expr_to_sql_inner(boxed_alias.expr.as_ref()), Expr::WindowFunction(window_fun) => { let WindowFunction { fun, @@ -515,7 +515,9 @@ impl Unparser<'_> { Expr::Placeholder(p) => { Ok(ast::Expr::value(ast::Value::Placeholder(p.id.to_string()))) } - Expr::OuterReferenceColumn(_, col) => self.col_to_sql(col), + Expr::OuterReferenceColumn(boxed_orc) => { + self.col_to_sql(&boxed_orc.as_ref().1) + } Expr::Unnest(unnest) => self.unnest_to_sql(unnest), } } diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index 4fb1e42d6028..c91fdb37be24 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -45,7 +45,7 @@ use datafusion_common::{ }; use datafusion_expr::expr::OUTER_REFERENCE_COLUMN_PREFIX; use datafusion_expr::{ - expr::Alias, BinaryExpr, Distinct, Expr, JoinConstraint, JoinType, LogicalPlan, + BinaryExpr, Distinct, Expr, JoinConstraint, JoinType, LogicalPlan, LogicalPlanBuilder, Operator, Projection, SortExpr, TableScan, Unnest, UserDefinedLogicalNode, }; @@ -1002,8 +1002,8 @@ impl Unparser<'_> { /// /// `outer_ref` is the display result of [Expr::OuterReferenceColumn] fn check_unnest_placeholder_with_outer_ref(expr: &Expr) -> Option { - if let Expr::Alias(Alias { expr, .. }) = expr { - if let Expr::Column(Column { name, .. }) = expr.as_ref() { + if let Expr::Alias(boxed_alias) = expr { + if let Expr::Column(Column { name, .. }) = boxed_alias.expr.as_ref() { if let Some(prefix) = name.strip_prefix(UNNEST_PLACEHOLDER) { if prefix.starts_with(&format!("({OUTER_REFERENCE_COLUMN_PREFIX}(")) { return Some(UnnestInputType::OuterReference); @@ -1210,16 +1210,16 @@ impl Unparser<'_> { fn select_item_to_sql(&self, expr: &Expr) -> Result { match expr { - Expr::Alias(Alias { expr, name, .. }) => { + Expr::Alias(boxed_alias) => { let inner = self.expr_to_sql(expr)?; // Determine the alias name to use let col_name = if let Some(rewritten_name) = - self.dialect.col_alias_overrides(name)? + self.dialect.col_alias_overrides(&boxed_alias.name)? { rewritten_name.to_string() } else { - name.to_string() + boxed_alias.name.to_string() }; Ok(ast::SelectItem::ExprWithAlias { diff --git a/datafusion/sql/src/unparser/rewrite.rs b/datafusion/sql/src/unparser/rewrite.rs index aa480cf4fff9..f9b317a64ecf 100644 --- a/datafusion/sql/src/unparser/rewrite.rs +++ b/datafusion/sql/src/unparser/rewrite.rs @@ -359,12 +359,12 @@ pub(super) fn inject_column_aliases( _ => None, }; - Expr::Alias(Alias { + Expr::Alias(Box::new(Alias { expr: Box::new(expr.clone()), relation, name: col_alias.value, metadata: None, - }) + })) }) .collect::>(); diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs index 52832e1324be..a48abd37a10c 100644 --- a/datafusion/sql/src/utils.rs +++ b/datafusion/sql/src/utils.rs @@ -30,9 +30,7 @@ use datafusion_common::{ HashMap, Result, ScalarValue, }; use datafusion_expr::builder::get_struct_unnested_columns; -use datafusion_expr::expr::{ - Alias, GroupingSet, Unnest, WindowFunction, WindowFunctionParams, -}; +use datafusion_expr::expr::{GroupingSet, Unnest, WindowFunction, WindowFunctionParams}; use datafusion_expr::utils::{expr_as_column_expr, find_column_exprs}; use datafusion_expr::{ col, expr_vec_fmt, ColumnUnnestList, Expr, ExprSchemable, LogicalPlan, @@ -181,7 +179,9 @@ pub(crate) fn extract_aliases(exprs: &[Expr]) -> HashMap { exprs .iter() .filter_map(|expr| match expr { - Expr::Alias(Alias { expr, name, .. }) => Some((name.clone(), *expr.clone())), + Expr::Alias(boxed_alias) => { + Some((boxed_alias.name.clone(), *boxed_alias.expr.clone())) + } _ => None, }) .collect::>() @@ -204,7 +204,7 @@ pub(crate) fn resolve_positions_to_exprs( let index = (position - 1) as usize; let select_expr = &select_exprs[index]; Ok(match select_expr { - Expr::Alias(Alias { expr, .. }) => *expr.clone(), + Expr::Alias(boxed_alias) => *boxed_alias.expr.clone(), _ => select_expr.clone(), }) } @@ -248,7 +248,7 @@ pub fn window_expr_common_partition_keys(window_exprs: &[Expr]) -> Result<&[Expr } = window_fun.as_ref(); Ok(partition_by) } - Expr::Alias(Alias { expr, .. }) => match expr.as_ref() { + Expr::Alias(boxed_alias) => match boxed_alias.expr.as_ref() { Expr::WindowFunction(window_fun) => { let WindowFunction { params: WindowFunctionParams { partition_by, .. }, diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt index 5d68ed35b2a9..785a17c7bee8 100644 --- a/datafusion/sqllogictest/test_files/joins.slt +++ b/datafusion/sqllogictest/test_files/joins.slt @@ -4013,12 +4013,12 @@ logical_plan 09)------------Unnest: lists[__unnest_placeholder(generate_series(Int64(1),outer_ref(t1.t1_int)))|depth=1] structs[] 10)--------------Projection: generate_series(Int64(1), CAST(outer_ref(t1.t1_int) AS Int64)) AS __unnest_placeholder(generate_series(Int64(1),outer_ref(t1.t1_int))) 11)----------------EmptyRelation -physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(UInt32, Column { relation: Some(Bare { table: "t1" }), name: "t1_int" }) +physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn((UInt32, Column { relation: Some(Bare { table: "t1" }), name: "t1_int" })) # Test CROSS JOIN LATERAL syntax (execution) # TODO: https://github.com/apache/datafusion/issues/10048 -query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn\(UInt32, Column \{ relation: Some\(Bare \{ table: "t1" \}\), name: "t1_int" \}\) +query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn\(\(UInt32, Column \{ relation: Some\(Bare \{ table: "t1" \}\), name: "t1_int" \}\)\) select t1_id, t1_name, i from join_t1 t1 cross join lateral (select * from unnest(generate_series(1, t1_int))) as series(i); @@ -4038,12 +4038,12 @@ logical_plan 09)------------Unnest: lists[__unnest_placeholder(generate_series(Int64(1),outer_ref(t2.t1_int)))|depth=1] structs[] 10)--------------Projection: generate_series(Int64(1), CAST(outer_ref(t2.t1_int) AS Int64)) AS __unnest_placeholder(generate_series(Int64(1),outer_ref(t2.t1_int))) 11)----------------EmptyRelation -physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(UInt32, Column { relation: Some(Bare { table: "t2" }), name: "t1_int" }) +physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn((UInt32, Column { relation: Some(Bare { table: "t2" }), name: "t1_int" })) # Test INNER JOIN LATERAL syntax (execution) # TODO: https://github.com/apache/datafusion/issues/10048 -query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn\(UInt32, Column \{ relation: Some\(Bare \{ table: "t2" \}\), name: "t1_int" \}\) +query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn\(\(UInt32, Column \{ relation: Some\(Bare \{ table: "t2" \}\), name: "t1_int" \}\)\) select t1_id, t1_name, i from join_t1 t2 inner join lateral (select * from unnest(generate_series(1, t1_int))) as series(i) on(t1_id > i); # Test RIGHT JOIN LATERAL syntax (unsupported) @@ -4648,7 +4648,7 @@ logical_plan 05)------Subquery: 06)--------Filter: outer_ref(j1.j1_id) < j2.j2_id 07)----------TableScan: j2 projection=[j2_string, j2_id] -physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(Int32, Column { relation: Some(Bare { table: "j1" }), name: "j1_id" }) +physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn((Int32, Column { relation: Some(Bare { table: "j1" }), name: "j1_id" })) query TT explain SELECT * FROM j1 JOIN (j2 JOIN j3 ON(j2_id = j3_id - 2)) ON(j1_id = j2_id), LATERAL (SELECT * FROM j3 WHERE j3_string = j2_string) as j4 @@ -4664,7 +4664,7 @@ logical_plan 08)----Subquery: 09)------Filter: j3.j3_string = outer_ref(j2.j2_string) 10)--------TableScan: j3 projection=[j3_string, j3_id] -physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(Utf8View, Column { relation: Some(Bare { table: "j2" }), name: "j2_string" }) +physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn((Utf8View, Column { relation: Some(Bare { table: "j2" }), name: "j2_string" })) query TT explain SELECT * FROM j1, LATERAL (SELECT * FROM j1, LATERAL (SELECT * FROM j2 WHERE j1_id = j2_id) as j2) as j2; @@ -4680,7 +4680,7 @@ logical_plan 08)----------Subquery: 09)------------Filter: outer_ref(j1.j1_id) = j2.j2_id 10)--------------TableScan: j2 projection=[j2_string, j2_id] -physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(Int32, Column { relation: Some(Bare { table: "j1" }), name: "j1_id" }) +physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn((Int32, Column { relation: Some(Bare { table: "j1" }), name: "j1_id" })) query TT explain SELECT j1_string, j2_string FROM j1 LEFT JOIN LATERAL (SELECT * FROM j2 WHERE j1_id < j2_id) AS j2 ON(true); @@ -4693,7 +4693,7 @@ logical_plan 05)------Subquery: 06)--------Filter: outer_ref(j1.j1_id) < j2.j2_id 07)----------TableScan: j2 projection=[j2_string, j2_id] -physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(Int32, Column { relation: Some(Bare { table: "j1" }), name: "j1_id" }) +physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn((Int32, Column { relation: Some(Bare { table: "j1" }), name: "j1_id" })) query TT explain SELECT * FROM j1, (j2 LEFT JOIN LATERAL (SELECT * FROM j3 WHERE j1_id + j2_id = j3_id) AS j3 ON(true)); @@ -4707,7 +4707,7 @@ logical_plan 06)------Subquery: 07)--------Filter: outer_ref(j1.j1_id) + outer_ref(j2.j2_id) = j3.j3_id 08)----------TableScan: j3 projection=[j3_string, j3_id] -physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(Int32, Column { relation: Some(Bare { table: "j1" }), name: "j1_id" }) +physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn((Int32, Column { relation: Some(Bare { table: "j1" }), name: "j1_id" })) query TT explain SELECT * FROM j1, LATERAL (SELECT 1) AS j2; diff --git a/datafusion/sqllogictest/test_files/unnest.slt b/datafusion/sqllogictest/test_files/unnest.slt index 92e6f9995ae3..67fec4c90cea 100644 --- a/datafusion/sqllogictest/test_files/unnest.slt +++ b/datafusion/sqllogictest/test_files/unnest.slt @@ -863,11 +863,11 @@ select count(*) from (select unnest(range(0, 100000)) id) t inner join (select u # Test implicit LATERAL support for UNNEST # Issue: https://github.com/apache/datafusion/issues/13659 # TODO: https://github.com/apache/datafusion/issues/10048 -query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn\(List\(Field \{ name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: \{\} \}\), Column \{ relation: Some\(Bare \{ table: "u" \}\), name: "column1" \}\) +query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn\(\(List\(Field \{ name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: \{\} \}\), Column \{ relation: Some\(Bare \{ table: "u" \}\), name: "column1" \}\)\) select * from unnest_table u, unnest(u.column1); # Test implicit LATERAL support for UNNEST (INNER JOIN) -query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn\(List\(Field \{ name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: \{\} \}\), Column \{ relation: Some\(Bare \{ table: "u" \}\), name: "column1" \}\) +query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn\(\(List\(Field \{ name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: \{\} \}\), Column \{ relation: Some\(Bare \{ table: "u" \}\), name: "column1" \}\)\) select * from unnest_table u INNER JOIN unnest(u.column1) AS t(column1) ON u.column3 = t.column1; # Test implicit LATERAL planning for UNNEST @@ -875,7 +875,7 @@ query TT explain select * from unnest_table u, unnest(u.column1); ---- logical_plan -01)Cross Join: +01)Cross Join: 02)--SubqueryAlias: u 03)----TableScan: unnest_table projection=[column1, column2, column3, column4, column5] 04)--Subquery: @@ -883,7 +883,7 @@ logical_plan 06)------Unnest: lists[__unnest_placeholder(outer_ref(u.column1))|depth=1] structs[] 07)--------Projection: outer_ref(u.column1) AS __unnest_placeholder(outer_ref(u.column1)) 08)----------EmptyRelation -physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(List(Field { name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), Column { relation: Some(Bare { table: "u" }), name: "column1" }) +physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn((List(Field { name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), Column { relation: Some(Bare { table: "u" }), name: "column1" })) # Test implicit LATERAL planning for UNNEST (INNER JOIN) query TT @@ -899,7 +899,7 @@ logical_plan 07)--------Unnest: lists[__unnest_placeholder(outer_ref(u.column1))|depth=1] structs[] 08)----------Projection: outer_ref(u.column1) AS __unnest_placeholder(outer_ref(u.column1)) 09)------------EmptyRelation -physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(List(Field { name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), Column { relation: Some(Bare { table: "u" }), name: "column1" }) +physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn((List(Field { name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), Column { relation: Some(Bare { table: "u" }), name: "column1" })) # uncorrelated EXISTS with unnest query I diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs index 42e1f962f1d1..9ef1e0749194 100644 --- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs +++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs @@ -144,7 +144,7 @@ pub fn to_substrait_rex( Expr::Wildcard { .. } => not_impl_err!("Cannot convert {expr:?} to Substrait"), Expr::GroupingSet(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"), Expr::Placeholder(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"), - Expr::OuterReferenceColumn(_, _) => { + Expr::OuterReferenceColumn(_) => { not_impl_err!("Cannot convert {expr:?} to Substrait") } Expr::Unnest(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"), diff --git a/datafusion/substrait/src/logical_plan/producer/rel/aggregate_rel.rs b/datafusion/substrait/src/logical_plan/producer/rel/aggregate_rel.rs index 4abd283a7ee0..255de0cd5266 100644 --- a/datafusion/substrait/src/logical_plan/producer/rel/aggregate_rel.rs +++ b/datafusion/substrait/src/logical_plan/producer/rel/aggregate_rel.rs @@ -19,7 +19,6 @@ use crate::logical_plan::producer::{ from_aggregate_function, substrait_field_ref, SubstraitProducer, }; use datafusion::common::{internal_err, not_impl_err, DFSchemaRef, DataFusionError}; -use datafusion::logical_expr::expr::Alias; use datafusion::logical_expr::{Aggregate, Distinct, Expr, GroupingSet}; use substrait::proto::aggregate_rel::{Grouping, Measure}; use substrait::proto::rel::RelType; @@ -170,8 +169,8 @@ pub fn to_substrait_agg_measure( ) -> datafusion::common::Result { match expr { Expr::AggregateFunction(agg_fn) => from_aggregate_function(producer, agg_fn, schema), - Expr::Alias(Alias { expr, .. }) => { - to_substrait_agg_measure(producer, expr, schema) + Expr::Alias(boxed_alias) => { + to_substrait_agg_measure(producer, boxed_alias.expr.as_ref(), schema) } _ => internal_err!( "Expression must be compatible with aggregation. Unsupported expression: {:?}. ExpressionType: {:?}",