Skip to content

Commit 35adf47

Browse files
OussamaSaoudijonahgao
OussamaSaoudi
and
jonahgao
authored
Add support for external tables with qualified names (#12645)
* Make support schemas * Set default name to table * Remove print statements and stale comment * Add tests for create table * Fix typo * Update datafusion/sql/src/statement.rs Co-authored-by: Jonah Gao <[email protected]> * convert create_external_table to objectname * Add sqllogic tests * Fix failing tests --------- Co-authored-by: Jonah Gao <[email protected]>
1 parent 23d7fff commit 35adf47

File tree

5 files changed

+41
-24
lines changed

5 files changed

+41
-24
lines changed

‎datafusion/core/src/catalog_common/mod.rs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -185,9 +185,7 @@ pub fn resolve_table_references(
185185
let _ = s.as_ref().visit(visitor);
186186
}
187187
DFStatement::CreateExternalTable(table) => {
188-
visitor
189-
.relations
190-
.insert(ObjectName(vec![Ident::from(table.name.as_str())]));
188+
visitor.relations.insert(table.name.clone());
191189
}
192190
DFStatement::CopyTo(CopyToStatement { source, .. }) => match source {
193191
CopyToSource::Relation(table_name) => {

‎datafusion/sql/src/parser.rs‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ pub(crate) type LexOrdering = Vec<OrderByExpr>;
181181
#[derive(Debug, Clone, PartialEq, Eq)]
182182
pub struct CreateExternalTable {
183183
/// Table name
184-
pub name: String,
184+
pub name: ObjectName,
185185
/// Optional schema
186186
pub columns: Vec<ColumnDef>,
187187
/// File type (Parquet, NDJSON, CSV, etc)
@@ -813,7 +813,7 @@ impl<'a> DFParser<'a> {
813813
}
814814

815815
let create = CreateExternalTable {
816-
name: table_name.to_string(),
816+
name: table_name,
817817
columns,
818818
file_type: builder.file_type.unwrap(),
819819
location: builder.location.unwrap(),
@@ -915,8 +915,9 @@ mod tests {
915915
// positive case
916916
let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv'";
917917
let display = None;
918+
let name = ObjectName(vec![Ident::from("t")]);
918919
let expected = Statement::CreateExternalTable(CreateExternalTable {
919-
name: "t".into(),
920+
name: name.clone(),
920921
columns: vec![make_column_def("c1", DataType::Int(display))],
921922
file_type: "CSV".to_string(),
922923
location: "foo.csv".into(),
@@ -932,7 +933,7 @@ mod tests {
932933
// positive case: leading space
933934
let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' ";
934935
let expected = Statement::CreateExternalTable(CreateExternalTable {
935-
name: "t".into(),
936+
name: name.clone(),
936937
columns: vec![make_column_def("c1", DataType::Int(None))],
937938
file_type: "CSV".to_string(),
938939
location: "foo.csv".into(),
@@ -949,7 +950,7 @@ mod tests {
949950
let sql =
950951
"CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' ;";
951952
let expected = Statement::CreateExternalTable(CreateExternalTable {
952-
name: "t".into(),
953+
name: name.clone(),
953954
columns: vec![make_column_def("c1", DataType::Int(None))],
954955
file_type: "CSV".to_string(),
955956
location: "foo.csv".into(),
@@ -966,7 +967,7 @@ mod tests {
966967
let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv' OPTIONS (format.delimiter '|')";
967968
let display = None;
968969
let expected = Statement::CreateExternalTable(CreateExternalTable {
969-
name: "t".into(),
970+
name: name.clone(),
970971
columns: vec![make_column_def("c1", DataType::Int(display))],
971972
file_type: "CSV".to_string(),
972973
location: "foo.csv".into(),
@@ -986,7 +987,7 @@ mod tests {
986987
let sql = "CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV PARTITIONED BY (p1, p2) LOCATION 'foo.csv'";
987988
let display = None;
988989
let expected = Statement::CreateExternalTable(CreateExternalTable {
989-
name: "t".into(),
990+
name: name.clone(),
990991
columns: vec![make_column_def("c1", DataType::Int(display))],
991992
file_type: "CSV".to_string(),
992993
location: "foo.csv".into(),
@@ -1013,7 +1014,7 @@ mod tests {
10131014
];
10141015
for (sql, compression) in sqls {
10151016
let expected = Statement::CreateExternalTable(CreateExternalTable {
1016-
name: "t".into(),
1017+
name: name.clone(),
10171018
columns: vec![make_column_def("c1", DataType::Int(display))],
10181019
file_type: "CSV".to_string(),
10191020
location: "foo.csv".into(),
@@ -1033,7 +1034,7 @@ mod tests {
10331034
// positive case: it is ok for parquet files not to have columns specified
10341035
let sql = "CREATE EXTERNAL TABLE t STORED AS PARQUET LOCATION 'foo.parquet'";
10351036
let expected = Statement::CreateExternalTable(CreateExternalTable {
1036-
name: "t".into(),
1037+
name: name.clone(),
10371038
columns: vec![],
10381039
file_type: "PARQUET".to_string(),
10391040
location: "foo.parquet".into(),
@@ -1049,7 +1050,7 @@ mod tests {
10491050
// positive case: it is ok for parquet files to be other than upper case
10501051
let sql = "CREATE EXTERNAL TABLE t STORED AS parqueT LOCATION 'foo.parquet'";
10511052
let expected = Statement::CreateExternalTable(CreateExternalTable {
1052-
name: "t".into(),
1053+
name: name.clone(),
10531054
columns: vec![],
10541055
file_type: "PARQUET".to_string(),
10551056
location: "foo.parquet".into(),
@@ -1065,7 +1066,7 @@ mod tests {
10651066
// positive case: it is ok for avro files not to have columns specified
10661067
let sql = "CREATE EXTERNAL TABLE t STORED AS AVRO LOCATION 'foo.avro'";
10671068
let expected = Statement::CreateExternalTable(CreateExternalTable {
1068-
name: "t".into(),
1069+
name: name.clone(),
10691070
columns: vec![],
10701071
file_type: "AVRO".to_string(),
10711072
location: "foo.avro".into(),
@@ -1082,7 +1083,7 @@ mod tests {
10821083
let sql =
10831084
"CREATE EXTERNAL TABLE IF NOT EXISTS t STORED AS PARQUET LOCATION 'foo.parquet'";
10841085
let expected = Statement::CreateExternalTable(CreateExternalTable {
1085-
name: "t".into(),
1086+
name: name.clone(),
10861087
columns: vec![],
10871088
file_type: "PARQUET".to_string(),
10881089
location: "foo.parquet".into(),
@@ -1099,7 +1100,7 @@ mod tests {
10991100
let sql =
11001101
"CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV PARTITIONED BY (p1 int) LOCATION 'foo.csv'";
11011102
let expected = Statement::CreateExternalTable(CreateExternalTable {
1102-
name: "t".into(),
1103+
name: name.clone(),
11031104
columns: vec![
11041105
make_column_def("c1", DataType::Int(None)),
11051106
make_column_def("p1", DataType::Int(None)),
@@ -1132,7 +1133,7 @@ mod tests {
11321133
let sql =
11331134
"CREATE EXTERNAL TABLE t STORED AS x OPTIONS ('k1' 'v1') LOCATION 'blahblah'";
11341135
let expected = Statement::CreateExternalTable(CreateExternalTable {
1135-
name: "t".into(),
1136+
name: name.clone(),
11361137
columns: vec![],
11371138
file_type: "X".to_string(),
11381139
location: "blahblah".into(),
@@ -1149,7 +1150,7 @@ mod tests {
11491150
let sql =
11501151
"CREATE EXTERNAL TABLE t STORED AS x OPTIONS ('k1' 'v1', k2 v2) LOCATION 'blahblah'";
11511152
let expected = Statement::CreateExternalTable(CreateExternalTable {
1152-
name: "t".into(),
1153+
name: name.clone(),
11531154
columns: vec![],
11541155
file_type: "X".to_string(),
11551156
location: "blahblah".into(),
@@ -1188,7 +1189,7 @@ mod tests {
11881189
];
11891190
for (sql, (asc, nulls_first)) in sqls.iter().zip(expected.into_iter()) {
11901191
let expected = Statement::CreateExternalTable(CreateExternalTable {
1191-
name: "t".into(),
1192+
name: name.clone(),
11921193
columns: vec![make_column_def("c1", DataType::Int(None))],
11931194
file_type: "CSV".to_string(),
11941195
location: "foo.csv".into(),
@@ -1214,7 +1215,7 @@ mod tests {
12141215
let sql = "CREATE EXTERNAL TABLE t(c1 int, c2 int) STORED AS CSV WITH ORDER (c1 ASC, c2 DESC NULLS FIRST) LOCATION 'foo.csv'";
12151216
let display = None;
12161217
let expected = Statement::CreateExternalTable(CreateExternalTable {
1217-
name: "t".into(),
1218+
name: name.clone(),
12181219
columns: vec![
12191220
make_column_def("c1", DataType::Int(display)),
12201221
make_column_def("c2", DataType::Int(display)),
@@ -1253,7 +1254,7 @@ mod tests {
12531254
let sql = "CREATE EXTERNAL TABLE t(c1 int, c2 int) STORED AS CSV WITH ORDER (c1 - c2 ASC) LOCATION 'foo.csv'";
12541255
let display = None;
12551256
let expected = Statement::CreateExternalTable(CreateExternalTable {
1256-
name: "t".into(),
1257+
name: name.clone(),
12571258
columns: vec![
12581259
make_column_def("c1", DataType::Int(display)),
12591260
make_column_def("c2", DataType::Int(display)),
@@ -1297,7 +1298,7 @@ mod tests {
12971298
'TRUNCATE' 'NO',
12981299
'format.has_header' 'true')";
12991300
let expected = Statement::CreateExternalTable(CreateExternalTable {
1300-
name: "t".into(),
1301+
name: name.clone(),
13011302
columns: vec![
13021303
make_column_def("c1", DataType::Int(None)),
13031304
make_column_def("c2", DataType::Float(None)),

‎datafusion/sql/src/statement.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1239,8 +1239,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
12391239
let ordered_exprs =
12401240
self.build_order_by(order_exprs, &df_schema, &mut planner_context)?;
12411241

1242-
// External tables do not support schemas at the moment, so the name is just a table name
1243-
let name = TableReference::bare(name);
1242+
let name = self.object_name_to_table_reference(name)?;
12441243
let constraints =
12451244
Constraints::new_from_table_constraints(&all_constraints, &df_schema)?;
12461245
Ok(LogicalPlan::Ddl(DdlStatement::CreateExternalTable(

‎datafusion/sql/tests/sql_integration.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1913,6 +1913,13 @@ fn create_external_table_with_pk() {
19131913
quick_test(sql, expected);
19141914
}
19151915

1916+
#[test]
1917+
fn create_external_table_wih_schema() {
1918+
let sql = "CREATE EXTERNAL TABLE staging.foo STORED AS CSV LOCATION 'foo.csv'";
1919+
let expected = "CreateExternalTable: Partial { schema: \"staging\", table: \"foo\" }";
1920+
quick_test(sql, expected);
1921+
}
1922+
19161923
#[test]
19171924
fn create_schema_with_quoted_name() {
19181925
let sql = "CREATE SCHEMA \"quoted_schema_name\"";

‎datafusion/sqllogictest/test_files/create_external_table.slt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,3 +275,15 @@ DROP TABLE t;
275275
# query should fail with bad column
276276
statement error DataFusion error: Error during planning: Column foo is not in schema
277277
CREATE EXTERNAL TABLE t STORED AS parquet LOCATION '../../parquet-testing/data/alltypes_plain.parquet' WITH ORDER (foo);
278+
279+
# Create external table with qualified name should belong to the schema
280+
statement ok
281+
CREATE SCHEMA staging;
282+
283+
statement ok
284+
CREATE EXTERNAL TABLE staging.foo STORED AS parquet LOCATION '../../parquet-testing/data/alltypes_plain.parquet';
285+
286+
# Create external table with qualified name, but no schema should error
287+
statement error DataFusion error: Error during planning: failed to resolve schema: release
288+
CREATE EXTERNAL TABLE release.bar STORED AS parquet LOCATION '../../parquet-testing/data/alltypes_plain.parquet';
289+

0 commit comments

Comments
 (0)