Skeleton for QGM generation from Hir

src/expr/src/lib.rs

@@ -37,7 +37,7 @@ pub use relation::func::{AggregateFunc, TableFunc};
 pub use relation::func::{AnalyzedRegex, CaptureGroupDesc};
 pub use relation::join_input_mapper::JoinInputMapper;
 pub use relation::{
-    compare_columns, AggregateExpr, ColumnOrder, IdGen, JoinImplementation, MirRelationExpr,
+    compare_columns, AggregateExpr, ColumnOrder, JoinImplementation, MirRelationExpr,
     RowSetFinishing,
 };
 pub use scalar::func::{self, BinaryFunc, NullaryFunc, UnaryFunc, VariadicFunc};

src/expr/src/relation/mod.rs

@@ -18,6 +18,7 @@ use serde::{Deserialize, Serialize};
 
 use lowertest::{MzEnumReflect, MzStructReflect};
 use ore::collections::CollectionExt;
+use ore::id_gen::IdGen;
 use repr::{ColumnType, Datum, Diff, RelationType, Row};
 
 use self::func::{AggregateFunc, TableFunc};
@@ -1480,21 +1481,6 @@ impl fmt::Display for ColumnOrder {
     }
 }
 
-/// Manages the allocation of locally unique IDs when building a [`MirRelationExpr`].
-#[derive(Debug, Default)]
-pub struct IdGen {
-    id: u64,
-}
-
-impl IdGen {
-    /// Allocates a new identifier and advances the generator.
-    pub fn allocate_id(&mut self) -> u64 {
-        let id = self.id;
-        self.id += 1;
-        id
-    }
-}
-
 /// Describes an aggregation expression.
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Hash, MzStructReflect)]
 pub struct AggregateExpr {

src/ore/src/id_gen.rs

@@ -0,0 +1,31 @@
+// Copyright Materialize, Inc. and contributors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License in the LICENSE file at the
+// root of this repository, or online at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! ID generation utilities.
+
+/// Manages the allocation of unique IDs.
+#[derive(Debug, Default)]
+pub struct IdGen {
+    id: u64,
+}
+
+impl IdGen {
+    /// Allocates a new identifier and advances the generator.
+    pub fn allocate_id(&mut self) -> u64 {
+        let id = self.id;
+        self.id += 1;
+        id
+    }
+}

src/ore/src/lib.rs

@@ -36,6 +36,7 @@ pub mod fmt;
 pub mod future;
 pub mod hash;
 pub mod hint;
+pub mod id_gen;
 pub mod iter;
 pub mod lex;
 #[cfg(feature = "metrics")]

src/sql/src/lib.rs

@@ -92,3 +92,5 @@ pub mod normalize;
 pub mod parse;
 pub mod plan;
 pub mod pure;
+#[cfg(test)]
+pub mod query_model;

src/sql/src/plan/explain.rs

@@ -22,7 +22,8 @@ use std::collections::{BTreeMap, HashMap};
 use std::fmt;
 
 use expr::explain::Indices;
-use expr::{ExprHumanizer, Id, IdGen, RowSetFinishing};
+use expr::{ExprHumanizer, Id, RowSetFinishing};
+use ore::id_gen::IdGen;
 use ore::str::{bracketed, separated};
 use repr::{RelationType, ScalarType};
 

src/sql/src/plan/lowering.rs

@@ -129,7 +129,7 @@ impl HirRelationExpr {
                 }
             }
             mut other => {
-                let mut id_gen = expr::IdGen::default();
+                let mut id_gen = ore::id_gen::IdGen::default();
                 transform_expr::split_subquery_predicates(&mut other);
                 transform_expr::try_simplify_quantified_comparisons(&mut other);
                 expr::MirRelationExpr::constant(vec![vec![]], RelationType::new(vec![]))
@@ -154,7 +154,7 @@ impl HirRelationExpr {
     /// assignment of values to outer rows.
     fn applied_to(
         self,
-        id_gen: &mut expr::IdGen,
+        id_gen: &mut ore::id_gen::IdGen,
         get_outer: expr::MirRelationExpr,
         col_map: &ColumnMap,
     ) -> expr::MirRelationExpr {
@@ -547,7 +547,7 @@ impl HirScalarExpr {
     /// each of these references can be found.
     fn applied_to(
         self,
-        id_gen: &mut expr::IdGen,
+        id_gen: &mut ore::id_gen::IdGen,
         col_map: &ColumnMap,
         inner: &mut expr::MirRelationExpr,
         subquery_map: &Option<&HashMap<HirScalarExpr, usize>>,
@@ -710,7 +710,7 @@ impl HirScalarExpr {
     /// by the returned join that will hold their results.
     fn lower_subqueries(
         exprs: &[Self],
-        id_gen: &mut expr::IdGen,
+        id_gen: &mut ore::id_gen::IdGen,
         col_map: &ColumnMap,
         inner: expr::MirRelationExpr,
     ) -> (expr::MirRelationExpr, HashMap<HirScalarExpr, usize>) {
@@ -852,7 +852,7 @@ impl HirScalarExpr {
 /// The caller must supply the `apply` function that applies the rewritten
 /// `inner` to `outer`.
 fn branch<F>(
-    id_gen: &mut expr::IdGen,
+    id_gen: &mut ore::id_gen::IdGen,
     outer: expr::MirRelationExpr,
     col_map: &ColumnMap,
     mut inner: HirRelationExpr,
@@ -861,7 +861,7 @@ fn branch<F>(
 ) -> expr::MirRelationExpr
 where
     F: FnOnce(
-        &mut expr::IdGen,
+        &mut ore::id_gen::IdGen,
         HirRelationExpr,
         expr::MirRelationExpr,
         &ColumnMap,
@@ -990,7 +990,7 @@ where
 }
 
 fn apply_scalar_subquery(
-    id_gen: &mut expr::IdGen,
+    id_gen: &mut ore::id_gen::IdGen,
     outer: expr::MirRelationExpr,
     col_map: &ColumnMap,
     scalar_subquery: HirRelationExpr,
@@ -1044,7 +1044,7 @@ fn apply_scalar_subquery(
 }
 
 fn apply_existential_subquery(
-    id_gen: &mut expr::IdGen,
+    id_gen: &mut ore::id_gen::IdGen,
     outer: expr::MirRelationExpr,
     col_map: &ColumnMap,
     subquery_expr: HirRelationExpr,
@@ -1081,7 +1081,7 @@ fn apply_existential_subquery(
 impl AggregateExpr {
     fn applied_to(
         self,
-        id_gen: &mut expr::IdGen,
+        id_gen: &mut ore::id_gen::IdGen,
         col_map: &ColumnMap,
         inner: &mut expr::MirRelationExpr,
     ) -> expr::AggregateExpr {
@@ -1106,7 +1106,7 @@ fn attempt_outer_join(
     on: expr::MirScalarExpr,
     kind: JoinKind,
     oa: usize,
-    id_gen: &mut expr::IdGen,
+    id_gen: &mut ore::id_gen::IdGen,
 ) -> Option<expr::MirRelationExpr> {
     use expr::BinaryFunc;
 

src/sql/src/query_model/README.md

@@ -0,0 +1,29 @@
+# Query Graph Model
+
+This module contains the implementation of the Query Graph Model as specified
+[here](../../../../doc/developer/design/20210707_qgm_sql_high_level_representation.md).
+
+## Testing
+
+Tests around the Query Graph Model, for both the model generation logic and the
+model transformat logic, are usually `datadriven` tests that result in multiple
+`graphviz` graphs, that need to be rendered and visually validated. The following
+shell function extracts all the `graphviz` graphs containing in one of these tests
+and renders them as PNG files. The snippet after it shows how it can be used with
+one of these tests.
+
+
+```sh
+extract_graph() {
+    sed -n "/^digraph G {$/,/^}$/p" $1 | csplit --prefix $1- --suffix-format "%04d.dot" --elide-empty-files - '/^digraph G {$/' '{*}'
+    for i in $1-*.dot; do
+        dot -Tpng -O $i
+    done
+}
+```
+
+```
+$ extract_graph src/sql/tests/querymodel/basic
+...
+$ eog src/sql/tests/querymodel/basic*.png &
+```