Merge remote-tracking branch 'upstream/main' into hash_join_batch_size

Author: korowa

.github/actions/setup-builder/action.yaml

@@ -38,19 +38,8 @@ runs:
         rustup toolchain install ${{ inputs.rust-version }}
         rustup default ${{ inputs.rust-version }}
         rustup component add rustfmt
-    - name: Disable debuginfo generation
-      # Disable full debug symbol generation to speed up CI build and keep memory down
-      # "1" means line tables only, which is useful for panic tracebacks.
-      shell: bash
-      run: echo "RUSTFLAGS=-C debuginfo=1" >> $GITHUB_ENV
-    - name: Disable incremental compilation
-      # Disable incremental compilation to save diskspace (the CI doesn't recompile modified files)
-      # https://github.com/apache/arrow-datafusion/issues/6676
-      shell: bash
-      run: echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
-    - name: Enable backtraces
-      shell: bash
-      run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
+    - name: Configure rust runtime env
+      uses: ./.github/actions/setup-rust-runtime
     - name: Fixup git permissions
       # https://github.com/actions/checkout/issues/766
       shell: bash

.github/actions/setup-macos-builder/action.yaml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder for MacOS
+description: 'Prepare Rust Build Environment for MacOS'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. stable)'
+    required: true
+    default: 'stable'
+runs:
+  using: "composite"
+  steps:
+    - name: Install protobuf compiler
+      shell: bash
+      run: |
+        mkdir -p $HOME/d/protoc
+        cd $HOME/d/protoc
+        export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"        
+        curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+        unzip $PROTO_ZIP
+        echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
+        export PATH=$PATH:$HOME/d/protoc/bin
+        protoc --version
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        rustup update stable
+        rustup toolchain install stable
+        rustup default stable
+        rustup component add rustfmt
+    - name: Configure rust runtime env
+      uses: ./.github/actions/setup-rust-runtime        

.github/actions/setup-rust-runtime/action.yaml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Setup Rust Runtime
+description: 'Setup Rust Runtime Environment'
+runs:
+  using: "composite"
+  steps:
+    - name: Run sccache-cache
+      uses: mozilla-actions/[email protected]   
+    - name: Configure runtime env
+      shell: bash
+      # do not produce debug symbols to keep memory usage down
+      # hardcoding other profile params to avoid profile override values
+      # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
+      # 
+      # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
+      # See for more details: https://github.com/rust-lang/rust/issues/119560
+      #
+      # set RUST_MIN_STACK to avoid rust stack overflows on tpc-ds tests
+      run: |
+        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV  
+        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV 
+        echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
+        echo "RUST_MIN_STACK=3000000" >> $GITHUB_ENV
+        echo "RUST_FLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV                                   
+     

.github/actions/setup-windows-builder/action.yaml

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder for Windows
+description: 'Prepare Rust Build Environment for Windows'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. stable)'
+    required: true
+    default: 'stable'
+runs:
+  using: "composite"
+  steps:
+    - name: Install protobuf compiler
+      shell: bash
+      run: |
+        mkdir -p $HOME/d/protoc
+        cd $HOME/d/protoc
+        export PROTO_ZIP="protoc-21.4-win64.zip"
+        curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+        unzip $PROTO_ZIP
+        export PATH=$PATH:$HOME/d/protoc/bin
+        protoc.exe --version
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        rustup update stable
+        rustup toolchain install stable
+        rustup default stable
+        rustup component add rustfmt
+    - name: Configure rust runtime env
+      uses: ./.github/actions/setup-rust-runtime        

.github/workflows/rust.yml

@@ -96,17 +96,9 @@ jobs:
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
-          rust-version: stable
+          rust-version: stable        
       - name: Run tests (excluding doctests)
         run: cargo test --lib --tests --bins --features avro,json,backtrace
-        env:
-          # do not produce debug symbols to keep memory usage down
-          # hardcoding other profile params to avoid profile override values
-          # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
-          RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C codegen-units=256"
-          RUST_BACKTRACE: "1"
-          # avoid rust stack overflows on tpc-ds tests
-          RUST_MINSTACK: "3000000"
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -284,80 +276,31 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - name: Install protobuf compiler
-        shell: bash
-        run: |
-          mkdir -p $HOME/d/protoc
-          cd $HOME/d/protoc
-          export PROTO_ZIP="protoc-21.4-win64.zip"
-          curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
-          unzip $PROTO_ZIP
-          export PATH=$PATH:$HOME/d/protoc/bin
-          protoc.exe --version
-      # TODO: this won't cache anything, which is expensive. Setup this action
-      # with a OS-dependent path.
       - name: Setup Rust toolchain
-        run: |
-          rustup update stable
-          rustup toolchain install stable
-          rustup default stable
-          rustup component add rustfmt
+        uses: ./.github/actions/setup-windows-builder
       - name: Run tests (excluding doctests)
         shell: bash
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
           cargo test --lib --tests --bins --features avro,json,backtrace
           cd datafusion-cli
           cargo test --lib --tests --bins --all-features
-        env:
-          # do not produce debug symbols to keep memory usage down
-          # use higher optimization level to overcome Windows rust slowness for tpc-ds
-          # and speed builds: https://github.com/apache/arrow-datafusion/issues/8696
-          # Cargo profile docs https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
-          RUSTFLAGS: "-C debuginfo=0 -C opt-level=1 -C target-feature=+crt-static -C incremental=false -C codegen-units=256"
-          RUST_BACKTRACE: "1"
-          # avoid rust stack overflows on tpc-ds tests
-          RUST_MINSTACK: "3000000"
+
   macos:
-    name: cargo test (mac)
+    name: cargo test (macos)
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
         with:
-          submodules: true
-      - name: Install protobuf compiler
-        shell: bash
-        run: |
-          mkdir -p $HOME/d/protoc
-          cd $HOME/d/protoc
-          export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"
-          curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
-          unzip $PROTO_ZIP
-          echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
-          export PATH=$PATH:$HOME/d/protoc/bin
-          protoc --version
-      # TODO: this won't cache anything, which is expensive. Setup this action
-      # with a OS-dependent path.
+          submodules: true      
       - name: Setup Rust toolchain
-        run: |
-          rustup update stable
-          rustup toolchain install stable
-          rustup default stable
-          rustup component add rustfmt
+        uses: ./.github/actions/setup-macos-builder       
       - name: Run tests (excluding doctests)
         shell: bash
         run: |
           cargo test --lib --tests --bins --features avro,json,backtrace
           cd datafusion-cli
-          cargo test --lib --tests --bins --all-features
-        env:
-          # do not produce debug symbols to keep memory usage down
-          # hardcoding other profile params to avoid profile override values
-          # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
-          RUSTFLAGS: "-C debuginfo=0 -C opt-level=0 -C incremental=false -C codegen-units=256"
-          RUST_BACKTRACE: "1"
-          # avoid rust stack overflows on tpc-ds tests
-          RUST_MINSTACK: "3000000"
+          cargo test --lib --tests --bins --all-features  
 
   test-datafusion-pyarrow:
     name: cargo test pyarrow (amd64)

datafusion-cli/src/exec.rs

@@ -51,8 +51,8 @@ use url::Url;
 /// run and execute SQL statements and commands, against a context with the given print options
 pub async fn exec_from_commands(
     ctx: &mut SessionContext,
-    print_options: &PrintOptions,
     commands: Vec<String>,
+    print_options: &PrintOptions,
 ) {
     for sql in commands {
         match exec_and_print(ctx, print_options, sql).await {
@@ -105,8 +105,8 @@ pub async fn exec_from_lines(
 }
 
 pub async fn exec_from_files(
-    files: Vec<String>,
     ctx: &mut SessionContext,
+    files: Vec<String>,
     print_options: &PrintOptions,
 ) {
     let files = files

datafusion-cli/src/main.rs

@@ -216,7 +216,7 @@ pub async fn main() -> Result<()> {
 
     if commands.is_empty() && files.is_empty() {
         if !rc.is_empty() {
-            exec::exec_from_files(rc, &mut ctx, &print_options).await
+            exec::exec_from_files(&mut ctx, rc, &print_options).await
         }
         // TODO maybe we can have thiserror for cli but for now let's keep it simple
         return exec::exec_from_repl(&mut ctx, &mut print_options)
@@ -225,11 +225,11 @@ pub async fn main() -> Result<()> {
     }
 
     if !files.is_empty() {
-        exec::exec_from_files(files, &mut ctx, &print_options).await;
+        exec::exec_from_files(&mut ctx, files, &print_options).await;
     }
 
     if !commands.is_empty() {
-        exec::exec_from_commands(&mut ctx, &print_options, commands).await;
+        exec::exec_from_commands(&mut ctx, commands, &print_options).await;
     }
 
     Ok(())

datafusion-cli/src/print_options.rs

@@ -141,7 +141,8 @@ impl PrintOptions {
         let mut row_count = 0_usize;
         let mut with_header = true;
 
-        while let Some(Ok(batch)) = stream.next().await {
+        while let Some(maybe_batch) = stream.next().await {
+            let batch = maybe_batch?;
             row_count += batch.num_rows();
             self.format.print_batches(
                 &mut writer,

datafusion-examples/README.md

@@ -62,6 +62,7 @@ cargo run --example csv_sql
 - [`simple_udf.rs`](examples/simple_udf.rs): Define and invoke a User Defined Scalar Function (UDF)
 - [`advanced_udf.rs`](examples/advanced_udf.rs): Define and invoke a more complicated User Defined Scalar Function (UDF)
 - [`simple_udaf.rs`](examples/simple_udaf.rs): Define and invoke a User Defined Aggregate Function (UDAF)
+- [`advanced_udaf.rs`](examples/advanced_udaf.rs): Define and invoke a more complicated User Defined Aggregate Function (UDAF)
 - [`simple_udfw.rs`](examples/simple_udwf.rs): Define and invoke a User Defined Window Function (UDWF)
 - [`advanced_udwf.rs`](examples/advanced_udwf.rs): Define and invoke a more complicated User Defined Window Function (UDWF)