Merge pull request #383 from chdb-io/update_ch_core

Feat(upgrade): update ch core to v25.8.2.29

Author: auxten

.clang-tidy

@@ -7,6 +7,9 @@
 
 HeaderFilterRegex: '^.*/(base|src|programs|utils)/.*(h|hpp)$'
 
+# We don't want to use clang-tidy diagnostics in the "contrib" folder.
+ExcludeHeaderFilterRegex: '^.*/contrib/.*'
+
 Checks: [
   '*',
 
@@ -28,6 +31,7 @@ Checks: [
   '-bugprone-unchecked-optional-access',
   '-bugprone-crtp-constructor-accessibility',
   '-bugprone-not-null-terminated-result',
+  '-bugprone-forward-declaration-namespace',
 
   '-cert-dcl16-c',
   '-cert-err58-cpp',

.github/PULL_REQUEST_TEMPLATE.md

@@ -17,7 +17,7 @@ tests/ci/cancel_and_rerun_workflow_lambda/app.py
 - Not for changelog (changelog entry is not required)
 
 
-### Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
+### Changelog entry (a [user-readable short description](https://github.com/ClickHouse/ClickHouse/blob/master/docs/changelog_entry_guidelines.md) of the changes that goes into CHANGELOG.md):
 ...
 
 ### Documentation entry for user-facing changes

.github/workflows/build_linux_arm64_wheels-gh.yml

@@ -106,6 +106,13 @@ jobs:
             python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel
             pyenv shell --unset
           done
+      - name: Upgrade Rust toolchain
+        run: |
+          rustup toolchain install nightly-2025-07-07
+          rustup default nightly-2025-07-07
+          rustup component add rust-src
+          rustc --version
+          cargo --version
       - name: Install clang++ for Ubuntu
         run: |
           pwd
@@ -116,6 +123,13 @@ jobs:
           which clang++-19
           clang++-19 --version
           sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget
+          # Install WebAssembly linker (wasm-ld) 
+          sudo apt-get install -y lld-19
+          # Create symlink for wasm-ld
+          if ! command -v wasm-ld &> /dev/null; then
+            sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true
+          fi
+          which wasm-ld || echo "wasm-ld not found in PATH"
           ccache -s
       - name: Update git
         run: |
@@ -148,6 +162,7 @@ jobs:
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
+          source ~/.cargo/env
           pyenv shell 3.8
           export CC=/usr/bin/clang
           export CXX=/usr/bin/clang++
@@ -161,6 +176,7 @@ jobs:
           export CC=/usr/bin/clang
           export CXX=/usr/bin/clang++
           export PATH="$HOME/.pyenv/bin:$PATH"
+          source ~/.cargo/env
           eval "$(pyenv init -)"
           pyenv shell 3.8
           bash ./chdb/build/build_static_lib.sh

.github/workflows/build_linux_x86_wheels.yml

@@ -106,6 +106,13 @@ jobs:
             python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel
             pyenv shell --unset
           done
+      - name: Upgrade Rust toolchain
+        run: |
+          rustup toolchain install nightly-2025-07-07
+          rustup default nightly-2025-07-07
+          rustup component add rust-src
+          rustc --version
+          cargo --version
       - name: Install clang++ for Ubuntu
         run: |
           pwd
@@ -116,6 +123,13 @@ jobs:
           which clang++-19
           clang++-19 --version
           sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget
+          # Install WebAssembly linker (wasm-ld) 
+          sudo apt-get install -y lld-19
+          # Create symlink for wasm-ld
+          if ! command -v wasm-ld &> /dev/null; then
+            sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true
+          fi
+          which wasm-ld || echo "wasm-ld not found in PATH"
           ccache -s
       - name: Update git
         run: |
@@ -148,6 +162,7 @@ jobs:
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
+          source ~/.cargo/env
           pyenv shell 3.8
           export CC=/usr/bin/clang
           export CXX=/usr/bin/clang++
@@ -161,6 +176,7 @@ jobs:
           export CC=/usr/bin/clang
           export CXX=/usr/bin/clang++
           export PATH="$HOME/.pyenv/bin:$PATH"
+          source ~/.cargo/env
           eval "$(pyenv init -)"
           pyenv shell 3.8
           bash ./chdb/build/build_static_lib.sh

.github/workflows/build_macos_arm64_wheels.yml

@@ -79,61 +79,37 @@ jobs:
           brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd
           brew install openssl@3 || echo "OpenSSL install failed, continuing..."
           brew install --ignore-dependencies llvm@19
-          brew install git ninja libtool gettext binutils grep findutils nasm
-          # brew install gcc || echo "GCC install failed, continuing..."
-          # brew install ccache || echo "ccache installation failed, continuing without it"
+          brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv
           brew install go
           cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm
-          export PATH=$(brew --prefix llvm@19)/bin:$PATH
+          export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH
           which clang++
           clang++ --version
+          which wasm-ld || echo "wasm-ld not found in PATH"
           which go
           go version
-          ccache -s || echo "ccache not available yet"
-      - name: Scan SQLite vulnerabilities with grype
+          ccache -s | echo "ccache not available yet"
+      - name: Upgrade Rust toolchain
         run: |
-          # Install grype
-          curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
-
-          # Update grype vulnerability database
-          grype db update
-
-          # Check SQLite vulnerabilities in Homebrew packages
-          echo "Scanning SQLite packages for vulnerabilities..."
-          GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true)
-          echo "Raw grype output:"
-          echo "$GRYPE_RAW_OUTPUT"
-
-          SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
-
-          if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
-            echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed."
-            echo "SQLite vulnerability details:"
-            echo "$SQLITE_SCAN_OUTPUT"
-            exit 1
-          else
-            echo "✅ No SQLite vulnerabilities found"
-          fi
-        continue-on-error: false
+          rustup toolchain install nightly-2025-07-07
+          rustup default nightly-2025-07-07
+          rustup component add rust-src
+          rustc --version
+          cargo --version
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Update submodules
         run: |
           git submodule update --init --recursive --jobs 4
-      # - name: ccache
-      #   uses: hendrikmuhs/[email protected]
-      #   with:
-      #     key: macos-13-xlarge
-      #     max-size: 5G
-      #     append-timestamp: true
       - name: Run chdb/build.sh
         timeout-minutes: 600
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
+          source ~/.cargo/env
           pyenv shell 3.8
-          export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
+          export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@19)/bin/clang
           export CXX=$(brew --prefix llvm@19)/bin/clang++
           bash gen_manifest.sh
@@ -143,9 +119,10 @@ jobs:
       - name: Run chdb/build/build_static_lib.sh
         timeout-minutes: 600
         run: |
-          export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
+          export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@19)/bin/clang
           export CXX=$(brew --prefix llvm@19)/bin/clang++
+          source ~/.cargo/env
           eval "$(pyenv init -)"
           pyenv shell 3.8
           bash ./chdb/build/build_static_lib.sh
@@ -168,28 +145,23 @@ jobs:
       - name: Scan chdb libraries with grype
         run: |
           echo "Scanning chdb libraries for vulnerabilities..."
-
           # Files to scan
           FILES_TO_SCAN=""
           [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
           [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
           FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)"
-
           SQLITE_VULNERABILITIES_FOUND=false
-
           for file in $FILES_TO_SCAN; do
             if [ -f "$file" ]; then
               echo "=== Scanning $file ==="
               SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
               echo "$SCAN_OUTPUT"
-
               if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
                 echo "❌ SQLite vulnerability found in $file"
                 SQLITE_VULNERABILITIES_FOUND=true
               fi
             fi
           done
-
           if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
             echo "❌ SQLite vulnerabilities detected in chdb libraries!"
             exit 1
@@ -200,26 +172,11 @@ jobs:
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh
-      # - name: Keep killall ccache and wait for ccache to finish
-      #   if: always()
-      #   run: |
-      #     sleep 60
-      #     while ps -ef | grep ccache | grep -v grep; do \
-      #       killall ccache || true; \
-      #       sleep 10; \
-      #     done
-      # - name: Check ccache statistics
-      #   run: |
-      #     ccache -s || echo "ccache not available"
-      #     ls -lh chdb
-      #     df -h
-      #   env:
-      #     CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@19)/bin/clang CXX=$(brew --prefix llvm@19)/bin/clang++"
       - name: Build wheels
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
-          export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
+          export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@19)/bin/clang
           export CXX=$(brew --prefix llvm@19)/bin/clang++
           pyenv shell 3.8

.github/workflows/build_macos_x86_wheels.yml

@@ -78,32 +78,28 @@ jobs:
           brew update
           brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd
           brew install --ignore-dependencies llvm@19
-          brew install git ninja libtool gettext gcc binutils grep findutils nasm
+          brew install git ninja libtool gettext gcc binutils grep findutils nasm lld@19 libiconv
           brew install ccache || echo "ccache installation failed, continuing without it"
           brew install go
           cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm
-          export PATH=$(brew --prefix llvm@19)/bin:$PATH
+          export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH
           which clang++
           clang++ --version
           which go
           go version
-          ccache -s
+          ccache -s || echo "ccache not available yet"
       - name: Scan SQLite vulnerabilities with grype
         run: |
           # Install grype
           curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
-
           # Update grype vulnerability database
           grype db update
-
           # Check SQLite vulnerabilities in Homebrew packages
           echo "Scanning SQLite packages for vulnerabilities..."
-          GRYPE_RAW_OUTPUT=$(grype dir:/usr/local --scope all-layers 2>/dev/null || true)
+          GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true)
           echo "Raw grype output:"
           echo "$GRYPE_RAW_OUTPUT"
-
           SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
-
           if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
             echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed."
             echo "SQLite vulnerability details:"
@@ -113,25 +109,27 @@ jobs:
             echo "✅ No SQLite vulnerabilities found"
           fi
         continue-on-error: false
+      - name: Upgrade Rust toolchain
+        run: |
+          rustup toolchain install nightly-2025-07-07
+          rustup default nightly-2025-07-07
+          rustup component add rust-src
+          rustc --version
+          cargo --version
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Update submodules
         run: |
           git submodule update --init --recursive --jobs 4
-      - name: ccache
-        uses: hendrikmuhs/[email protected]
-        with:
-          key: macos-14-x86_64
-          max-size: 5G
-          append-timestamp: true
       - name: Run chdb/build.sh
         timeout-minutes: 600
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
+          source ~/.cargo/env
           pyenv shell 3.8
-          export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
+          export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@19)/bin/clang
           export CXX=$(brew --prefix llvm@19)/bin/clang++
           bash gen_manifest.sh
@@ -142,9 +140,10 @@ jobs:
       - name: Run chdb/build/build_static_lib.sh
         timeout-minutes: 600
         run: |
-          export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
+          export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@19)/bin/clang
           export CXX=$(brew --prefix llvm@19)/bin/clang++
+          source ~/.cargo/env
           eval "$(pyenv init -)"
           pyenv shell 3.8
           bash ./chdb/build/build_static_lib.sh
@@ -167,28 +166,23 @@ jobs:
       - name: Scan chdb libraries with grype
         run: |
           echo "Scanning chdb libraries for vulnerabilities..."
-
           # Files to scan
           FILES_TO_SCAN=""
           [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
           [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
           FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)"
-
           SQLITE_VULNERABILITIES_FOUND=false
-
           for file in $FILES_TO_SCAN; do
             if [ -f "$file" ]; then
               echo "=== Scanning $file ==="
               SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
               echo "$SCAN_OUTPUT"
-
               if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
                 echo "❌ SQLite vulnerability found in $file"
                 SQLITE_VULNERABILITIES_FOUND=true
               fi
             fi
           done
-
           if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
             echo "❌ SQLite vulnerabilities detected in chdb libraries!"
             exit 1
@@ -199,26 +193,12 @@ jobs:
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh
-      - name: Keep killall ccache and wait for ccache to finish
-        if: always()
-        run: |
-          sleep 60
-          while ps -ef | grep ccache | grep -v grep; do \
-            killall ccache; \
-            sleep 10; \
-          done
-      - name: Check ccache statistics
-        run: |
-          ccache -s
-          ls -lh chdb
-          df -h
-        env:
-          CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@19)/bin/clang CXX=$(brew --prefix llvm@19)/bin/clang++"
       - name: Build wheels
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
-          export PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
+          source ~/.cargo/env
+          export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@19)/bin/clang
           export CXX=$(brew --prefix llvm@19)/bin/clang++
           pyenv shell 3.8