diff --git a/.github/actionlint.yml b/.github/actionlint.yml index cf5f575e3c74..a7a76bd423dd 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -4,6 +4,10 @@ self-hosted-runner: - func-tester - func-tester-aarch64 - fuzzer-unit-tester + - altinity-on-demand + - altinity-type-cpx51 + - altinity-in-ash + - altinity-image-x86-system-ubuntu-22.04 - style-checker - style-checker-aarch64 - release-maker diff --git a/.github/actions/common_setup/action.yml b/.github/actions/common_setup/action.yml index b9299c64e72a..de7215d2103a 100644 --- a/.github/actions/common_setup/action.yml +++ b/.github/actions/common_setup/action.yml @@ -28,6 +28,16 @@ runs: run: | # to remove every leftovers sudo rm -fr "$TEMP_PATH" && mkdir -p "$TEMP_PATH" + - name: Setup zram + shell: bash + run: | + sudo modprobe zram + MemTotal=$(grep -Po "(?<=MemTotal:)\s+\d+" /proc/meminfo) # KiB + Percent=200 + ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB + .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0 + sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0 + sudo sysctl vm.swappiness=200 - name: Tune vm.mmap_rnd_bits for sanitizers shell: bash run: | diff --git a/.github/actions/docker_setup/action.yml b/.github/actions/docker_setup/action.yml new file mode 100644 index 000000000000..60c9a17519a6 --- /dev/null +++ b/.github/actions/docker_setup/action.yml @@ -0,0 +1,31 @@ +name: Docker setup +description: Setup docker +inputs: + test_name: + description: name of the test, used in determining ipv6 configs. + default: None + type: string +runs: + using: "composite" + steps: + - name: Docker IPv6 configuration + shell: bash + env: + ipv6_subnet: ${{ contains(inputs.test_name, 'Integration') && '2001:db8:1::/64' || '2001:3984:3989::/64' }} + run: | + # make sure docker uses proper IPv6 config + sudo touch /etc/docker/daemon.json + sudo chown ubuntu:ubuntu /etc/docker/daemon.json + sudo cat < /etc/docker/daemon.json + { + "ipv6": true, + "fixed-cidr-v6": "${{ env.ipv6_subnet }}" + } + EOT + sudo chown root:root /etc/docker/daemon.json + sudo systemctl restart docker + sudo systemctl status docker + - name: Docker info + shell: bash + run: | + docker info diff --git a/.github/retry.sh b/.github/retry.sh new file mode 100755 index 000000000000..566c2cf11315 --- /dev/null +++ b/.github/retry.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Execute command until exitcode is 0 or +# maximum number of retries is reached +# Example: +# ./retry +retries=$1 +delay=$2 +command="${@:3}" +exitcode=0 +try=0 +until [ "$try" -ge $retries ] +do + echo "$command" + eval "$command" + exitcode=$? + if [ $exitcode -eq 0 ]; then + break + fi + try=$((try+1)) + sleep $2 +done +exit $exitcode diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 15e071420254..ad2e805f477a 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -22,7 +22,7 @@ jobs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version @@ -164,7 +164,7 @@ jobs: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f - name: Download reports run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds @@ -194,7 +194,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Install packages (aarch64) - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce data: ${{ needs.RunConfig.outputs.data }} run_command: | python3 install_check.py "$CHECK_NAME" @@ -229,7 +229,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Stress test (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 data: ${{ needs.RunConfig.outputs.data }} ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -240,7 +240,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Integration tests (asan, old analyzer) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsTsan: needs: [RunConfig, BuilderDebTsan] @@ -248,7 +248,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Integration tests (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 data: ${{ needs.RunConfig.outputs.data }} FinishCheck: if: ${{ !cancelled() }} @@ -265,7 +265,7 @@ jobs: runs-on: [self-hosted, style-checker] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true - name: Finish label diff --git a/.github/workflows/cherry_pick.yml b/.github/workflows/cherry_pick.yml index 315673d4abcc..8e5191eb33cc 100644 --- a/.github/workflows/cherry_pick.yml +++ b/.github/workflows/cherry_pick.yml @@ -28,7 +28,7 @@ jobs: REPO_TEAM=core EOF - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}} diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 8b7ba6e62f6c..84182ec03aff 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -34,7 +34,7 @@ jobs: runs-on: [self-hosted, release-maker] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}} fetch-depth: 0 diff --git a/.github/workflows/docker_test_images.yml b/.github/workflows/docker_test_images.yml index 2138420f378e..be54cdaf3ddf 100644 --- a/.github/workflows/docker_test_images.yml +++ b/.github/workflows/docker_test_images.yml @@ -10,16 +10,31 @@ name: Build docker images description: set latest tag for resulting multiarch manifest required: false type: boolean - default: false + default: false + secrets: + secret_envs: + description: if given, it's passed to the environments + required: false + AWS_SECRET_ACCESS_KEY: + description: the access key to the aws param store. + required: true + AWS_ACCESS_KEY_ID: + description: the access key id to the aws param store. + required: true + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} jobs: DockerBuildAarch64: - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none] if: | !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_aarch64) != '[]' steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: ref: ${{ fromJson(inputs.data).git_ref }} - name: Build images @@ -29,12 +44,12 @@ jobs: --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \ --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_aarch64) }}' DockerBuildAmd64: - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] if: | !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_amd64) != '[]' steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: ref: ${{ fromJson(inputs.data).git_ref }} - name: Build images @@ -45,12 +60,12 @@ jobs: --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_amd64) }}' DockerMultiArchManifest: needs: [DockerBuildAmd64, DockerBuildAarch64] - runs-on: [self-hosted, style-checker] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] if: | !failure() && !cancelled() && (toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]' || inputs.set_latest) steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: ref: ${{ fromJson(inputs.data).git_ref }} - name: Build images diff --git a/.github/workflows/jepsen.yml b/.github/workflows/jepsen.yml index 92e4ce10ade5..1ef44d5aa902 100644 --- a/.github/workflows/jepsen.yml +++ b/.github/workflows/jepsen.yml @@ -16,7 +16,7 @@ jobs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version @@ -43,7 +43,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: ClickHouse Keeper Jepsen - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce data: ${{ needs.RunConfig.outputs.data }} run_command: | python3 jepsen_check.py keeper @@ -53,7 +53,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: ClickHouse Server Jepsen - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce data: ${{ needs.RunConfig.outputs.data }} run_command: | python3 jepsen_check.py server @@ -63,7 +63,7 @@ jobs: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f - name: Check Workflow results if: ${{ !cancelled() }} run: | diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b76bbbbbdbe6..09a6c2dbc747 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,7 +16,7 @@ jobs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version @@ -118,7 +118,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Builds - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce data: ${{ needs.RunConfig.outputs.data }} FinishCheck: @@ -127,7 +127,7 @@ jobs: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f - name: Finish label run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml index 45ce81c2cafd..376c153faa33 100644 --- a/.github/workflows/merge_queue.yml +++ b/.github/workflows/merge_queue.yml @@ -15,7 +15,7 @@ jobs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get a version @@ -56,7 +56,7 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Style check - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce run_command: | python3 style_check.py data: ${{ needs.RunConfig.outputs.data }} @@ -99,7 +99,7 @@ jobs: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f - name: Check and set merge status if: ${{ needs.StyleCheck.result == 'success' }} run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1cea94e75002..670019aac796 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -16,7 +16,7 @@ jobs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version @@ -50,7 +50,7 @@ jobs: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f - name: Check Workflow results if: ${{ !cancelled() }} run: | diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index acd392978b63..aaec2037261c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -21,12 +21,12 @@ concurrency: jobs: RunConfig: - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce] outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get a version @@ -77,22 +77,17 @@ jobs: uses: ./.github/workflows/reusable_test.yml with: test_name: Style check - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce run_command: | - python3 style_check.py + python3 style_check.py --no-push data: ${{ needs.RunConfig.outputs.data }} - secrets: - secret_envs: | - ROBOT_CLICKHOUSE_SSH_KEY<> ${GITHUB_OUTPUT} + env: + input: ${{ inputs.runner_type }} + + Common: + strategy: + fail-fast: false + matrix: + SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, rbac, selects, session_timezone, ssl_server, tiered_storage, window_functions] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=${{ matrix.SUITE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} + + Alter: + strategy: + fail-fast: false + matrix: + ONLY: [replace, attach, move] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=alter + STORAGE=/${{ matrix.ONLY }}_partition + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u alter/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --only "/alter/${{ matrix.ONLY }} partition/*" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: alter-${{ matrix.ONLY }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} + + Benchmark: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=ontime_benchmark + STORAGE=/${{ matrix.STORAGE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/benchmark.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --storage ${{ matrix.STORAGE }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: benchmark-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + ClickHouseKeeperSSL: + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{runner.temp}}/reports_dir + SUITE=clickhouse_keeper + STORAGE=/ssl + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --ssl + --clickhouse-binary-path ${{ env.clickhouse_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ inputs.arch }}-ssl-artifacts + path: ${{ env.artifact_paths }} + + LDAP: + strategy: + fail-fast: false + matrix: + SUITE: [authentication, external_user_directory, role_mapping] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=ldap/${{ matrix.SUITE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ldap-${{ matrix.SUITE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + Parquet: + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=parquet + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + ParquetS3: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=parquet + STORAGE=${{ matrix.STORAGE}} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --storage ${{ matrix.STORAGE }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ env.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths }} + + S3: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, aws_s3, gcs, azure] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=s3 + STORAGE=/${{ matrix.STORAGE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --storage ${{ matrix.STORAGE }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} + --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --azure-account-name ${{ secrets.AZURE_ACCOUNT_NAME }} + --azure-storage-key ${{ secrets.AZURE_STORAGE_KEY }} + --azure-container ${{ secrets.AZURE_CONTAINER_NAME }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} + + TieredStorage: + strategy: + fail-fast: false + matrix: + STORAGE: [minio, s3amazon, s3gcs] + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout regression repo + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.commit }} + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + REPORTS_PATH=${{ runner.temp }}/reports_dir + SUITE=tiered_storage + STORAGE=/${{ matrix.STORAGE }} + EOF + - name: Download json reports + uses: actions/download-artifact@v4 + with: + path: ${{ env.REPORTS_PATH }} + name: build_report_package_${{ inputs.arch }} + - name: Rename reports + run: | + mv ${{ env.REPORTS_PATH }}/build_report_*.json ${{ env.REPORTS_PATH }}/build_report_package_${{ inputs.arch }}.json + - name: Setup + run: .github/setup.sh + - name: Get deb url + run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV + - name: Run ${{ env.SUITE }} suite + run: EXITCODE=0; + python3 + -u ${{ env.SUITE }}/regression.py + --clickhouse-binary-path ${{ env.clickhouse_path }} + --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} + --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} + --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ + --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} + --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} + --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} + --with-${{ matrix.STORAGE }} + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="$GITHUB_RUN_ID" job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh; + exit $EXITCODE + - name: Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh 1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts + path: ${{ env.artifact_paths}} diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 0a84d0931975..7a7cd617dbfa 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -4,32 +4,42 @@ name: ReleaseBranchCI env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} on: # yamllint disable-line rule:truthy + pull_request: + types: + - synchronize + - reopened + - opened + branches: + # Anything/24.8 (e.g customizations/24.8.x) + - '**/24.8*' + release: + types: + - published + - prereleased push: branches: - # 22.1 and 22.10 - - '2[1-9].[1-9][0-9]' - - '2[1-9].[1-9]' + - 'releases/24.8**' + workflow_dispatch: jobs: RunConfig: - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 - name: Debug Info uses: ./.github/actions/debug - - name: Labels check - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 run_check.py - name: Python unit tests run: | cd "$GITHUB_WORKSPACE/tests/ci" @@ -56,23 +66,26 @@ jobs: needs: [RunConfig] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/docker_test_images.yml + secrets: inherit with: data: ${{ needs.RunConfig.outputs.data }} CompatibilityCheckX86: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Compatibility check (release) - runner_type: style-checker + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} CompatibilityCheckAarch64: needs: [RunConfig, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Compatibility check (aarch64) - runner_type: style-checker + runner_type: altinity-on-demand, altinity-func-tester-aarch64 data: ${{ needs.RunConfig.outputs.data }} ######################################################################################### #################################### ORDINARY BUILDS #################################### @@ -81,6 +94,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_release checkout_depth: 0 @@ -91,6 +105,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_aarch64 checkout_depth: 0 @@ -101,6 +116,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_asan data: ${{ needs.RunConfig.outputs.data }} @@ -108,6 +124,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_ubsan data: ${{ needs.RunConfig.outputs.data }} @@ -115,6 +132,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_tsan data: ${{ needs.RunConfig.outputs.data }} @@ -122,6 +140,7 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_msan data: ${{ needs.RunConfig.outputs.data }} @@ -129,30 +148,11 @@ jobs: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_build.yml + secrets: inherit with: build_name: package_debug data: ${{ needs.RunConfig.outputs.data }} force: true - BuilderBinDarwin: - needs: [RunConfig, BuildDockers] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/reusable_build.yml - with: - build_name: binary_darwin - checkout_depth: 0 - data: ${{ needs.RunConfig.outputs.data }} - # always rebuild on release branches to be able to publish from any commit - force: true - BuilderBinDarwinAarch64: - needs: [RunConfig, BuildDockers] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/reusable_build.yml - with: - build_name: binary_darwin_aarch64 - checkout_depth: 0 - data: ${{ needs.RunConfig.outputs.data }} - # always rebuild on release branches to be able to publish from any commit - force: true ############################################################################################ ##################################### Docker images ####################################### ############################################################################################ @@ -160,17 +160,19 @@ jobs: needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Docker server image - runner_type: style-checker + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} DockerKeeperImage: needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Docker keeper image - runner_type: style-checker + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} ############################################################################################ ##################################### BUILD REPORTER ####################################### @@ -178,29 +180,29 @@ jobs: Builds_Report: # run report check for failed builds to indicate the CI error if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }} - needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64, BuilderDebAsan, BuilderDebUBsan, BuilderDebMsan, BuilderDebTsan, BuilderDebDebug, BuilderBinDarwin, BuilderBinDarwinAarch64] - runs-on: [self-hosted, style-checker-aarch64] + needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64, BuilderDebAsan, BuilderDebUBsan, BuilderDebMsan, BuilderDebTsan, BuilderDebDebug] + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 - name: Download reports run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds - name: Builds report run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_msan package_ubsan package_tsan package_debug binary_darwin binary_darwin_aarch64 + python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_msan package_ubsan package_tsan package_debug - name: Set status + # NOTE(vnemkov): generate and upload the report even if previous step failed + if: success() || failure() run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --post --job-name Builds MarkReleaseReady: if: ${{ !failure() && !cancelled() }} needs: - - BuilderBinDarwin - - BuilderBinDarwinAarch64 - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, style-checker-aarch64] + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] steps: - name: Debug run: | @@ -218,7 +220,7 @@ jobs: run: exit 1 - name: Check out repository code if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 - name: Mark Commit Release Ready if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} run: | @@ -231,9 +233,10 @@ jobs: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Install packages (release) - runner_type: style-checker + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} run_command: | python3 install_check.py "$CHECK_NAME" @@ -241,9 +244,10 @@ jobs: needs: [RunConfig, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Install packages (aarch64) - runner_type: style-checker-aarch64 + runner_type: altinity-on-demand, altinity-func-tester-aarch64 data: ${{ needs.RunConfig.outputs.data }} run_command: | python3 install_check.py "$CHECK_NAME" @@ -254,57 +258,64 @@ jobs: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (release) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestAarch64: needs: [RunConfig, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (aarch64) - runner_type: func-tester-aarch64 + runner_type: altinity-on-demand, altinity-func-tester-aarch64 data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestAsan: needs: [RunConfig, BuilderDebAsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (asan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestTsan: needs: [RunConfig, BuilderDebTsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestMsan: needs: [RunConfig, BuilderDebMsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (msan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestUBsan: needs: [RunConfig, BuilderDebUBsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (ubsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestDebug: needs: [RunConfig, BuilderDebDebug] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateless tests (debug) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### @@ -313,57 +324,64 @@ jobs: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (release) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestAarch64: needs: [RunConfig, BuilderDebAarch64] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (aarch64) - runner_type: func-tester-aarch64 + runner_type: altinity-on-demand, altinity-func-tester-aarch64 data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestAsan: needs: [RunConfig, BuilderDebAsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (asan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestTsan: needs: [RunConfig, BuilderDebTsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestMsan: needs: [RunConfig, BuilderDebMsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (msan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestUBsan: needs: [RunConfig, BuilderDebUBsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (ubsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatefulTestDebug: needs: [RunConfig, BuilderDebDebug] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stateful tests (debug) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} ############################################################################################## ######################################### STRESS TESTS ####################################### @@ -372,41 +390,46 @@ jobs: needs: [RunConfig, BuilderDebAsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stress test (asan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} StressTestTsan: needs: [RunConfig, BuilderDebTsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stress test (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} StressTestMsan: needs: [RunConfig, BuilderDebMsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stress test (msan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} StressTestUBsan: needs: [RunConfig, BuilderDebUBsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stress test (ubsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} StressTestDebug: needs: [RunConfig, BuilderDebDebug] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Stress test (debug) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} ############################################################################################# ############################# INTEGRATION TESTS ############################################# @@ -415,33 +438,137 @@ jobs: needs: [RunConfig, BuilderDebAsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Integration tests (asan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsAnalyzerAsan: needs: [RunConfig, BuilderDebAsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Integration tests (asan, old analyzer) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsTsan: needs: [RunConfig, BuilderDebTsan] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Integration tests (tsan) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsRelease: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml + secrets: inherit with: test_name: Integration tests (release) - runner_type: func-tester + runner_type: altinity-on-demand, altinity-func-tester + data: ${{ needs.RunConfig.outputs.data }} + IntegrationTestAarch64: + needs: [RunConfig, BuilderDebAarch64] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + secrets: inherit + with: + test_name: Integration tests (aarch64) + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none + data: ${{ needs.RunConfig.outputs.data }} +############################################################################################# +####################################### AST FUZZERS ######################################### +############################################################################################# + ASTFuzzerAsan: + needs: [RunConfig, BuilderDebAsan] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + secrets: inherit + with: + test_name: AST fuzzer (asan) + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + data: ${{ needs.RunConfig.outputs.data }} + ASTFuzzerTsan: + needs: [RunConfig, BuilderDebTsan] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + secrets: inherit + with: + test_name: AST fuzzer (tsan) + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + data: ${{ needs.RunConfig.outputs.data }} + ASTFuzzerMsan: + needs: [RunConfig, BuilderDebMsan] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + secrets: inherit + with: + test_name: AST fuzzer (msan) + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + data: ${{ needs.RunConfig.outputs.data }} + ASTFuzzerUBsan: + needs: [RunConfig, BuilderDebUBsan] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + secrets: inherit + with: + test_name: AST fuzzer (ubsan) + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + data: ${{ needs.RunConfig.outputs.data }} + ASTFuzzerDebug: + needs: [RunConfig, BuilderDebDebug] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_test.yml + secrets: inherit + with: + test_name: AST fuzzer (debug) + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + data: ${{ needs.RunConfig.outputs.data }} +############################################################################################# +##################################### REGRESSION TESTS ###################################### +############################################################################################# + RegressionTestsRelease: + needs: [BuilderDebRelease] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/regression.yml + secrets: inherit + with: + runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression + commit: 18c56fe46fae827748c557e9ebda0599c11b0a55 + arch: release + build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout_minutes: 300 + RegressionTestsAarch64: + needs: [BuilderDebAarch64] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/regression.yml + secrets: inherit + with: + runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression + commit: 18c56fe46fae827748c557e9ebda0599c11b0a55 + arch: aarch64 + build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout_minutes: 300 + SignRelease: + needs: [RunConfig, BuilderDebRelease] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_sign.yml + secrets: inherit + with: + test_name: Sign release + runner_type: altinity-on-demand, altinity-func-tester + data: ${{ needs.RunConfig.outputs.data }} + SignAarch64: + needs: [RunConfig, BuilderDebAarch64] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/reusable_sign.yml + secrets: inherit + with: + test_name: Sign aarch64 + runner_type: altinity-on-demand, altinity-func-tester data: ${{ needs.RunConfig.outputs.data }} FinishCheck: if: ${{ !cancelled() }} @@ -475,10 +602,13 @@ jobs: - IntegrationTestsRelease - CompatibilityCheckX86 - CompatibilityCheckAarch64 - runs-on: [self-hosted, style-checker] + - RegressionTestsRelease + - RegressionTestsAarch64 + - SignRelease + runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: clear-repository: true - name: Finish label diff --git a/.github/workflows/reusable_build.yml b/.github/workflows/reusable_build.yml index 5e254d785ecc..c64d0aaec500 100644 --- a/.github/workflows/reusable_build.yml +++ b/.github/workflows/reusable_build.yml @@ -4,6 +4,9 @@ env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} name: Build ClickHouse 'on': @@ -37,6 +40,12 @@ name: Build ClickHouse secret_envs: description: if given, it's passed to the environments required: false + AWS_SECRET_ACCESS_KEY: + description: the access key to the aws param store. + required: true + AWS_ACCESS_KEY_ID: + description: the access key id to the aws param store. + required: true jobs: Build: @@ -44,10 +53,10 @@ jobs: if: ${{ contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.build_name) || inputs.force }} env: GITHUB_JOB_OVERRIDDEN: Build-${{inputs.build_name}} - runs-on: [self-hosted, '${{inputs.runner_type}}'] + runs-on: [self-hosted, altinity-type-ccx53, altinity-on-demand, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: clear-repository: true ref: ${{ fromJson(inputs.data).git_ref }} @@ -98,6 +107,11 @@ jobs: if: ${{ !cancelled() }} run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.build_name}}' + - name: Upload json report + uses: actions/upload-artifact@v4 + with: + path: ${{ env.TEMP_PATH }}/build_report_*.json + name: build_report_${{inputs.build_name}} - name: Clean if: always() uses: ./.github/actions/clean diff --git a/.github/workflows/reusable_sign.yml b/.github/workflows/reusable_sign.yml new file mode 100644 index 000000000000..2bd8ae430fa5 --- /dev/null +++ b/.github/workflows/reusable_sign.yml @@ -0,0 +1,164 @@ +### For the pure soul wishes to move it to another place +# https://github.com/orgs/community/discussions/9050 + +name: Testing workflow +'on': + workflow_call: + inputs: + test_name: + description: the value of test type from tests/ci/ci_config.py, ends up as $CHECK_NAME ENV + required: true + type: string + runner_type: + description: the label of runner to use + required: true + type: string + run_command: + description: the command to launch the check + default: "" + required: false + type: string + checkout_depth: + description: the value of the git shallow checkout + required: false + type: number + default: 1 + submodules: + description: if the submodules should be checked out + required: false + type: boolean + default: false + additional_envs: + description: additional ENV variables to setup the job + type: string + data: + description: ci data + type: string + required: true + working-directory: + description: sets custom working directory + type: string + default: "$GITHUB_WORKSPACE/tests/ci" + secrets: + secret_envs: + description: if given, it's passed to the environments + required: false + AWS_SECRET_ACCESS_KEY: + description: the access key to the aws param store. + required: true + AWS_ACCESS_KEY_ID: + description: the access key id to the aws param store. + required: true + GPG_BINARY_SIGNING_KEY: + description: gpg signing key for packages. + required: true + GPG_BINARY_SIGNING_PASSPHRASE: + description: gpg signing key passphrase. + required: true + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + CHECK_NAME: ${{inputs.test_name}} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + +jobs: + runner_labels_setup: + name: Compute proper runner labels for the rest of the jobs + runs-on: ubuntu-latest + outputs: + runner_labels: ${{ steps.setVariables.outputs.runner_labels }} + steps: + - id: setVariables + name: Prepare runner_labels variables for the later steps + run: | + + # Prepend self-hosted + input="self-hosted, ${input}" + + # Remove all whitespace + input="$(echo ${input} | tr -d [:space:])" + # Make something like a JSON array from comma-separated list + input="[ '${input//\,/\'\, \'}' ]" + + echo "runner_labels=$input" >> ${GITHUB_OUTPUT} + env: + input: ${{ inputs.runner_type }} + + Test: + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} + name: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} + env: + GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} + strategy: + fail-fast: false # we always wait for entire matrix + matrix: + batch: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].batches }} + steps: + - name: Check out repository code + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 + with: + clear-repository: true + ref: ${{ fromJson(inputs.data).git_ref }} + submodules: ${{inputs.submodules}} + fetch-depth: ${{inputs.checkout_depth}} + filter: tree:0 + - name: Set build envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + CHECK_NAME=${{ inputs.test_name }} + ${{inputs.additional_envs}} + ${{secrets.secret_envs}} + DOCKER_TAG< 1 }} + run: | + cat >> "$GITHUB_ENV" << 'EOF' + RUN_BY_HASH_NUM=${{matrix.batch}} + RUN_BY_HASH_TOTAL=${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches }} + EOF + - name: Pre run + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.test_name}}' + - name: Sign release + env: + GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }} + GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }} + run: | + cd "${{ inputs.working-directory }}" + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" \ + --infile ${{ toJson(inputs.data) }} \ + --job-name '${{inputs.test_name}}' \ + --run \ + --force \ + --run-command '''python3 sign_release.py''' + - name: Post run + if: ${{ !cancelled() }} + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.test_name}}' + - name: Mark as done + if: ${{ !cancelled() }} + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --mark-success --job-name '${{inputs.test_name}}' --batch ${{matrix.batch}} + - name: Upload signed hashes + uses: actions/upload-artifact@v4 + with: + name: ${{inputs.test_name}} signed-hashes + path: ${{ env.TEMP_PATH }}/*.gpg + - name: Clean + if: always() + uses: ./.github/actions/clean diff --git a/.github/workflows/reusable_simple_job.yml b/.github/workflows/reusable_simple_job.yml index 7df98d96f790..9ecb28109450 100644 --- a/.github/workflows/reusable_simple_job.yml +++ b/.github/workflows/reusable_simple_job.yml @@ -63,7 +63,7 @@ jobs: GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: clear-repository: true ref: ${{ inputs.git_ref }} diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml index ca6df5df14e0..4867cab03480 100644 --- a/.github/workflows/reusable_test.yml +++ b/.github/workflows/reusable_test.yml @@ -43,17 +43,47 @@ name: Testing workflow secret_envs: description: if given, it's passed to the environments required: false - + AWS_SECRET_ACCESS_KEY: + description: the access key to the aws param store. + required: true + AWS_ACCESS_KEY_ID: + description: the access key id to the aws param store. + required: true env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 CHECK_NAME: ${{inputs.test_name}} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} jobs: + runner_labels_setup: + name: Compute proper runner labels for the rest of the jobs + runs-on: ubuntu-latest + outputs: + runner_labels: ${{ steps.setVariables.outputs.runner_labels }} + steps: + - id: setVariables + name: Prepare runner_labels variables for the later steps + run: | + + # Prepend self-hosted + input="self-hosted, ${input}" + + # Remove all whitespace + input="$(echo ${input} | tr -d [:space:])" + # Make something like a JSON array from comma-separated list + input="[ '${input//\,/\'\, \'}' ]" + + echo "runner_labels=$input" >> ${GITHUB_OUTPUT} + env: + input: ${{ inputs.runner_type }} + Test: - runs-on: [self-hosted, '${{inputs.runner_type}}'] - if: ${{ !failure() && !cancelled() && contains(fromJson(inputs.data).jobs_data.jobs_to_do, inputs.test_name) }} + needs: [runner_labels_setup] + runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} name: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} env: GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}}${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 && format('-{0}',matrix.batch) || '' }} @@ -63,7 +93,7 @@ jobs: batch: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].batches }} steps: - name: Check out repository code - uses: ClickHouse/checkout@v1 + uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 with: clear-repository: true ref: ${{ fromJson(inputs.data).git_ref }} @@ -84,6 +114,10 @@ jobs: uses: ./.github/actions/common_setup with: job_type: test + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: ${{ inputs.test_name }} - name: Setup batch if: ${{ fromJson(inputs.data).jobs_data.jobs_params[inputs.test_name].num_batches > 1 }} run: | @@ -101,6 +135,7 @@ jobs: --infile ${{ toJson(inputs.data) }} \ --job-name '${{inputs.test_name}}' \ --run \ + --force \ --run-command '''${{inputs.run_command}}''' # shellcheck disable=SC2319 echo "JOB_EXIT_CODE=$?" >> "$GITHUB_ENV" diff --git a/README.md b/README.md index 2120a4d1211a..5f41b67e5ed8 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,60 @@
-[![Website](https://img.shields.io/website?up_message=AVAILABLE&down_message=DOWN&url=https%3A%2F%2Fclickhouse.com&style=for-the-badge)](https://clickhouse.com) +[![Website](https://img.shields.io/website?up_message=AVAILABLE&down_message=DOWN&url=https://docs.altinity.com/altinitystablebuilds&style=for-the-badge)](https://docs.altinity.com/altinitystablebuilds/) [![Apache 2.0 License](https://img.shields.io/badge/license-Apache%202.0-blueviolet?style=for-the-badge)](https://www.apache.org/licenses/LICENSE-2.0) - - - The ClickHouse company logo. + + + Altinity company logo -

ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time.

+

Altinity Stable Builds®

-## How To Install (Linux, macOS, FreeBSD) -``` -curl https://clickhouse.com/ | sh -``` +**Altinity Stable Builds** are releases of ClickHouse® that undergo rigorous testing to verify they are secure and ready for production use. Among other things, they are: -## Useful Links +* Supported for three years +* Validated against client libraries and visualization tools +* Tested for cloud use, including Kubernetes +* 100% open source and 100% compatible with ClickHouse upstream builds +* Available in FIPS-compatible versions + +**We encourage you to use Altinity Stable Builds whether you're an Altinity Support customer or not.** + +## Acknowledgement + +We at Altinity, Inc. are thoroughly grateful to the worldwide ClickHouse community, including the core committers who make ClickHouse the world's best analytics database. -* [Official website](https://clickhouse.com/) has a quick high-level overview of ClickHouse on the main page. -* [ClickHouse Cloud](https://clickhouse.cloud) ClickHouse as a service, built by the creators and maintainers. -* [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query a small ClickHouse cluster. -* [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information. -* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format. -* [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time. -* [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events. -* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev. -* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any. +## What should I do if I find a bug in an Altinity Stable Build? -## Monthly Release & Community Call +ClickHouse’s thousands of core features are all well-tested and stable. To maintain that stability, Altinity Stable Builds are built with a CI system that runs tens of thousands of tests against every commit. But of course, things can always go wrong. If that happens, let us know! **We stand behind our work.** -Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know. +### If you're an Altinity customer: -* [v24.8 Community Call](https://clickhouse.com/company/events/v24-8-community-release-call) - August 29 +1. [Contact Altinity support](https://docs.altinity.com/support/) to file an issue. -## Upcoming Events +### If you're not an Altinity customer: + +1. Try to upgrade to the latest bugfix release. If you’re using v23.8.8 but you know that v23.8.11.29 exists, start by upgrading to the bugfix. Upgrades to the latest maintenance releases are smooth and safe. +2. Look for similar issues in the [Altinity/ClickHouse](https://github.com/Altinity/ClickHouse/issues) or [ClickHouse/ClickHouse](https://github.com/ClickHouse/ClickHouse/issues) repos; it's possible the problem has been logged and a fix is on the way. +3. If you can reproduce the bug, try to isolate it. For example, remove pieces of a failing query one by one, creating the simplest scenario where the error still occurs. Creating [a minimal reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) is a huge step towards a solution. +4. [File an issue](https://github.com/Altinity/ClickHouse/issues/new/choose) in the Altinity/ClickHouse repo. + +## Useful Links -Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `` clickhouse `` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc. +* [Release notes](https://docs.altinity.com/releasenotes/altinity-stable-release-notes/) - Complete details on the changes and fixes in each Altinity Stable Build release +* [Builds page](https://builds.altinity.cloud/) - Download and installation instructions for Altinity Stable Builds +* [Dockerhub page](https://hub.docker.com/r/altinity/clickhouse-server) - Home of the Altinity Stable Build container images +* [Knowledge base](https://kb.altinity.com/) - Insight, knowledge, and advice from the Altinity Engineering and Support teams +* [Documentation](https://docs.altinity.com/altinitystablebuilds/) - A list of current releases and their lifecycles +* [Altinity support for ClickHouse](https://altinity.com/clickhouse-support/) - The best ClickHouse support in the industry, delivered by the most knowledgeable ClickHouse team in the industry +* [Altinity administrator training for ClickHouse](https://altinity.com/clickhouse-training/) - Understand how ClickHouse works, not just how to use it +* [Altinity blog](https://altinity.com/blog/) - The latest news on Altinity's ClickHouse-related projects, release notes, tutorials, and more +* [Altinity YouTube channel](https://www.youtube.com/@AltinityB) - ClickHouse tutorials, webinars, conference presentations, and other useful things +* [Altinity Slack channel](https://altinitydbworkspace.slack.com/join/shared_invite/zt-1togw9b4g-N0ZOXQyEyPCBh_7IEHUjdw#/shared-invite/email) - The Altinity Slack channel -* MORE COMING SOON! +
-## Recent Recordings -* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments" -* **Recording available**: [**v24.4 Release Call**](https://www.youtube.com/watch?v=dtUqgcfOGmE) All the features of 24.4, one convenient video! Watch it now! - - ## Interested in joining ClickHouse and making it your full-time job? - -We are a globally diverse and distributed team, united behind a common goal of creating industry-leading, real-time analytics. Here, you will have an opportunity to solve some of the most cutting-edge technical challenges and have direct ownership of your work and vision. If you are a contributor by nature, a thinker and a doer - we’ll definitely click! - -Check out our **current openings** here: https://clickhouse.com/company/careers - -Can't find what you are looking for, but want to let us know you are interested in joining ClickHouse? Email careers@clickhouse.com! +*©2024 Altinity Inc. All rights reserved. Altinity®, Altinity.Cloud®, and Altinity Stable Builds® are registered trademarks of Altinity, Inc. ClickHouse® is a registered trademark of ClickHouse, Inc.* diff --git a/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h b/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h index acc79ec92b23..6e44d9f45b7b 100644 --- a/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h +++ b/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h @@ -188,8 +188,9 @@ namespace Crypto pFile = fopen(keyFile.c_str(), "r"); if (pFile) { - pem_password_cb * pCB = pass.empty() ? (pem_password_cb *)0 : &passCB; - void * pPassword = pass.empty() ? (void *)0 : (void *)pass.c_str(); + pem_password_cb * pCB = &passCB; + static constexpr char * no_password = ""; + void * pPassword = pass.empty() ? (void *)no_password : (void *)pass.c_str(); if (readFunc(pFile, &pKey, pCB, pPassword)) { fclose(pFile); @@ -225,6 +226,13 @@ namespace Crypto error: if (pFile) fclose(pFile); + if (*ppKey) + { + if constexpr (std::is_same_v) + EVP_PKEY_free(*ppKey); + else + EC_KEY_free(*ppKey); + } throw OpenSSLException("EVPKey::loadKey(string)"); } @@ -286,6 +294,13 @@ namespace Crypto error: if (pBIO) BIO_free(pBIO); + if (*ppKey) + { + if constexpr (std::is_same_v) + EVP_PKEY_free(*ppKey); + else + EC_KEY_free(*ppKey); + } throw OpenSSLException("EVPKey::loadKey(stream)"); } diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h index c19eecf5c737..2c56875835e7 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h @@ -248,6 +248,9 @@ namespace Net SSL_CTX * sslContext() const; /// Returns the underlying OpenSSL SSL Context object. + SSL_CTX * takeSslContext(); + /// Takes ownership of the underlying OpenSSL SSL Context object. + Usage usage() const; /// Returns whether the context is for use by a client or by a server /// and whether TLSv1 is required. @@ -401,6 +404,13 @@ namespace Net return _pSSLContext; } + inline SSL_CTX * Context::takeSslContext() + { + auto * result = _pSSLContext; + _pSSLContext = nullptr; + return result; + } + inline bool Context::extendedCertificateVerificationEnabled() const { diff --git a/base/poco/NetSSL_OpenSSL/src/Context.cpp b/base/poco/NetSSL_OpenSSL/src/Context.cpp index da1c121286b6..69c88eef63ac 100644 --- a/base/poco/NetSSL_OpenSSL/src/Context.cpp +++ b/base/poco/NetSSL_OpenSSL/src/Context.cpp @@ -106,6 +106,11 @@ Context::Context( Context::~Context() { + if (_pSSLContext == nullptr) + { + return; + } + try { SSL_CTX_free(_pSSLContext); diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 740c72f3a364..0f4c7f442055 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -6,7 +6,15 @@ SET(VERSION_REVISION 54499) SET(VERSION_MAJOR 24) SET(VERSION_MINOR 8) SET(VERSION_PATCH 11) + SET(VERSION_GITHASH cd10bd2f1fa142a3661d819922c86638b1ba957e) -SET(VERSION_DESCRIBE v24.8.11.1-lts) -SET(VERSION_STRING 24.8.11.1) + +#1000 for altinitystable candidates +#2000 for altinityedge candidates +SET(VERSION_TWEAK 51000) +SET(VERSION_FLAVOUR altinitystable) + +SET(VERSION_DESCRIBE v24.8.11.51000.altinitystable) +SET(VERSION_STRING 24.8.11.51000.altinitystable) + # end of autochange diff --git a/cmake/version.cmake b/cmake/version.cmake index 9ca21556f4d4..b008c989c0b0 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -3,9 +3,10 @@ include(${PROJECT_SOURCE_DIR}/cmake/autogenerated_versions.txt) set(VERSION_EXTRA "" CACHE STRING "") set(VERSION_TWEAK "" CACHE STRING "") -if (VERSION_TWEAK) - string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_TWEAK}) -endif () +# NOTE(vnemkov): we rely on VERSION_TWEAK portion to be already present in VERSION_STRING +# if (VERSION_TWEAK) +# string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_TWEAK}) +# endif () if (VERSION_EXTRA) string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_EXTRA}) @@ -19,5 +20,5 @@ set (VERSION_STRING_SHORT "${VERSION_MAJOR}.${VERSION_MINOR}") math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000") if(CLICKHOUSE_OFFICIAL_BUILD) - set(VERSION_OFFICIAL " (official build)") + set(VERSION_OFFICIAL " (altinity build)") endif() diff --git a/contrib/NuRaft b/contrib/NuRaft index c2b0811f164a..c11f7fce6873 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit c2b0811f164a7948208489562dab4f186eb305ce +Subproject commit c11f7fce68737cdc67a1d61678b2717d617ebb5a diff --git a/docker/docs/builder/Dockerfile b/docker/docs/builder/Dockerfile index b7b706a8a5ce..a61b44b60798 100644 --- a/docker/docs/builder/Dockerfile +++ b/docker/docs/builder/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/docs-builder . +# docker build -t altinityinfra/docs-builder . # nodejs 17 prefers ipv6 and is broken in our environment FROM node:16-alpine diff --git a/docker/images.json b/docker/images.json index 716b76ee217d..80221cb6bb1c 100644 --- a/docker/images.json +++ b/docker/images.json @@ -1,125 +1,125 @@ { "docker/packager/binary-builder": { - "name": "clickhouse/binary-builder", + "name": "altinityinfra/binary-builder", "dependent": [] }, "docker/packager/cctools": { - "name": "clickhouse/cctools", + "name": "altinityinfra/cctools", "dependent": [] }, "docker/test/compatibility/centos": { - "name": "clickhouse/test-old-centos", + "name": "altinityinfra/test-old-centos", "dependent": [] }, "docker/test/compatibility/ubuntu": { - "name": "clickhouse/test-old-ubuntu", + "name": "altinityinfra/test-old-ubuntu", "dependent": [] }, "docker/test/integration/base": { - "name": "clickhouse/integration-test", + "name": "altinityinfra/integration-test", "dependent": [] }, "docker/test/fuzzer": { - "name": "clickhouse/fuzzer", + "name": "altinityinfra/fuzzer", "dependent": [] }, "docker/test/libfuzzer": { - "name": "clickhouse/libfuzzer", + "name": "altinityinfra/libfuzzer", "dependent": [] }, "docker/test/performance-comparison": { - "name": "clickhouse/performance-comparison", + "name": "altinityinfra/performance-comparison", "dependent": [] }, "docker/test/util": { - "name": "clickhouse/test-util", + "name": "altinityinfra/test-util", "dependent": [ "docker/test/base", "docker/test/fasttest" ] }, "docker/test/stateless": { - "name": "clickhouse/stateless-test", + "name": "altinityinfra/stateless-test", "dependent": [ "docker/test/stateful" ] }, "docker/test/stateful": { - "name": "clickhouse/stateful-test", + "name": "altinityinfra/stateful-test", "dependent": [ "docker/test/stress", "docker/test/upgrade" ] }, "docker/test/unit": { - "name": "clickhouse/unit-test", + "name": "altinityinfra/unit-test", "dependent": [] }, "docker/test/stress": { - "name": "clickhouse/stress-test", + "name": "altinityinfra/stress-test", "dependent": [] }, "docker/test/upgrade": { - "name": "clickhouse/upgrade-check", + "name": "altinityinfra/upgrade-check", "dependent": [] }, "docker/test/integration/runner": { - "name": "clickhouse/integration-tests-runner", + "name": "altinityinfra/integration-tests-runner", "dependent": [] }, "docker/test/fasttest": { - "name": "clickhouse/fasttest", + "name": "altinityinfra/fasttest", "dependent": [ "docker/packager/binary-builder" ] }, "docker/test/style": { - "name": "clickhouse/style-test", + "name": "altinityinfra/style-test", "dependent": [] }, "docker/test/integration/s3_proxy": { - "name": "clickhouse/s3-proxy", + "name": "altinityinfra/s3-proxy", "dependent": [] }, "docker/test/integration/resolver": { - "name": "clickhouse/python-bottle", + "name": "altinityinfra/python-bottle", "dependent": [] }, "docker/test/integration/helper_container": { - "name": "clickhouse/integration-helper", + "name": "altinityinfra/integration-helper", "dependent": [] }, "docker/test/integration/mysql_golang_client": { - "name": "clickhouse/mysql-golang-client", + "name": "altinityinfra/mysql-golang-client", "dependent": [] }, "docker/test/integration/dotnet_client": { - "name": "clickhouse/dotnet-client", + "name": "altinityinfra/dotnet-client", "dependent": [] }, "docker/test/integration/mysql_java_client": { - "name": "clickhouse/mysql-java-client", + "name": "altinityinfra/mysql-java-client", "dependent": [] }, "docker/test/integration/mysql_js_client": { - "name": "clickhouse/mysql-js-client", + "name": "altinityinfra/mysql-js-client", "dependent": [] }, "docker/test/integration/mysql_php_client": { - "name": "clickhouse/mysql-php-client", + "name": "altinityinfra/mysql-php-client", "dependent": [] }, "docker/test/integration/postgresql_java_client": { - "name": "clickhouse/postgresql-java-client", + "name": "altinityinfra/postgresql-java-client", "dependent": [] }, "docker/test/integration/kerberos_kdc": { "only_amd64": true, - "name": "clickhouse/kerberos-kdc", + "name": "altinityinfra/kerberos-kdc", "dependent": [] }, "docker/test/base": { - "name": "clickhouse/test-base", + "name": "altinityinfra/test-base", "dependent": [ "docker/test/clickbench", "docker/test/fuzzer", @@ -133,49 +133,44 @@ "docker/test/unit" ] }, - "docker/test/integration/kerberized_hadoop": { - "only_amd64": true, - "name": "clickhouse/kerberized-hadoop", - "dependent": [] - }, "docker/test/sqlancer": { - "name": "clickhouse/sqlancer-test", + "name": "altinityinfra/sqlancer-test", "dependent": [] }, "docker/test/keeper-jepsen": { - "name": "clickhouse/keeper-jepsen-test", + "name": "altinityinfra/keeper-jepsen-test", "dependent": [] }, "docker/test/server-jepsen": { - "name": "clickhouse/server-jepsen-test", + "name": "altinityinfra/server-jepsen-test", "dependent": [] }, "docker/test/clickbench": { - "name": "clickhouse/clickbench", + "name": "altinityinfra/clickbench", "dependent": [] }, "docker/test/install/deb": { - "name": "clickhouse/install-deb-test", + "name": "altinityinfra/install-deb-test", "dependent": [] }, "docker/test/install/rpm": { - "name": "clickhouse/install-rpm-test", + "name": "altinityinfra/install-rpm-test", "dependent": [] }, "docker/docs/builder": { - "name": "clickhouse/docs-builder", + "name": "altinityinfra/docs-builder", "dependent": [] }, "docker/test/sqllogic": { - "name": "clickhouse/sqllogic-test", + "name": "altinityinfra/sqllogic-test", "dependent": [] }, "docker/test/sqltest": { - "name": "clickhouse/sqltest", + "name": "altinityinfra/sqltest", "dependent": [] }, "docker/test/integration/nginx_dav": { - "name": "clickhouse/nginx-dav", + "name": "altinityinfra/nginx-dav", "dependent": [] } } diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index a44664259fbc..646ef74164ba 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -12,7 +12,9 @@ RUN arch=${TARGETARCH:-amd64} \ && ln -s "${rarch}-linux-gnu" /lib/linux-gnu -FROM alpine +# As of Nov 3rd 2024, alpine:3.20.3 has 1 "unknown"-type CVE: +# * https://security.alpinelinux.org/vuln/CVE-2024-9143 +FROM alpine:3.20.3 ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ diff --git a/docker/packager/binary-builder/Dockerfile b/docker/packager/binary-builder/Dockerfile index 7d6acdcd856c..2f6778ba2112 100644 --- a/docker/packager/binary-builder/Dockerfile +++ b/docker/packager/binary-builder/Dockerfile @@ -1,11 +1,12 @@ -# docker build -t clickhouse/binary-builder . +# docker build -t altinityinfra/binary-builder . ARG FROM_TAG=latest -FROM clickhouse/fasttest:$FROM_TAG +FROM altinityinfra/fasttest:$FROM_TAG ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} # If the cctools is updated, then first build it in the CI, then update here in a different commit -COPY --from=clickhouse/cctools:d9e3596e706b /cctools /cctools +# NOTE: a9e509c9bbc0 created from commit 1bc0e6ee583d49a472fd01f0e9960a11bf240c6c +COPY --from=altinityinfra/cctools:a9e509c9bbc0 /cctools /cctools # Rust toolchain and libraries ENV RUSTUP_HOME=/rust/rustup diff --git a/docker/packager/cctools/Dockerfile b/docker/packager/cctools/Dockerfile index 570a42d42d51..3555bf7c428b 100644 --- a/docker/packager/cctools/Dockerfile +++ b/docker/packager/cctools/Dockerfile @@ -1,10 +1,10 @@ -# docker build -t clickhouse/cctools . +# docker build -t altinityinfra/cctools . -# This is a hack to significantly reduce the build time of the clickhouse/binary-builder +# This is a hack to significantly reduce the build time of the altinityinfra/binary-builder # It's based on the assumption that we don't care of the cctools version so much -# It event does not depend on the clickhouse/fasttest in the `docker/images.json` +# It event does not depend on the altinityinfra/fasttest in the `docker/images.json` ARG FROM_TAG=latest -FROM clickhouse/fasttest:$FROM_TAG as builder +FROM altinityinfra/fasttest:$FROM_TAG as builder ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} diff --git a/docker/packager/packager b/docker/packager/packager index da4af7fc1bee..7975b6a03aae 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -10,7 +10,7 @@ from typing import List, Optional SCRIPT_PATH = Path(__file__).absolute() IMAGE_TYPE = "binary-builder" -IMAGE_NAME = f"clickhouse/{IMAGE_TYPE}" +IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}" class BuildException(Exception): @@ -130,9 +130,11 @@ def parse_env_variables( sanitizer: str, package_type: str, cache: str, + s3_access_key_id: str, s3_bucket: str, s3_directory: str, s3_rw_access: bool, + s3_secret_access_key: str, clang_tidy: bool, version: str, official: bool, @@ -321,6 +323,10 @@ def parse_env_variables( result.append(f"SCCACHE_S3_KEY_PREFIX={sccache_dir}") if not s3_rw_access: result.append("SCCACHE_S3_NO_CREDENTIALS=true") + if s3_access_key_id: + result.append(f"AWS_ACCESS_KEY_ID={s3_access_key_id}") + if s3_secret_access_key: + result.append(f"AWS_SECRET_ACCESS_KEY={s3_secret_access_key}") if clang_tidy: # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above. @@ -447,6 +453,14 @@ def parse_args() -> argparse.Namespace: type=dir_name, help="a directory with ccache", ) + parser.add_argument( + "--s3-access-key-id", + help="an S3 access key id used for sscache bucket", + ) + parser.add_argument( + "--s3-secret-access-key", + help="an S3 secret access key used for sscache bucket", + ) parser.add_argument( "--s3-bucket", help="an S3 bucket used for sscache and clang-tidy-cache", @@ -530,9 +544,11 @@ def main() -> None: args.sanitizer, args.package_type, args.cache, + args.s3_access_key_id, args.s3_bucket, args.s3_directory, args.s3_rw_access, + args.s3_secret_access_key, args.clang_tidy, args.version, args.official, diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 6ff18fb301c6..9e22f9154967 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 # see https://github.com/moby/moby/issues/4032#issuecomment-192327844 # It could be removed after we move on a version 23:04+ diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 848da4ef5b0e..1fc5a1f5f424 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/test-base . +# docker build -t altinityinfra/test-base . ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +FROM altinityinfra/test-util:$FROM_TAG RUN apt-get update \ && apt-get install \ diff --git a/docker/test/clickbench/Dockerfile b/docker/test/clickbench/Dockerfile index 0b6b1736e031..214191a8b488 100644 --- a/docker/test/clickbench/Dockerfile +++ b/docker/test/clickbench/Dockerfile @@ -1,5 +1,5 @@ ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG ENV TZ=Europe/Amsterdam RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone diff --git a/docker/test/compatibility/centos/Dockerfile b/docker/test/compatibility/centos/Dockerfile index 628609e374f6..1edb42422b1f 100644 --- a/docker/test/compatibility/centos/Dockerfile +++ b/docker/test/compatibility/centos/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/test-old-centos . +# docker build -t altinityinfra/test-old-centos . FROM centos:5 CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \ diff --git a/docker/test/compatibility/ubuntu/Dockerfile b/docker/test/compatibility/ubuntu/Dockerfile index ddd0a76bd446..0eb283ff3daf 100644 --- a/docker/test/compatibility/ubuntu/Dockerfile +++ b/docker/test/compatibility/ubuntu/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/test-old-ubuntu . +# docker build -t altinityinfra/test-old-ubuntu . FROM ubuntu:12.04 CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \ diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 5d311c673a42..c4fae94d4abe 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,6 +1,6 @@ -# docker build -t clickhouse/fasttest . +# docker build -t altinityinfra/fasttest . ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +FROM altinityinfra/test-util:$FROM_TAG RUN apt-get update \ && apt-get install \ diff --git a/docker/test/fasttest/requirements.txt b/docker/test/fasttest/requirements.txt index 993ea22e5ae4..bc5afd26b6c7 100644 --- a/docker/test/fasttest/requirements.txt +++ b/docker/test/fasttest/requirements.txt @@ -1,41 +1,41 @@ -Jinja2==3.1.3 -MarkupSafe==2.1.5 -PyJWT==2.3.0 -PyYAML==6.0.1 -Pygments==2.11.2 -SecretStorage==3.3.1 -blinker==1.4 -certifi==2020.6.20 -chardet==4.0.0 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -idna==3.3 -importlib-metadata==4.6.4 -jeepney==0.7.1 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -lxml==4.8.0 -more-itertools==8.10.0 -numpy==1.26.3 -oauthlib==3.2.0 -packaging==24.1 -pandas==1.5.3 -pip==24.1.1 -pipdeptree==2.23.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -python-dateutil==2.9.0.post0 -pytz==2024.1 -requests==2.32.3 -scipy==1.12.0 -setuptools==59.6.0 -six==1.16.0 -termcolor==1.1.0 -urllib3==1.26.5 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +Jinja2~=3.1.3 +MarkupSafe~=2.1.5 +PyJWT~=2.3.0 +PyYAML~=6.0.1 +Pygments~=2.11.2 +SecretStorage~=3.3.1 +blinker~=1.4 +certifi +chardet~=4.0.0 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +idna~=3.3 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +lxml~=4.8.0 +more-itertools~=8.10.0 +numpy~=1.26.3 +oauthlib~=3.2.0 +packaging~=24.1 +pandas~=1.5.3 +pip~=24.1.1 +pipdeptree~=2.23.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +python-dateutil~=2.9.0 +pytz +requests~=2.32.3 +scipy~=1.12.0 +setuptools~=59.6.0 +six~=1.16.0 +termcolor~=1.1.0 +urllib3~=1.26.5 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/fuzzer/Dockerfile b/docker/test/fuzzer/Dockerfile index e1fb09b8ed57..f79fa706e72e 100644 --- a/docker/test/fuzzer/Dockerfile +++ b/docker/test/fuzzer/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/fuzzer . +# docker build -t altinityinfra/fuzzer . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" @@ -41,5 +41,5 @@ CMD set -o pipefail \ && cd /workspace \ && timeout -s 9 1h /run-fuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log -# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/fuzzer +# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> altinityinfra/fuzzer diff --git a/docker/test/fuzzer/requirements.txt b/docker/test/fuzzer/requirements.txt index 3dce93e023b2..fa70d89e7ac3 100644 --- a/docker/test/fuzzer/requirements.txt +++ b/docker/test/fuzzer/requirements.txt @@ -1,27 +1,27 @@ -blinker==1.4 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -Jinja2==3.1.4 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -MarkupSafe==2.1.5 -more-itertools==8.10.0 -oauthlib==3.2.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +Jinja2~=3.1.4 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +MarkupSafe~=2.1.5 +more-itertools~=8.10.0 +oauthlib~=3.2.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index ae1b9e94bed4..821d1c453baf 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -23,7 +23,7 @@ BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.a function git_clone_with_retry { for _ in 1 2 3 4; do - if git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -- "$1" 2>&1 | ts '%Y-%m-%d %H:%M:%S';then + if git clone --depth 1 https://github.com/Altinity/ClickHouse.git -- "$1" 2>&1 | ts '%Y-%m-%d %H:%M:%S';then return 0 else sleep 0.5 @@ -448,7 +448,7 @@ p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-s
-

AST Fuzzer for PR #${PR_TO_TEST} @ ${SHA_TO_TEST}

+

AST Fuzzer for PR #${PR_TO_TEST} @ ${SHA_TO_TEST}

run.log fuzzer.log.zst diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index dc4d470a2623..8af06623deaf 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/integration-test . +# docker build -t altinityinfra/integration-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG SHELL ["/bin/bash", "-c"] @@ -29,6 +29,7 @@ RUN apt-get update \ python3-pip \ libcurl4-openssl-dev \ libssl-dev \ + iptables \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* diff --git a/docker/test/integration/base/requirements.txt b/docker/test/integration/base/requirements.txt index d195d8deaf60..98df4d94bf58 100644 --- a/docker/test/integration/base/requirements.txt +++ b/docker/test/integration/base/requirements.txt @@ -1,26 +1,26 @@ -blinker==1.4 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -more-itertools==8.10.0 -oauthlib==3.2.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -pycurl==7.45.3 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +more-itertools~=8.10.0 +oauthlib~=3.2.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +pycurl~=7.45.3 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/integration/helper_container/Dockerfile b/docker/test/integration/helper_container/Dockerfile index 49a3d3cd84b8..a453533ce852 100644 --- a/docker/test/integration/helper_container/Dockerfile +++ b/docker/test/integration/helper_container/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/integration-helper . +# docker build -t altinityinfra/integration-helper . # Helper docker container to run iptables without sudo FROM alpine:3.18 diff --git a/docker/test/integration/kerberized_hadoop/Dockerfile b/docker/test/integration/kerberized_hadoop/Dockerfile index 62dcd2ca9973..ddc0850b3c5a 100644 --- a/docker/test/integration/kerberized_hadoop/Dockerfile +++ b/docker/test/integration/kerberized_hadoop/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/kerberized-hadoop . +# docker build -t altinityinfra/kerberized-hadoop . FROM sequenceiq/hadoop-docker:2.7.0 diff --git a/docker/test/integration/kerberos_kdc/Dockerfile b/docker/test/integration/kerberos_kdc/Dockerfile index a203c33a3313..a7f989bf4a56 100644 --- a/docker/test/integration/kerberos_kdc/Dockerfile +++ b/docker/test/integration/kerberos_kdc/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/kerberos-kdc . +# docker build -t altinityinfra/kerberos-kdc . FROM centos:6 RUN sed -i '/^mirrorlist/s/^/#/;/^#baseurl/{s/#//;s/mirror.centos.org\/centos\/$releasever/vault.centos.org\/6.10/}' /etc/yum.repos.d/*B* diff --git a/docker/test/integration/mysql_golang_client/Dockerfile b/docker/test/integration/mysql_golang_client/Dockerfile index 5281f786ae2d..52be68126e47 100644 --- a/docker/test/integration/mysql_golang_client/Dockerfile +++ b/docker/test/integration/mysql_golang_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/mysql-golang-client . +# docker build -t altinityinfra/mysql-golang-client . # MySQL golang client docker container FROM golang:1.17 diff --git a/docker/test/integration/mysql_java_client/Dockerfile b/docker/test/integration/mysql_java_client/Dockerfile index 38fefac070e7..5826ee77d501 100644 --- a/docker/test/integration/mysql_java_client/Dockerfile +++ b/docker/test/integration/mysql_java_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/mysql-java-client . +# docker build -t altinityinfra/mysql-java-client . # MySQL Java client docker container FROM openjdk:8-jdk-alpine diff --git a/docker/test/integration/mysql_js_client/Dockerfile b/docker/test/integration/mysql_js_client/Dockerfile index 4c9df10ace1c..2b821f243234 100644 --- a/docker/test/integration/mysql_js_client/Dockerfile +++ b/docker/test/integration/mysql_js_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/mysql-js-client . +# docker build -t altinityinfra/mysql-js-client . # MySQL JavaScript client docker container FROM node:16.14.2 diff --git a/docker/test/integration/mysql_php_client/Dockerfile b/docker/test/integration/mysql_php_client/Dockerfile index 0e11ae023e63..b060e93f70a3 100644 --- a/docker/test/integration/mysql_php_client/Dockerfile +++ b/docker/test/integration/mysql_php_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/mysql-php-client . +# docker build -t altinityinfra/mysql-php-client . # MySQL PHP client docker container FROM php:8-cli-alpine diff --git a/docker/test/integration/postgresql_java_client/Dockerfile b/docker/test/integration/postgresql_java_client/Dockerfile index c5583085ef37..5a7458cc1d2f 100644 --- a/docker/test/integration/postgresql_java_client/Dockerfile +++ b/docker/test/integration/postgresql_java_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/postgresql-java-client . +# docker build -t altinityinfra/postgresql-java-client . # PostgreSQL Java client docker container FROM ubuntu:18.04 diff --git a/docker/test/integration/resolver/Dockerfile b/docker/test/integration/resolver/Dockerfile index b35a7262651b..a7fec32620e1 100644 --- a/docker/test/integration/resolver/Dockerfile +++ b/docker/test/integration/resolver/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/python-bottle . +# docker build -t altinityinfra/python-bottle . # Helper docker container to run python bottle apps FROM python:3 diff --git a/docker/test/integration/resolver/requirements.txt b/docker/test/integration/resolver/requirements.txt index fbf852953296..314b112319b3 100644 --- a/docker/test/integration/resolver/requirements.txt +++ b/docker/test/integration/resolver/requirements.txt @@ -1,6 +1,6 @@ -bottle==0.12.25 -packaging==24.1 -pip==23.2.1 -pipdeptree==2.23.0 -setuptools==69.0.3 -wheel==0.42.0 +bottle~=0.13 +packaging~=24.1 +pip~=23.2.1 +pipdeptree~=2.23.0 +setuptools~=69.0.3 +wheel~=0.42.0 diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index d62009f1be38..f5784428e5fd 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/integration-tests-runner . +# docker build -t altinityinfra/integration-tests-runner . FROM ubuntu:22.04 # ARG for quick switch to a given ubuntu mirror diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index 8882daa38ea3..84baa6b1f342 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -4,12 +4,12 @@ set -e mkdir -p /etc/docker/ echo '{ "ipv6": true, - "fixed-cidr-v6": "fd00::/8", + "fixed-cidr-v6": "2001:db8:1::/64", "ip-forward": true, "log-level": "debug", "storage-driver": "overlay2", - "insecure-registries" : ["dockerhub-proxy.dockerhub-proxy-zone:5000"], - "registry-mirrors" : ["http://dockerhub-proxy.dockerhub-proxy-zone:5000"] + "insecure-registries" : ["65.108.242.32:5000"], + "registry-mirrors" : ["http://65.108.242.32:5000"] }' | dd of=/etc/docker/daemon.json 2>/dev/null if [ -f /sys/fs/cgroup/cgroup.controllers ]; then diff --git a/docker/test/integration/runner/requirements.txt b/docker/test/integration/runner/requirements.txt index 2c9df73ccca3..f1ab83e89ec4 100644 --- a/docker/test/integration/runner/requirements.txt +++ b/docker/test/integration/runner/requirements.txt @@ -1,116 +1,116 @@ -PyHDFS==0.3.1 -PyJWT==2.3.0 -PyMySQL==1.1.0 -PyNaCl==1.5.0 -PyYAML==5.3.1 -SecretStorage==3.3.1 -argon2-cffi-bindings==21.2.0 -argon2-cffi==23.1.0 -async-timeout==4.0.3 -asyncio==3.4.3 -attrs==23.2.0 -avro==1.10.2 -azure-core==1.30.1 -azure-storage-blob==12.19.0 -bcrypt==4.1.3 -beautifulsoup4==4.12.3 -blinker==1.4 -boto3==1.34.24 -botocore==1.34.101 -bs4==0.0.2 -cassandra-driver==3.29.0 -certifi==2024.2.2 -cffi==1.16.0 -charset-normalizer==3.3.2 -click==8.1.7 -confluent-kafka==2.3.0 -cryptography==3.4.8 -dbus-python==1.2.18 -decorator==5.1.1 -delta-spark==2.3.0 -dict2xml==1.7.4 -dicttoxml==1.7.16 -distro-info==1.1+ubuntu0.2 -distro==1.7.0 -docker-compose==1.29.2 -docker==6.1.3 -dockerpty==0.4.1 -docopt==0.6.2 -exceptiongroup==1.2.1 -execnet==2.1.1 -geomet==0.2.1.post1 -grpcio-tools==1.60.0 -grpcio==1.60.0 -gssapi==1.8.3 -httplib2==0.20.2 -idna==3.7 -importlib-metadata==4.6.4 -iniconfig==2.0.0 -isodate==0.6.1 -jeepney==0.7.1 -jmespath==1.0.1 -jsonschema==3.2.0 -jwcrypto==1.5.6 -kafka-python==2.0.2 -kazoo==2.9.0 -keyring==23.5.0 -krb5==0.5.1 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -lxml==5.1.0 -lz4==4.3.3 -minio==7.2.3 -more-itertools==8.10.0 -nats-py==2.6.0 -oauthlib==3.2.0 -packaging==24.0 -paramiko==3.4.0 -pika==1.2.0 -pip==24.1.1 -pipdeptree==2.23.0 -pluggy==1.5.0 -protobuf==4.25.2 -psycopg2-binary==2.9.6 -py4j==0.10.9.5 -py==1.11.0 -pyarrow==17.0.0 -pycparser==2.22 -pycryptodome==3.20.0 -pymongo==3.11.0 -pyparsing==2.4.7 -pyrsistent==0.20.0 -pyspark==3.3.2 -pyspnego==0.10.2 -pytest-order==1.0.0 -pytest-random==0.2 -pytest-repeat==0.9.3 -pytest-reportlog==0.4.0 -pytest-timeout==2.2.0 -pytest-xdist==3.5.0 -pytest==7.4.4 -python-apt==2.4.0+ubuntu3 -python-dateutil==2.9.0.post0 -python-dotenv==0.21.1 -pytz==2023.3.post1 -redis==5.0.1 -requests-kerberos==0.14.0 -requests==2.31.0 -retry==0.9.2 -s3transfer==0.10.1 -setuptools==59.6.0 -simplejson==3.19.2 -six==1.16.0 -soupsieve==2.5 -texttable==1.7.0 -tomli==2.0.1 -typing_extensions==4.11.0 -tzlocal==2.1 -unattended-upgrades==0.1 -urllib3==2.0.7 -wadllib==1.3.6 -websocket-client==0.59.0 -wheel==0.37.1 -zipp==1.0.0 +PyHDFS~=0.3.1 +PyJWT~=2.3.0 +PyMySQL~=1.1.0 +PyNaCl~=1.5.0 +PyYAML~=5.3.1 +SecretStorage~=3.3.1 +argon2-cffi-bindings~=21.2.0 +argon2-cffi~=23.1.0 +async-timeout~=4.0.3 +asyncio~=3.4.3 +attrs~=23.2.0 +avro~=1.10.2 +azure-core~=1.30.1 +azure-storage-blob~=12.19.0 +bcrypt~=4.1.3 +beautifulsoup4~=4.12.3 +blinker~=1.4 +boto3~=1.34.24 +botocore~=1.34.101 +bs4~=0.0.2 +cassandra-driver~=3.29.0 +certifi +cffi~=1.16.0 +charset-normalizer~=3.3.2 +click~=8.1.7 +confluent-kafka~=2.3.0 +cryptography~=3.4.8 +dbus-python~=1.2.18 +decorator~=5.1.1 +delta-spark~=2.3.0 +dict2xml~=1.7.4 +dicttoxml~=1.7.16 +distro-info~=1.1 +distro~=1.7.0 +docker-compose~=1.29.2 +docker~=6.1.3 +dockerpty~=0.4.1 +docopt~=0.6.2 +exceptiongroup~=1.2.1 +execnet~=2.1.1 +geomet~=0.2.1 +grpcio-tools~=1.60.0 +grpcio~=1.60.0 +gssapi~=1.8.3 +httplib2~=0.20.2 +idna~=3.7 +importlib-metadata~=4.6.4 +iniconfig~=2.0.0 +isodate~=0.6.1 +jeepney~=0.7.1 +jmespath~=1.0.1 +jsonschema~=3.2.0 +jwcrypto~=1.5.6 +kafka-python~=2.0.2 +kazoo~=2.9.0 +keyring~=23.5.0 +krb5~=0.5.1 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +lxml~=5.1.0 +lz4~=4.3.3 +minio~=7.2.3 +more-itertools~=8.10.0 +nats-py~=2.6.0 +oauthlib~=3.2.0 +packaging~=24.0 +paramiko~=3.4.0 +pika~=1.2.0 +pip~=24.1.1 +pipdeptree~=2.23.0 +pluggy~=1.5.0 +protobuf~=4.25.2 +psycopg2-binary~=2.9.6 +py4j~=0.10.9.5 +py~=1.11.0 +pyarrow~=17.0.0 +pycparser~=2.22 +pycryptodome~=3.20.0 +pymongo~=3.11.0 +pyparsing~=2.4.7 +pyrsistent~=0.20.0 +pyspark~=3.3.2 +pyspnego~=0.10.2 +pytest-order~=1.0.0 +pytest-random~=0.2 +pytest-repeat~=0.9.3 +pytest-reportlog~=0.4.0 +pytest-timeout~=2.2.0 +pytest-xdist~=3.5.0 +pytest~=7.4.4 +python-apt~=2.4.0 +python-dateutil~=2.9.0 +python-dotenv~=0.21.1 +pytz +redis~=5.0.1 +requests-kerberos~=0.14.0 +requests~=2.31.0 +retry~=0.9.2 +s3transfer~=0.10.1 +setuptools~=59.6.0 +simplejson~=3.19.2 +six~=1.16.0 +soupsieve~=2.5 +texttable~=1.7.0 +tomli~=2.0.1 +typing_extensions~=4.11.0 +tzlocal~=2.1 +unattended-upgrades~=0.1 +urllib3~=2.0.7 +wadllib~=1.3.6 +websocket-client~=0.59.0 +wheel~=0.37.1 +zipp~=1.0.0 deltalake==0.16.0 diff --git a/docker/test/integration/s3_proxy/Dockerfile b/docker/test/integration/s3_proxy/Dockerfile index 5858218e4e4c..df8d8f00f216 100644 --- a/docker/test/integration/s3_proxy/Dockerfile +++ b/docker/test/integration/s3_proxy/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/s3-proxy . +# docker build -t altinityinfra/s3-proxy . FROM nginx:alpine COPY run.sh /run.sh diff --git a/docker/test/keeper-jepsen/Dockerfile b/docker/test/keeper-jepsen/Dockerfile index 3c5d0a6ecb42..d3080a526711 100644 --- a/docker/test/keeper-jepsen/Dockerfile +++ b/docker/test/keeper-jepsen/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/keeper-jepsen-test . +# docker build -t altinityinfra/keeper-jepsen-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/libfuzzer/Dockerfile b/docker/test/libfuzzer/Dockerfile index 3ffae0cd921f..6f3f3aeb6eb7 100644 --- a/docker/test/libfuzzer/Dockerfile +++ b/docker/test/libfuzzer/Dockerfile @@ -1,6 +1,6 @@ -# docker build -t clickhouse/libfuzzer . +# docker build -t altinityinfra/libfuzzer . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" @@ -37,5 +37,5 @@ ENV FUZZER_ARGS="-max_total_time=60" SHELL ["/bin/bash", "-c"] -# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/libfuzzer +# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> altinityinfra/libfuzzer diff --git a/docker/test/libfuzzer/requirements.txt b/docker/test/libfuzzer/requirements.txt index 3dce93e023b2..fa70d89e7ac3 100644 --- a/docker/test/libfuzzer/requirements.txt +++ b/docker/test/libfuzzer/requirements.txt @@ -1,27 +1,27 @@ -blinker==1.4 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -Jinja2==3.1.4 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -MarkupSafe==2.1.5 -more-itertools==8.10.0 -oauthlib==3.2.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +Jinja2~=3.1.4 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +MarkupSafe~=2.1.5 +more-itertools~=8.10.0 +oauthlib~=3.2.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index f71392752826..ad4866ed742a 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -1,7 +1,7 @@ -# docker build -t clickhouse/performance-comparison . +# docker build -t altinityinfra/performance-comparison . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ @@ -46,4 +46,4 @@ ENV PATH="/opt/gdb/bin:${PATH}" CMD ["bash", "/run.sh"] -# docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/performance-comparison +# docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> altinityinfra/performance-comparison diff --git a/docker/test/performance-comparison/requirements.txt b/docker/test/performance-comparison/requirements.txt index 932527cc0226..eb927584d593 100644 --- a/docker/test/performance-comparison/requirements.txt +++ b/docker/test/performance-comparison/requirements.txt @@ -1,32 +1,32 @@ -blinker==1.4 -clickhouse-driver==0.2.7 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -more-itertools==8.10.0 -numpy==1.26.3 -oauthlib==3.2.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -Pygments==2.11.2 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -pytz==2023.4 -PyYAML==6.0.1 -scipy==1.12.0 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -tzlocal==2.1 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +clickhouse-driver~=0.2.7 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +more-itertools~=8.10.0 +numpy~=1.26.3 +oauthlib~=3.2.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +Pygments~=2.11.2 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4 +pytz +PyYAML~=6.0.1 +scipy~=1.12.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +tzlocal~=2.1 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/server-jepsen/Dockerfile b/docker/test/server-jepsen/Dockerfile index fd70fc457020..5207f31b953f 100644 --- a/docker/test/server-jepsen/Dockerfile +++ b/docker/test/server-jepsen/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/server-jepsen-test . +# docker build -t altinityinfra/server-jepsen-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/sqlancer/Dockerfile b/docker/test/sqlancer/Dockerfile index 3b919ffb3e39..b77066bcbac4 100644 --- a/docker/test/sqlancer/Dockerfile +++ b/docker/test/sqlancer/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/sqlancer-test . +# docker build -t altinityinfra/sqlancer-test . FROM ubuntu:22.04 # ARG for quick switch to a given ubuntu mirror diff --git a/docker/test/sqllogic/Dockerfile b/docker/test/sqllogic/Dockerfile index 0d21a2da44ee..767e5eecfa84 100644 --- a/docker/test/sqllogic/Dockerfile +++ b/docker/test/sqllogic/Dockerfile @@ -1,6 +1,6 @@ -# docker build -t clickhouse/sqllogic-test . +# docker build -t altinityinfra/sqllogic-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG RUN apt-get update --yes \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/sqllogic/requirements.txt b/docker/test/sqllogic/requirements.txt index abc0a3686590..3a9ec5f1c556 100644 --- a/docker/test/sqllogic/requirements.txt +++ b/docker/test/sqllogic/requirements.txt @@ -1,30 +1,30 @@ -blinker==1.4 -cryptography==3.4.8 -dbus-python==1.2.18 -deepdiff==7.0.1 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -more-itertools==8.10.0 -numpy==1.26.4 -oauthlib==3.2.0 -ordered-set==4.1.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -PyJWT==2.3.0 -pyodbc==5.1.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -sqlglot==23.16.0 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +cryptography~=3.4.8 +dbus-python~=1.2.18 +deepdiff~=7.0.1 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +more-itertools~=8.10.0 +numpy~=1.26.4 +oauthlib~=3.2.0 +ordered-set~=4.1.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +PyJWT~=2.3.0 +pyodbc~=5.1.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +sqlglot~=23.16.0 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/sqltest/Dockerfile b/docker/test/sqltest/Dockerfile index b805bb03c2b0..e21cb2d7febb 100644 --- a/docker/test/sqltest/Dockerfile +++ b/docker/test/sqltest/Dockerfile @@ -1,6 +1,6 @@ -# docker build -t clickhouse/sqltest . +# docker build -t altinityinfra/sqltest . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG RUN apt-get update --yes \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/sqltest/requirements.txt b/docker/test/sqltest/requirements.txt index 4a0ae3edbacd..4676ffd9ae8f 100644 --- a/docker/test/sqltest/requirements.txt +++ b/docker/test/sqltest/requirements.txt @@ -1,29 +1,29 @@ -blinker==1.4 -clickhouse-driver==0.2.7 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -httplib2==0.20.2 -importlib-metadata==4.6.4 -jeepney==0.7.1 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -more-itertools==8.10.0 -oauthlib==3.2.0 -packaging==24.1 -pip==24.1.1 -pipdeptree==2.23.0 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -pytz==2024.1 -PyYAML==6.0.1 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -tzlocal==5.2 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +blinker~=1.4 +clickhouse-driver~=0.2.7 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +httplib2~=0.20.2 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +more-itertools~=8.10.0 +oauthlib~=3.2.0 +packaging~=24.1 +pip~=24.1.1 +pipdeptree~=2.23.0 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +pytz +PyYAML~=6.0.1 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +tzlocal~=5.2 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 0daf88cad7ee..e0f77526e242 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #47031 -# docker build -t clickhouse/stateful-test . +# docker build -t altinityinfra/stateful-test . ARG FROM_TAG=latest -FROM clickhouse/stateless-test:$FROM_TAG +FROM altinityinfra/stateless-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index c072eeb0fa8b..b08d396d1d56 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -22,8 +22,7 @@ source /utils.lib # install test configs /usr/share/clickhouse-test/config/install.sh -azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence & - +azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & ./setup_minio.sh stateful ./mc admin trace clickminio > /test_output/minio.log & MC_ADMIN_PID=$! @@ -241,7 +240,7 @@ function run_tests() ) if [[ -n "$USE_PARALLEL_REPLICAS" ]] && [[ "$USE_PARALLEL_REPLICAS" -eq 1 ]]; then TEST_ARGS+=( - --client="clickhouse-client --allow_experimental_parallel_reading_from_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 --max_parallel_replicas=100 --cluster_for_parallel_replicas='parallel_replicas'" + --client="clickhouse-client --enable_parallel_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 --max_parallel_replicas=100 --cluster_for_parallel_replicas='parallel_replicas'" --no-parallel-replicas ) fi diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index b0c4914a4e88..7e57ad900a49 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/stateless-test . +# docker build -t altinityinfra/stateless-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz" @@ -87,7 +87,7 @@ ENV MINIO_ROOT_PASSWORD="clickhouse" ENV EXPORT_S3_STORAGE_POLICIES=1 ENV CLICKHOUSE_GRPC_CLIENT="/usr/share/clickhouse-utils/grpc-client/clickhouse-grpc-client.py" -RUN npm install -g azurite@3.30.0 \ +RUN npm install -g azurite@^3.33.0 \ && npm install -g tslib && npm install -g node COPY run.sh / diff --git a/docker/test/stateless/requirements.txt b/docker/test/stateless/requirements.txt index 74860d5fec3a..1537cd261900 100644 --- a/docker/test/stateless/requirements.txt +++ b/docker/test/stateless/requirements.txt @@ -1,53 +1,53 @@ -awscli==1.22.34 -blinker==1.4 -botocore==1.23.34 -certifi==2020.6.20 -chardet==4.0.0 -colorama==0.4.4 -cryptography==3.4.8 -dbus-python==1.2.18 -distro==1.7.0 -docutils==0.17.1 -grpcio==1.47.0 -gyp==0.1 -httplib2==0.20.2 -idna==3.3 -importlib-metadata==4.6.4 -jeepney==0.7.1 -Jinja2==3.1.3 -jmespath==0.10.0 -keyring==23.5.0 -launchpadlib==1.10.16 -lazr.restfulclient==0.14.4 -lazr.uri==1.0.6 -lxml==4.8.0 -MarkupSafe==2.1.5 -more-itertools==8.10.0 -numpy==1.26.3 -oauthlib==3.2.0 -packaging==24.1 -pandas==1.5.3 -pip==24.1.1 -pipdeptree==2.23.0 -protobuf==4.25.3 -pyarrow==15.0.0 -pyasn1==0.4.8 -PyJWT==2.3.0 -pyparsing==2.4.7 -python-apt==2.4.0+ubuntu3 -python-dateutil==2.8.1 -pytz==2024.1 -PyYAML==6.0.1 -requests==2.32.3 -roman==3.3 -rsa==4.8 -s3transfer==0.5.0 -scipy==1.12.0 -SecretStorage==3.3.1 -setuptools==59.6.0 -six==1.16.0 -termcolor==1.1.0 -urllib3==1.26.5 -wadllib==1.3.6 -wheel==0.37.1 -zipp==1.0.0 +awscli~=1.22.34 +blinker~=1.4 +botocore~=1.23.34 +certifi +chardet~=4.0.0 +colorama~=0.4.4 +cryptography~=3.4.8 +dbus-python~=1.2.18 +distro~=1.7.0 +docutils~=0.17.1 +grpcio~=1.47.0 +gyp~=0.1 +httplib2~=0.20.2 +idna~=3.3 +importlib-metadata~=4.6.4 +jeepney~=0.7.1 +Jinja2~=3.1.3 +jmespath~=0.10.0 +keyring~=23.5.0 +launchpadlib~=1.10.16 +lazr.restfulclient~=0.14.4 +lazr.uri~=1.0.6 +lxml~=4.8.0 +MarkupSafe~=2.1.5 +more-itertools~=8.10.0 +numpy~=1.26.3 +oauthlib~=3.2.0 +packaging~=24.1 +pandas~=1.5.3 +pip~=24.1.1 +pipdeptree~=2.23.0 +protobuf~=4.25.3 +pyarrow~=15.0.0 +pyasn1~=0.4.8 +PyJWT~=2.3.0 +pyparsing~=2.4.7 +python-apt~=2.4.0 +python-dateutil~=2.8.1 +pytz +PyYAML~=6.0.1 +requests~=2.32.3 +roman~=3.3 +rsa~=4.8 +s3transfer~=0.5.0 +scipy~=1.12.0 +SecretStorage~=3.3.1 +setuptools~=59.6.0 +six~=1.16.0 +termcolor~=1.1.0 +urllib3~=1.26.5 +wadllib~=1.3.6 +wheel~=0.37.1 +zipp~=1.0.0 diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index ad0cd321cc55..13d79243713b 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -50,6 +50,12 @@ source /utils.lib # install test configs /usr/share/clickhouse-test/config/install.sh +if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then + echo "Azure is disabled" +else + azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & +fi + ./setup_minio.sh stateless ./setup_hdfs_minicluster.sh diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index 0f81a1cd07fb..507bf90bc2dd 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/stress-test . +# docker build -t altinityinfra/stress-test . ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG +FROM altinityinfra/stateful-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index b21114e456f4..48348aa131ca 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -55,6 +55,7 @@ export ZOOKEEPER_FAULT_INJECTION=1 # available for dump via clickhouse-local configure +azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & ./setup_minio.sh stateless # to have a proper environment config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index cdc1d1fa095d..88894e444c63 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/style-test . +# docker build -t altinityinfra/style-test . FROM ubuntu:22.04 ARG ACT_VERSION=0.2.33 ARG ACTIONLINT_VERSION=1.6.22 diff --git a/docker/test/style/requirements.txt b/docker/test/style/requirements.txt index ed73d0d3636c..b3f62d122efb 100644 --- a/docker/test/style/requirements.txt +++ b/docker/test/style/requirements.txt @@ -1,58 +1,58 @@ -aiohttp==3.9.5 -aiosignal==1.3.1 -astroid==3.1.0 -async-timeout==4.0.3 -attrs==23.2.0 -black==24.4.2 -boto3==1.34.131 -botocore==1.34.131 -certifi==2024.6.2 -cffi==1.16.0 -charset-normalizer==3.3.2 -click==8.1.7 -codespell==2.2.1 -cryptography==42.0.8 -Deprecated==1.2.14 -dill==0.3.8 -flake8==4.0.1 -frozenlist==1.4.1 -idna==3.7 -isort==5.13.2 -jmespath==1.0.1 -jwt==1.3.1 -mccabe==0.6.1 -multidict==6.0.5 -mypy==1.8.0 -mypy-extensions==1.0.0 -packaging==24.1 -pathspec==0.9.0 -pip==24.1.1 -pipdeptree==2.23.0 -platformdirs==4.2.2 -pycodestyle==2.8.0 -pycparser==2.22 -pyflakes==2.4.0 -PyGithub==2.3.0 -PyJWT==2.8.0 -pylint==3.1.0 -PyNaCl==1.5.0 -python-dateutil==2.9.0.post0 -python-magic==0.4.24 -PyYAML==6.0.1 -rapidfuzz==3.9.3 -requests==2.32.3 -s3transfer==0.10.1 -setuptools==59.6.0 -six==1.16.0 -thefuzz==0.22.1 -tomli==2.0.1 -tomlkit==0.12.5 -tqdm==4.66.4 -types-requests==2.32.0.20240622 -typing_extensions==4.12.2 -unidiff==0.7.5 -urllib3==2.2.2 -wheel==0.37.1 -wrapt==1.16.0 -yamllint==1.26.3 -yarl==1.9.4 +aiohttp~=3.9.5 +aiosignal~=1.3.1 +astroid~=3.1.0 +async-timeout~=4.0.3 +attrs~=23.2.0 +black~=24.4.2 +boto3~=1.34.131 +botocore~=1.34.131 +certifi +cffi~=1.16.0 +charset-normalizer~=3.3.2 +click~=8.1.7 +codespell~=2.2.1 +cryptography~=42.0.8 +Deprecated~=1.2.14 +dill~=0.3.8 +flake8~=4.0.1 +frozenlist~=1.4.1 +idna~=3.7 +isort~=5.13.2 +jmespath~=1.0.1 +jwt~=1.3.1 +mccabe~=0.6.1 +multidict~=6.0.5 +mypy~=1.8.0 +mypy-extensions~=1.0.0 +packaging~=24.1 +pathspec~=0.9.0 +pip~=24.1.1 +pipdeptree~=2.23.0 +platformdirs~=4.2.2 +pycodestyle~=2.8.0 +pycparser~=2.22 +pyflakes~=2.4.0 +PyGithub~=2.3.0 +PyJWT~=2.8.0 +pylint~=3.1.0 +PyNaCl~=1.5.0 +python-dateutil~=2.9.0 +python-magic~=0.4.24 +PyYAML~=6.0.1 +rapidfuzz~=3.9.3 +requests~=2.32.3 +s3transfer~=0.10.1 +setuptools~=59.6.0 +six~=1.16.0 +thefuzz~=0.22.1 +tomli~=2.0.1 +tomlkit~=0.12.5 +tqdm~=4.66.4 +types-requests~=2.32.0 +typing_extensions~=4.12.2 +unidiff~=0.7.5 +urllib3~=2.2.2 +wheel~=0.37.1 +wrapt~=1.16.0 +yamllint~=1.26.3 +yarl~=1.9.4 diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index 9f4b86aa0ca7..adc72011bd1d 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/unit-test . +# docker build -t altinityinfra/unit-test . ARG FROM_TAG=latest -FROM clickhouse/test-base:$FROM_TAG +FROM altinityinfra/test-base:$FROM_TAG COPY run.sh / RUN chmod +x run.sh diff --git a/docker/test/upgrade/Dockerfile b/docker/test/upgrade/Dockerfile index 78d912fd0312..c66868c2a046 100644 --- a/docker/test/upgrade/Dockerfile +++ b/docker/test/upgrade/Dockerfile @@ -1,7 +1,7 @@ # rebuild in #33610 -# docker build -t clickhouse/upgrade-check . +# docker build -t altinityinfra/upgrade-check . ARG FROM_TAG=latest -FROM clickhouse/stateful-test:$FROM_TAG +FROM altinityinfra/stateful-test:$FROM_TAG RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index 8b949ed95dbd..6601acc708fe 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t clickhouse/test-util . +# docker build -t altinityinfra/test-util . FROM ubuntu:22.04 # ARG for quick switch to a given ubuntu mirror @@ -34,9 +34,8 @@ RUN apt-get update \ # Install cmake 3.20+ for Rust support # Used https://askubuntu.com/a/1157132 as reference -RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | \ - gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg && \ - echo "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" >> /etc/apt/sources.list +RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg && \ + echo "deb [signed-by=/etc/apt/trusted.gpg.d/kitware.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" >> /etc/apt/sources.list.d/kitware.list # initial packages RUN apt-get update \ diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 5c2af401b049..56c983c5a395 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1588,25 +1588,12 @@ This setting is useful for any replicated table. An arbitrary integer expression that can be used to split work between replicas for a specific table. The value can be any integer expression. -A query may be processed faster if it is executed on several servers in parallel but it depends on the used [parallel_replicas_custom_key](#parallel_replicas_custom_key) -and [parallel_replicas_custom_key_filter_type](#parallel_replicas_custom_key_filter_type). Simple expressions using primary keys are preferred. If the setting is used on a cluster that consists of a single shard with multiple replicas, those replicas will be converted into virtual shards. Otherwise, it will behave same as for `SAMPLE` key, it will use multiple replicas of each shard. -## parallel_replicas_custom_key_filter_type {#parallel_replicas_custom_key_filter_type} - -How to use `parallel_replicas_custom_key` expression for splitting work between replicas. - -Possible values: - -- `default` — Use the default implementation using modulo operation on the `parallel_replicas_custom_key`. -- `range` — Split the entire value space of the expression in the ranges. This type of filtering is useful if values of `parallel_replicas_custom_key` are uniformly spread across the entire integer space, e.g. hash values. - -Default value: `default`. - ## parallel_replicas_custom_key_range_lower {#parallel_replicas_custom_key_range_lower} Allows the filter type `range` to split the work evenly between replicas based on the custom range `[parallel_replicas_custom_key_range_lower, INT_MAX]`. @@ -1621,9 +1608,9 @@ Allows the filter type `range` to split the work evenly between replicas based o When used in conjuction with [parallel_replicas_custom_key_range_lower](#parallel_replicas_custom_key_range_lower), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`. -Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing. +Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing -## allow_experimental_parallel_reading_from_replicas +## enable_parallel_replicas Enables or disables sending SELECT queries to all replicas of a table (up to `max_parallel_replicas`). Reading is parallelized and coordinated dynamically. It will work for any kind of MergeTree table. diff --git a/docs/logo_horizontal_blue_black.png b/docs/logo_horizontal_blue_black.png new file mode 100644 index 000000000000..b67e8ed45385 Binary files /dev/null and b/docs/logo_horizontal_blue_black.png differ diff --git a/docs/logo_horizontal_blue_white.png b/docs/logo_horizontal_blue_white.png new file mode 100644 index 000000000000..44ed6cca6d75 Binary files /dev/null and b/docs/logo_horizontal_blue_white.png differ diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 84bbf6c83d31..a6b45f4a7920 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -4215,3 +4215,9 @@ SELECT toFloat64('1.7091'), toFloat64('1.5008753E7') SETTINGS precise_float_pars │ 1.7091 │ 15008753 │ └─────────────────────┴──────────────────────────┘ ``` + +## push_external_roles_in_interserver_queries + +Позволяет передавать роли пользователя от инициатора запроса другим нодам при выполнении запроса. + +Значение по умолчанию: `true`. diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 48dca82eb2b1..36f774a3c12b 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -75,6 +75,8 @@ class Benchmark : public Poco::Util::Application const String & default_database_, const String & user_, const String & password_, + const String & proto_send_chunked_, + const String & proto_recv_chunked_, const String & quota_key_, const String & stage, bool randomize_, @@ -128,7 +130,9 @@ class Benchmark : public Poco::Util::Application connections.emplace_back(std::make_unique( concurrency, cur_host, cur_port, - default_database_, user_, password_, quota_key_, + default_database_, user_, password_, + proto_send_chunked_, proto_recv_chunked_, + quota_key_, /* cluster_= */ "", /* cluster_secret_= */ "", /* client_name_= */ std::string(DEFAULT_CLIENT_NAME), @@ -662,6 +666,50 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) Strings hosts = options.count("host") ? options["host"].as() : Strings({"localhost"}); + String proto_send_chunked {"notchunked"}; + String proto_recv_chunked {"notchunked"}; + + if (options.count("proto_caps")) + { + std::string proto_caps_str = options["proto_caps"].as(); + + std::vector proto_caps; + splitInto<','>(proto_caps, proto_caps_str); + + for (auto cap_str : proto_caps) + { + std::string direction; + + if (cap_str.starts_with("send_")) + { + direction = "send"; + cap_str = cap_str.substr(std::string_view("send_").size()); + } + else if (cap_str.starts_with("recv_")) + { + direction = "recv"; + cap_str = cap_str.substr(std::string_view("recv_").size()); + } + + if (cap_str != "chunked" && cap_str != "notchunked" && cap_str != "chunked_optional" && cap_str != "notchunked_optional") + throw Exception(ErrorCodes::BAD_ARGUMENTS, "proto_caps option is incorrect ({})", proto_caps_str); + + if (direction.empty()) + { + proto_send_chunked = cap_str; + proto_recv_chunked = cap_str; + } + else + { + if (direction == "send") + proto_send_chunked = cap_str; + else + proto_recv_chunked = cap_str; + } + } + } + + Benchmark benchmark( options["concurrency"].as(), options["delay"].as(), @@ -673,6 +721,8 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) options["database"].as(), options["user"].as(), options["password"].as(), + proto_send_chunked, + proto_recv_chunked, options["quota_key"].as(), options["stage"].as(), options.count("randomize"), diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 25c94c56aa6d..d24f44bceb20 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -210,7 +210,7 @@ std::vector Client::loadWarningMessages() "" /* query_id */, QueryProcessingStage::Complete, &client_context->getSettingsRef(), - &client_context->getClientInfo(), false, {}); + &client_context->getClientInfo(), false, {}, {}); while (true) { Packet packet = connection->receivePacket(); diff --git a/programs/client/clickhouse-client.xml b/programs/client/clickhouse-client.xml index c32b63413e93..9ce7d1cb2233 100644 --- a/programs/client/clickhouse-client.xml +++ b/programs/client/clickhouse-client.xml @@ -38,6 +38,21 @@ {display_name} \e[1;31m:)\e[0m + + + 9000 + + + diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index e50521a07307..8ee777cf503f 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -366,6 +366,13 @@ void ContextAccess::setUser(const UserPtr & user_) const current_roles_with_admin_option = user->granted_roles.findGrantedWithAdminOption(*params.current_roles); } + if (params.external_roles && !params.external_roles->empty()) + { + current_roles.insert(current_roles.end(), params.external_roles->begin(), params.external_roles->end()); + auto new_granted_with_admin_option = user->granted_roles.findGrantedWithAdminOption(*params.external_roles); + current_roles_with_admin_option.insert(current_roles_with_admin_option.end(), new_granted_with_admin_option.begin(), new_granted_with_admin_option.end()); + } + subscription_for_roles_changes.reset(); enabled_roles = access_control->getEnabledRoles(current_roles, current_roles_with_admin_option); subscription_for_roles_changes = enabled_roles->subscribeForChanges([weak_ptr = weak_from_this()](const std::shared_ptr & roles_info_) @@ -516,7 +523,6 @@ std::optional ContextAccess::getQuotaUsage() const return getQuota()->getUsage(); } - SettingsChanges ContextAccess::getDefaultSettings() const { std::lock_guard lock{mutex}; diff --git a/src/Access/ContextAccessParams.cpp b/src/Access/ContextAccessParams.cpp index 07374e9bbf52..52d92c9b241f 100644 --- a/src/Access/ContextAccessParams.cpp +++ b/src/Access/ContextAccessParams.cpp @@ -11,6 +11,7 @@ ContextAccessParams::ContextAccessParams( bool full_access_, bool use_default_roles_, const std::shared_ptr> & current_roles_, + const std::shared_ptr> & external_roles_, const Settings & settings_, const String & current_database_, const ClientInfo & client_info_) @@ -18,6 +19,7 @@ ContextAccessParams::ContextAccessParams( , full_access(full_access_) , use_default_roles(use_default_roles_) , current_roles(current_roles_) + , external_roles(external_roles_) , readonly(settings_.readonly) , allow_ddl(settings_.allow_ddl) , allow_introspection(settings_.allow_introspection_functions) @@ -52,6 +54,17 @@ String ContextAccessParams::toString() const } out << "]"; } + if (external_roles && !external_roles->empty()) + { + out << separator() << "external_roles = ["; + for (size_t i = 0; i != external_roles->size(); ++i) + { + if (i) + out << ", "; + out << (*external_roles)[i]; + } + out << "]"; + } if (readonly) out << separator() << "readonly = " << readonly; if (allow_ddl) @@ -101,6 +114,7 @@ bool operator ==(const ContextAccessParams & left, const ContextAccessParams & r CONTEXT_ACCESS_PARAMS_EQUALS(full_access) CONTEXT_ACCESS_PARAMS_EQUALS(use_default_roles) CONTEXT_ACCESS_PARAMS_EQUALS(current_roles) + CONTEXT_ACCESS_PARAMS_EQUALS(external_roles) CONTEXT_ACCESS_PARAMS_EQUALS(readonly) CONTEXT_ACCESS_PARAMS_EQUALS(allow_ddl) CONTEXT_ACCESS_PARAMS_EQUALS(allow_introspection) @@ -153,6 +167,7 @@ bool operator <(const ContextAccessParams & left, const ContextAccessParams & ri CONTEXT_ACCESS_PARAMS_LESS(full_access) CONTEXT_ACCESS_PARAMS_LESS(use_default_roles) CONTEXT_ACCESS_PARAMS_LESS(current_roles) + CONTEXT_ACCESS_PARAMS_LESS(external_roles) CONTEXT_ACCESS_PARAMS_LESS(readonly) CONTEXT_ACCESS_PARAMS_LESS(allow_ddl) CONTEXT_ACCESS_PARAMS_LESS(allow_introspection) diff --git a/src/Access/ContextAccessParams.h b/src/Access/ContextAccessParams.h index 07503a3af6de..82592d630dd2 100644 --- a/src/Access/ContextAccessParams.h +++ b/src/Access/ContextAccessParams.h @@ -19,6 +19,7 @@ class ContextAccessParams bool full_access_, bool use_default_roles_, const std::shared_ptr> & current_roles_, + const std::shared_ptr> & external_roles_, const Settings & settings_, const String & current_database_, const ClientInfo & client_info_); @@ -31,6 +32,7 @@ class ContextAccessParams const bool use_default_roles; const std::shared_ptr> current_roles; + const std::shared_ptr> external_roles; const UInt64 readonly; const bool allow_ddl; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 3206b20b691b..917ad7cbb922 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -26,7 +26,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } - LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControl & access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix) : IAccessStorage(storage_name_), access_control(access_control_), memory_storage(storage_name_, access_control.getChangesNotifier(), false) { @@ -191,8 +190,8 @@ void LDAPAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id, } else { - granted_role_names.erase(role_id); granted_role_ids.erase(role_name); + granted_role_names.erase(role_id); } } @@ -320,6 +319,10 @@ std::set LDAPAccessStorage::mapExternalRolesNoLock(const LDAPClient::Sea { std::set role_names; + // If this node can't access LDAP server (or has not privileges to fetch roles) and gets empty list of external roles + if (external_roles.empty()) + return role_names; + if (external_roles.size() != role_search_params.size()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unable to map external roles"); diff --git a/src/Client/ClientApplicationBase.cpp b/src/Client/ClientApplicationBase.cpp index 71d13ad4f53a..0649aa5f5d7c 100644 --- a/src/Client/ClientApplicationBase.cpp +++ b/src/Client/ClientApplicationBase.cpp @@ -158,6 +158,8 @@ void ClientApplicationBase::init(int argc, char ** argv) ("config-file,C", po::value(), "config-file path") + ("proto_caps", po::value(), "enable/disable chunked protocol: chunked_optional, notchunked, notchunked_optional, send_chunked, send_chunked_optional, send_notchunked, send_notchunked_optional, recv_chunked, recv_chunked_optional, recv_notchunked, recv_notchunked_optional") + ("query,q", po::value>()->multitoken(), R"(Query. Can be specified multiple times (--query "SELECT 1" --query "SELECT 2") or once with multiple comma-separated queries (--query "SELECT 1; SELECT 2;"). In the latter case, INSERT queries with non-VALUE format must be separated by empty lines.)") ("queries-file", po::value>()->multitoken(), "file path with queries to execute; multiple files can be specified (--queries-file file1 file2...)") ("multiquery,n", "Obsolete, does nothing") @@ -337,6 +339,41 @@ void ClientApplicationBase::init(int argc, char ** argv) if (options.count("server_logs_file")) server_logs_file = options["server_logs_file"].as(); + if (options.count("proto_caps")) + { + std::string proto_caps_str = options["proto_caps"].as(); + + std::vector proto_caps; + splitInto<','>(proto_caps, proto_caps_str); + + for (auto cap_str : proto_caps) + { + std::string direction; + + if (cap_str.starts_with("send_")) + { + direction = "send"; + cap_str = cap_str.substr(std::string_view("send_").size()); + } + else if (cap_str.starts_with("recv_")) + { + direction = "recv"; + cap_str = cap_str.substr(std::string_view("recv_").size()); + } + + if (cap_str != "chunked" && cap_str != "notchunked" && cap_str != "chunked_optional" && cap_str != "notchunked_optional") + throw Exception(ErrorCodes::BAD_ARGUMENTS, "proto_caps option is incorrect ({})", proto_caps_str); + + if (direction.empty()) + { + config().setString("proto_caps.send", std::string(cap_str)); + config().setString("proto_caps.recv", std::string(cap_str)); + } + else + config().setString("proto_caps." + direction, std::string(cap_str)); + } + } + query_processing_stage = QueryProcessingStage::fromString(options["stage"].as()); query_kind = parseQueryKind(options["query_kind"].as()); profile_events.print = options.count("print-profile-events"); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 8f69eaf54b44..eb574b98edd5 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -73,9 +73,11 @@ #include #include #include +#include #include #include +#include #include "config.h" #include #include @@ -914,6 +916,8 @@ void ClientBase::processTextAsSingleQuery(const String & full_query) } catch (Exception & e) { + if (server_exception) + server_exception->rethrow(); if (!is_interactive) e.addMessage("(in query: {})", full_query); throw; @@ -1032,19 +1036,29 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa query_interrupt_handler.start(signals_before_stop); SCOPE_EXIT({ query_interrupt_handler.stop(); }); - connection->sendQuery( - connection_parameters.timeouts, - query, - query_parameters, - client_context->getCurrentQueryId(), - query_processing_stage, - &client_context->getSettingsRef(), - &client_context->getClientInfo(), - true, - [&](const Progress & progress) { onProgress(progress); }); - - if (send_external_tables) - sendExternalTables(parsed_query); + try { + connection->sendQuery( + connection_parameters.timeouts, + query, + query_parameters, + client_context->getCurrentQueryId(), + query_processing_stage, + &client_context->getSettingsRef(), + &client_context->getClientInfo(), + true, + {}, + [&](const Progress & progress) { onProgress(progress); }); + + if (send_external_tables) + sendExternalTables(parsed_query); + } + catch (const NetException &) + { + // We still want to attempt to process whatever we already received or can receive (socket receive buffer can be not empty) + receiveResult(parsed_query, signals_before_stop, settings.partial_result_on_first_cancel); + throw; + } + receiveResult(parsed_query, signals_before_stop, settings.partial_result_on_first_cancel); break; @@ -1491,6 +1505,7 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars &client_context->getSettingsRef(), &client_context->getClientInfo(), true, + {}, [&](const Progress & progress) { onProgress(progress); }); if (send_external_tables) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 07f4bf19f053..a08f71bfa906 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -5,8 +5,6 @@ #include #include #include -#include -#include #include #include #include @@ -16,6 +14,7 @@ #include #include #include +#include "Common/logger_useful.h" #include #include #include @@ -24,8 +23,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -85,6 +84,7 @@ Connection::~Connection() Connection::Connection(const String & host_, UInt16 port_, const String & default_database_, const String & user_, const String & password_, + const String & proto_send_chunked_, const String & proto_recv_chunked_, [[maybe_unused]] const SSHKey & ssh_private_key_, const String & jwt_, const String & quota_key_, @@ -95,6 +95,7 @@ Connection::Connection(const String & host_, UInt16 port_, Protocol::Secure secure_) : host(host_), port(port_), default_database(default_database_) , user(user_), password(password_) + , proto_send_chunked(proto_send_chunked_), proto_recv_chunked(proto_recv_chunked_) #if USE_SSH , ssh_private_key(ssh_private_key_) #endif @@ -211,10 +212,10 @@ void Connection::connect(const ConnectionTimeouts & timeouts) , tcp_keep_alive_timeout_in_sec); } - in = std::make_shared(*socket); + in = std::make_shared(*socket); in->setAsyncCallback(async_callback); - out = std::make_shared(*socket); + out = std::make_shared(*socket); out->setAsyncCallback(async_callback); connected = true; setDescription(); @@ -222,9 +223,61 @@ void Connection::connect(const ConnectionTimeouts & timeouts) sendHello(); receiveHello(timeouts.handshake_timeout); + if (server_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS) + { + /// Client side of chunked protocol negotiation. + /// Server advertises its protocol capabilities (separate for send and receive channels) by sending + /// in its 'Hello' response one of four types - chunked, notchunked, chunked_optional, notchunked_optional. + /// Not optional types are strict meaning that server only supports this type, optional means that + /// server prefer this type but capable to work in opposite. + /// Client selects which type it is going to communicate based on the settings from config or arguments, + /// and sends either "chunked" or "notchunked" protocol request in addendum section of handshake. + /// Client can detect if server's protocol capabilities are not compatible with client's settings (for example + /// server strictly requires chunked protocol but client's settings only allows notchunked protocol) - in such case + /// client should interrupt this connection. However if client continues with incompatible protocol type request, server + /// will send appropriate exception and disconnect client. + + auto is_chunked = [](const String & chunked_srv_str, const String & chunked_cl_str, const String & direction) + { + bool chunked_srv = chunked_srv_str.starts_with("chunked"); + bool optional_srv = chunked_srv_str.ends_with("_optional"); + bool chunked_cl = chunked_cl_str.starts_with("chunked"); + bool optional_cl = chunked_cl_str.ends_with("_optional"); + + if (optional_srv) + return chunked_cl; + if (optional_cl) + return chunked_srv; + if (chunked_cl != chunked_srv) + throw NetException( + ErrorCodes::NETWORK_ERROR, + "Incompatible protocol: {} set to {}, server requires {}", + direction, + chunked_cl ? "chunked" : "notchunked", + chunked_srv ? "chunked" : "notchunked"); + + return chunked_srv; + }; + + proto_send_chunked = is_chunked(proto_recv_chunked_srv, proto_send_chunked, "send") ? "chunked" : "notchunked"; + proto_recv_chunked = is_chunked(proto_send_chunked_srv, proto_recv_chunked, "recv") ? "chunked" : "notchunked"; + } + else + { + if (proto_send_chunked == "chunked" || proto_recv_chunked == "chunked") + throw NetException( + ErrorCodes::NETWORK_ERROR, + "Incompatible protocol: server's version is too old and doesn't support chunked protocol while client settings require it."); + } + if (server_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_ADDENDUM) sendAddendum(); + if (proto_send_chunked == "chunked") + out->enableChunked(); + if (proto_recv_chunked == "chunked") + in->enableChunked(); + LOG_TRACE(log_wrapper.get(), "Connected to {} server version {}.{}.{}.", server_name, server_version_major, server_version_minor, server_version_patch); } @@ -393,6 +446,16 @@ void Connection::sendAddendum() { if (server_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_QUOTA_KEY) writeStringBinary(quota_key, *out); + + if (server_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS) + { + writeStringBinary(proto_send_chunked, *out); + writeStringBinary(proto_recv_chunked, *out); + } + + if (server_revision >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL) + writeVarUInt(DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION, *out); + out->next(); } @@ -463,6 +526,8 @@ void Connection::receiveHello(const Poco::Timespan & handshake_timeout) readVarUInt(server_version_major, *in); readVarUInt(server_version_minor, *in); readVarUInt(server_revision, *in); + if (server_revision >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL) + readVarUInt(server_parallel_replicas_protocol_version, *in); if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE) readStringBinary(server_timezone, *in); if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) @@ -472,6 +537,12 @@ void Connection::receiveHello(const Poco::Timespan & handshake_timeout) else server_version_patch = server_revision; + if (server_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS) + { + readStringBinary(proto_send_chunked_srv, *in); + readStringBinary(proto_recv_chunked_srv, *in); + } + if (server_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES) { UInt64 rules_size; @@ -611,6 +682,7 @@ bool Connection::ping(const ConnectionTimeouts & timeouts) UInt64 pong = 0; writeVarUInt(Protocol::Client::Ping, *out); + out->finishChunk(); out->next(); if (in->eof()) @@ -660,6 +732,7 @@ TablesStatusResponse Connection::getTablesStatus(const ConnectionTimeouts & time writeVarUInt(Protocol::Client::TablesStatusRequest, *out); request.write(*out, server_revision); + out->finishChunk(); out->next(); UInt64 response_type = 0; @@ -685,6 +758,7 @@ void Connection::sendQuery( const Settings * settings, const ClientInfo * client_info, bool with_pending_data, + const std::vector & external_roles, std::function) { OpenTelemetry::SpanHolder span("Connection::sendQuery()", OpenTelemetry::SpanKind::CLIENT); @@ -758,6 +832,18 @@ void Connection::sendQuery( else writeStringBinary("" /* empty string is a marker of the end of settings */, *out); + String external_roles_str; + if (server_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_INTERSERVER_EXTERNALLY_GRANTED_ROLES) + { + WriteBufferFromString buffer(external_roles_str); + writeVectorBinary(external_roles, buffer); + buffer.finalize(); + + LOG_TRACE(log_wrapper.get(), "Sending external_roles with query: [{}] ({})", fmt::join(external_roles, ", "), external_roles.size()); + + writeStringBinary(external_roles_str, *out); + } + /// Interserver secret if (server_revision >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET) { @@ -778,6 +864,9 @@ void Connection::sendQuery( data += query; data += query_id; data += client_info->initial_user; + // Also for backwards compatibility + if (server_revision >= DBMS_MIN_PROTOCOL_VERSION_WITH_INTERSERVER_EXTERNALLY_GRANTED_ROLES) + data += external_roles_str; /// TODO: add source/target host/ip-address std::string hash = encodeSHA256(data); @@ -813,6 +902,8 @@ void Connection::sendQuery( block_profile_events_in.reset(); block_out.reset(); + out->finishChunk(); + /// Send empty block which means end of data. if (!with_pending_data) { @@ -829,6 +920,7 @@ void Connection::sendCancel() return; writeVarUInt(Protocol::Client::Cancel, *out); + out->finishChunk(); out->next(); } @@ -854,7 +946,10 @@ void Connection::sendData(const Block & block, const String & name, bool scalar) size_t prev_bytes = out->count(); block_out->write(block); - maybe_compressed_out->next(); + if (maybe_compressed_out != out) + maybe_compressed_out->next(); + if (!block) + out->finishChunk(); out->next(); if (throttler) @@ -865,6 +960,7 @@ void Connection::sendIgnoredPartUUIDs(const std::vector & uuids) { writeVarUInt(Protocol::Client::IgnoredPartUUIDs, *out); writeVectorBinary(uuids, *out); + out->finishChunk(); out->next(); } @@ -874,6 +970,7 @@ void Connection::sendReadTaskResponse(const String & response) writeVarUInt(Protocol::Client::ReadTaskResponse, *out); writeVarUInt(DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION, *out); writeStringBinary(response, *out); + out->finishChunk(); out->next(); } @@ -881,7 +978,8 @@ void Connection::sendReadTaskResponse(const String & response) void Connection::sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) { writeVarUInt(Protocol::Client::MergeTreeReadTaskResponse, *out); - response.serialize(*out); + response.serialize(*out, server_parallel_replicas_protocol_version); + out->finishChunk(); out->next(); } @@ -899,6 +997,8 @@ void Connection::sendPreparedData(ReadBuffer & input, size_t size, const String copyData(input, *out); else copyData(input, *out, size); + + out->finishChunk(); out->next(); } @@ -927,6 +1027,8 @@ void Connection::sendScalarsData(Scalars & data) sendData(elem.second, elem.first, true /* scalar */); } + out->finishChunk(); + out_bytes = out->count() - out_bytes; maybe_compressed_out_bytes = maybe_compressed_out->count() - maybe_compressed_out_bytes; double elapsed = watch.elapsedSeconds(); @@ -1069,13 +1171,13 @@ std::optional Connection::getResolvedAddress() const bool Connection::poll(size_t timeout_microseconds) { - return static_cast(*in).poll(timeout_microseconds); + return in->poll(timeout_microseconds); } bool Connection::hasReadPendingData() const { - return last_input_packet_type.has_value() || static_cast(*in).hasPendingData(); + return last_input_packet_type.has_value() || in->hasBufferedData(); } @@ -1330,7 +1432,7 @@ ParallelReadRequest Connection::receiveParallelReadRequest() const InitialAllRangesAnnouncement Connection::receiveInitialParallelReadAnnouncement() const { - return InitialAllRangesAnnouncement::deserialize(*in); + return InitialAllRangesAnnouncement::deserialize(*in, server_parallel_replicas_protocol_version); } @@ -1349,6 +1451,8 @@ ServerConnectionPtr Connection::createConnection(const ConnectionParameters & pa parameters.default_database, parameters.user, parameters.password, + parameters.proto_send_chunked, + parameters.proto_recv_chunked, parameters.ssh_private_key, parameters.jwt, parameters.quota_key, diff --git a/src/Client/Connection.h b/src/Client/Connection.h index 0f4b3e436df2..29939e5d5541 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -8,8 +8,8 @@ #include -#include -#include +#include +#include #include #include @@ -52,6 +52,7 @@ class Connection : public IServerConnection Connection(const String & host_, UInt16 port_, const String & default_database_, const String & user_, const String & password_, + const String & proto_send_chunked_, const String & proto_recv_chunked_, const SSHKey & ssh_private_key_, const String & jwt_, const String & quota_key_, @@ -107,6 +108,7 @@ class Connection : public IServerConnection const Settings * settings/* = nullptr */, const ClientInfo * client_info/* = nullptr */, bool with_pending_data/* = false */, + const std::vector & external_roles, std::function process_progress_callback) override; void sendCancel() override; @@ -170,6 +172,10 @@ class Connection : public IServerConnection String default_database; String user; String password; + String proto_send_chunked; + String proto_recv_chunked; + String proto_send_chunked_srv; + String proto_recv_chunked_srv; #if USE_SSH SSHKey ssh_private_key; #endif @@ -205,12 +211,13 @@ class Connection : public IServerConnection UInt64 server_version_minor = 0; UInt64 server_version_patch = 0; UInt64 server_revision = 0; + UInt64 server_parallel_replicas_protocol_version = 0; String server_timezone; String server_display_name; std::unique_ptr socket; - std::shared_ptr in; - std::shared_ptr out; + std::shared_ptr in; + std::shared_ptr out; std::optional last_input_packet_type; String query_id; diff --git a/src/Client/ConnectionParameters.cpp b/src/Client/ConnectionParameters.cpp index 303bebc30d2d..4d0a9ffa08c2 100644 --- a/src/Client/ConnectionParameters.cpp +++ b/src/Client/ConnectionParameters.cpp @@ -107,6 +107,9 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati } } + proto_send_chunked = config.getString("proto_caps.send", "notchunked"); + proto_recv_chunked = config.getString("proto_caps.recv", "notchunked"); + quota_key = config.getString("quota_key", ""); /// By default compression is disabled if address looks like localhost. diff --git a/src/Client/ConnectionParameters.h b/src/Client/ConnectionParameters.h index c305c7813f2d..382bfe34a3d6 100644 --- a/src/Client/ConnectionParameters.h +++ b/src/Client/ConnectionParameters.h @@ -20,6 +20,8 @@ struct ConnectionParameters std::string default_database; std::string user; std::string password; + std::string proto_send_chunked = "notchunked"; + std::string proto_recv_chunked = "notchunked"; std::string quota_key; SSHKey ssh_private_key; std::string jwt; diff --git a/src/Client/ConnectionPool.cpp b/src/Client/ConnectionPool.cpp index ed2e7c3c7255..ab8ad08826cb 100644 --- a/src/Client/ConnectionPool.cpp +++ b/src/Client/ConnectionPool.cpp @@ -13,6 +13,8 @@ ConnectionPoolPtr ConnectionPoolFactory::get( String default_database, String user, String password, + String proto_send_chunked, + String proto_recv_chunked, String quota_key, String cluster, String cluster_secret, @@ -22,7 +24,7 @@ ConnectionPoolPtr ConnectionPoolFactory::get( Priority priority) { Key key{ - max_connections, host, port, default_database, user, password, quota_key, cluster, cluster_secret, client_name, compression, secure, priority}; + max_connections, host, port, default_database, user, password, proto_send_chunked, proto_recv_chunked, quota_key, cluster, cluster_secret, client_name, compression, secure, priority}; std::lock_guard lock(mutex); auto [it, inserted] = pools.emplace(key, ConnectionPoolPtr{}); @@ -39,6 +41,8 @@ ConnectionPoolPtr ConnectionPoolFactory::get( default_database, user, password, + proto_send_chunked, + proto_recv_chunked, quota_key, cluster, cluster_secret, diff --git a/src/Client/ConnectionPool.h b/src/Client/ConnectionPool.h index 0fcb3c4e7e15..219548b62a04 100644 --- a/src/Client/ConnectionPool.h +++ b/src/Client/ConnectionPool.h @@ -73,6 +73,8 @@ class ConnectionPool : public IConnectionPool, private PoolBase const String & default_database_, const String & user_, const String & password_, + const String & proto_send_chunked_, + const String & proto_recv_chunked_, const String & quota_key_, const String & cluster_, const String & cluster_secret_, @@ -85,6 +87,8 @@ class ConnectionPool : public IConnectionPool, private PoolBase , default_database(default_database_) , user(user_) , password(password_) + , proto_send_chunked(proto_send_chunked_) + , proto_recv_chunked(proto_recv_chunked_) , quota_key(quota_key_) , cluster(cluster_) , cluster_secret(cluster_secret_) @@ -116,7 +120,9 @@ class ConnectionPool : public IConnectionPool, private PoolBase { return std::make_shared( host, port, - default_database, user, password, SSHKey(), /*jwt*/ "", quota_key, + default_database, user, password, + proto_send_chunked, proto_recv_chunked, + SSHKey(), /*jwt*/ "", quota_key, cluster, cluster_secret, client_name, compression, secure); } @@ -125,6 +131,8 @@ class ConnectionPool : public IConnectionPool, private PoolBase String default_database; String user; String password; + String proto_send_chunked; + String proto_recv_chunked; String quota_key; /// For inter-server authorization @@ -150,6 +158,8 @@ class ConnectionPoolFactory final : private boost::noncopyable String default_database; String user; String password; + String proto_send_chunked; + String proto_recv_chunked; String quota_key; String cluster; String cluster_secret; @@ -173,6 +183,8 @@ class ConnectionPoolFactory final : private boost::noncopyable String default_database, String user, String password, + String proto_send_chunked, + String proto_recv_chunked, String quota_key, String cluster, String cluster_secret, @@ -190,6 +202,7 @@ inline bool operator==(const ConnectionPoolFactory::Key & lhs, const ConnectionP { return lhs.max_connections == rhs.max_connections && lhs.host == rhs.host && lhs.port == rhs.port && lhs.default_database == rhs.default_database && lhs.user == rhs.user && lhs.password == rhs.password + && lhs.proto_send_chunked == rhs.proto_send_chunked && lhs.proto_recv_chunked == rhs.proto_recv_chunked && lhs.quota_key == rhs.quota_key && lhs.cluster == rhs.cluster && lhs.cluster_secret == rhs.cluster_secret && lhs.client_name == rhs.client_name && lhs.compression == rhs.compression && lhs.secure == rhs.secure && lhs.priority == rhs.priority; diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index dd8348ea04f1..39d92a075b0d 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -146,7 +146,8 @@ void HedgedConnections::sendQuery( const String & query_id, UInt64 stage, ClientInfo & client_info, - bool with_pending_data) + bool with_pending_data, + const std::vector & external_roles) { std::lock_guard lock(cancel_mutex); @@ -173,7 +174,7 @@ void HedgedConnections::sendQuery( hedged_connections_factory.skipReplicasWithTwoLevelAggregationIncompatibility(); } - auto send_query = [this, timeouts, query, query_id, stage, client_info, with_pending_data](ReplicaState & replica) + auto send_query = [this, timeouts, query, query_id, stage, client_info, with_pending_data, external_roles](ReplicaState & replica) { Settings modified_settings = settings; @@ -203,7 +204,8 @@ void HedgedConnections::sendQuery( modified_settings.set("allow_experimental_analyzer", static_cast(modified_settings.allow_experimental_analyzer)); replica.connection->sendQuery( - timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {}); + timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, external_roles, {}); + replica.change_replica_timeout.setRelative(timeouts.receive_data_timeout); replica.packet_receiver->setTimeout(hedged_connections_factory.getConnectionTimeouts().receive_timeout); }; diff --git a/src/Client/HedgedConnections.h b/src/Client/HedgedConnections.h index 7f538804e5a6..e64f17658d8b 100644 --- a/src/Client/HedgedConnections.h +++ b/src/Client/HedgedConnections.h @@ -90,7 +90,8 @@ class HedgedConnections : public IConnections const String & query_id, UInt64 stage, ClientInfo & client_info, - bool with_pending_data) override; + bool with_pending_data, + const std::vector & external_roles) override; void sendReadTaskResponse(const String &) override { diff --git a/src/Client/IConnections.h b/src/Client/IConnections.h index 09211de53b0a..a521fdd8b004 100644 --- a/src/Client/IConnections.h +++ b/src/Client/IConnections.h @@ -23,7 +23,8 @@ class IConnections : boost::noncopyable const String & query_id, UInt64 stage, ClientInfo & client_info, - bool with_pending_data) = 0; + bool with_pending_data, + const std::vector & external_roles) = 0; virtual void sendReadTaskResponse(const String &) = 0; virtual void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) = 0; diff --git a/src/Client/IServerConnection.h b/src/Client/IServerConnection.h index 6ab4234bca24..f6f62eb90d8f 100644 --- a/src/Client/IServerConnection.h +++ b/src/Client/IServerConnection.h @@ -102,6 +102,7 @@ class IServerConnection : boost::noncopyable const Settings * settings, const ClientInfo * client_info, bool with_pending_data, + const std::vector & external_roles, std::function process_progress_callback) = 0; virtual void sendCancel() = 0; diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 7595a29912b2..0a4e90ecaaa4 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -95,6 +95,7 @@ void LocalConnection::sendQuery( const Settings *, const ClientInfo * client_info, bool, + const std::vector & /*external_roles*/, std::function process_progress_callback) { /// Last query may not have been finished or cancelled due to exception on client side. diff --git a/src/Client/LocalConnection.h b/src/Client/LocalConnection.h index b424c5b5aa3d..ae0e93e40999 100644 --- a/src/Client/LocalConnection.h +++ b/src/Client/LocalConnection.h @@ -114,6 +114,7 @@ class LocalConnection : public IServerConnection, WithContext const Settings * settings/* = nullptr */, const ClientInfo * client_info/* = nullptr */, bool with_pending_data/* = false */, + const std::vector & external_roles, std::function process_progress_callback) override; void sendCancel() override; diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index 244eccf1ed96..484df6154c2f 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -118,7 +118,8 @@ void MultiplexedConnections::sendQuery( const String & query_id, UInt64 stage, ClientInfo & client_info, - bool with_pending_data) + bool with_pending_data, + const std::vector & external_roles) { std::lock_guard lock(cancel_mutex); @@ -171,14 +172,14 @@ void MultiplexedConnections::sendQuery( modified_settings.parallel_replica_offset = i; replica_states[i].connection->sendQuery( - timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {}); + timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, external_roles, {}); } } else { /// Use single replica. replica_states[0].connection->sendQuery( - timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {}); + timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, external_roles, {}); } sent_query = true; diff --git a/src/Client/MultiplexedConnections.h b/src/Client/MultiplexedConnections.h index dec32e52d4f8..4b308dca02e9 100644 --- a/src/Client/MultiplexedConnections.h +++ b/src/Client/MultiplexedConnections.h @@ -36,7 +36,8 @@ class MultiplexedConnections final : public IConnections const String & query_id, UInt64 stage, ClientInfo & client_info, - bool with_pending_data) override; + bool with_pending_data, + const std::vector & external_roles) override; void sendReadTaskResponse(const String &) override; void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) override; diff --git a/src/Client/Suggest.cpp b/src/Client/Suggest.cpp index affd620f83ab..afb98b7d95c1 100644 --- a/src/Client/Suggest.cpp +++ b/src/Client/Suggest.cpp @@ -164,7 +164,7 @@ void Suggest::load(IServerConnection & connection, void Suggest::fetch(IServerConnection & connection, const ConnectionTimeouts & timeouts, const std::string & query, const ClientInfo & client_info) { connection.sendQuery( - timeouts, query, {} /* query_parameters */, "" /* query_id */, QueryProcessingStage::Complete, nullptr, &client_info, false, {}); + timeouts, query, {} /* query_parameters */, "" /* query_id */, QueryProcessingStage::Complete, nullptr, &client_info, false, {} /* external_roles*/, {}); while (true) { diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index d40e320e741c..0e9e0dbb7ac4 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -51,12 +51,12 @@ AsyncLoader::Pool::Pool(const AsyncLoader::PoolInitializer & init) , priority(init.priority) , max_threads(init.max_threads > 0 ? init.max_threads : getNumberOfPhysicalCPUCores()) , thread_pool(std::make_unique( - init.metric_threads, - init.metric_active_threads, - init.metric_scheduled_threads, - /* max_threads = */ std::numeric_limits::max(), // Unlimited number of threads, we do worker management ourselves - /* max_free_threads = */ 0, // We do not require free threads - /* queue_size = */0)) // Unlimited queue to avoid blocking during worker spawning + init.metric_threads, + init.metric_active_threads, + init.metric_scheduled_threads, + /* max_threads = */ ThreadPool::MAX_THEORETICAL_THREAD_COUNT, // Unlimited number of threads, we do worker management ourselves + /* max_free_threads = */ 0, // We do not require free threads + /* queue_size = */ 0)) // Unlimited queue to avoid blocking during worker spawning {} AsyncLoader::Pool::Pool(Pool&& o) noexcept diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 23ef44f3b323..adc074738786 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -75,9 +75,9 @@ M(GlobalThread, "Number of threads in global thread pool.") \ M(GlobalThreadActive, "Number of threads in global thread pool running a task.") \ M(GlobalThreadScheduled, "Number of queued or active jobs in global thread pool.") \ - M(LocalThread, "Number of threads in local thread pools. The threads in local thread pools are taken from the global thread pool.") \ - M(LocalThreadActive, "Number of threads in local thread pools running a task.") \ - M(LocalThreadScheduled, "Number of queued or active jobs in local thread pools.") \ + M(LocalThread, "Obsolete. Number of threads in local thread pools. The threads in local thread pools are taken from the global thread pool.") \ + M(LocalThreadActive, "Obsolete. Number of threads in local thread pools running a task.") \ + M(LocalThreadScheduled, "Obsolete. Number of queued or active jobs in local thread pools.") \ M(MergeTreeDataSelectExecutorThreads, "Number of threads in the MergeTreeDataSelectExecutor thread pool.") \ M(MergeTreeDataSelectExecutorThreadsActive, "Number of threads in the MergeTreeDataSelectExecutor thread pool running a task.") \ M(MergeTreeDataSelectExecutorThreadsScheduled, "Number of queued or active jobs in the MergeTreeDataSelectExecutor thread pool.") \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 91eb0959d6cf..894a8667f28a 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -86,6 +86,20 @@ M(NetworkReceiveBytes, "Total number of bytes received from network. Only ClickHouse-related network interaction is included, not by 3rd party libraries.") \ M(NetworkSendBytes, "Total number of bytes send to network. Only ClickHouse-related network interaction is included, not by 3rd party libraries.") \ \ + M(GlobalThreadPoolExpansions, "Counts the total number of times new threads have been added to the global thread pool. This metric indicates the frequency of expansions in the global thread pool to accommodate increased processing demands.") \ + M(GlobalThreadPoolShrinks, "Counts the total number of times the global thread pool has shrunk by removing threads. This occurs when the number of idle threads exceeds max_thread_pool_free_size, indicating adjustments in the global thread pool size in response to decreased thread utilization.") \ + M(GlobalThreadPoolThreadCreationMicroseconds, "Total time spent waiting for new threads to start.") \ + M(GlobalThreadPoolLockWaitMicroseconds, "Total time threads have spent waiting for locks in the global thread pool.") \ + M(GlobalThreadPoolJobs, "Counts the number of jobs that have been pushed to the global thread pool.") \ + M(GlobalThreadPoolJobWaitTimeMicroseconds, "Measures the elapsed time from when a job is scheduled in the thread pool to when it is picked up for execution by a worker thread. This metric helps identify delays in job processing, indicating the responsiveness of the thread pool to new tasks.") \ + M(LocalThreadPoolExpansions, "Counts the total number of times threads have been borrowed from the global thread pool to expand local thread pools.") \ + M(LocalThreadPoolShrinks, "Counts the total number of times threads have been returned to the global thread pool from local thread pools.") \ + M(LocalThreadPoolThreadCreationMicroseconds, "Total time local thread pools have spent waiting to borrow a thread from the global pool.") \ + M(LocalThreadPoolLockWaitMicroseconds, "Total time threads have spent waiting for locks in the local thread pools.") \ + M(LocalThreadPoolJobs, "Counts the number of jobs that have been pushed to the local thread pools.") \ + M(LocalThreadPoolBusyMicroseconds, "Total time threads have spent executing the actual work.") \ + M(LocalThreadPoolJobWaitTimeMicroseconds, "Measures the elapsed time from when a job is scheduled in the thread pool to when it is picked up for execution by a worker thread. This metric helps identify delays in job processing, indicating the responsiveness of the thread pool to new tasks.") \ + \ M(DiskS3GetRequestThrottlerCount, "Number of DiskS3 GET and SELECT requests passed through throttler.") \ M(DiskS3GetRequestThrottlerSleepMicroseconds, "Total time a query was sleeping to conform DiskS3 GET and SELECT request throttling.") \ M(DiskS3PutRequestThrottlerCount, "Number of DiskS3 PUT, COPY, POST and LIST requests passed through throttler.") \ @@ -351,10 +365,13 @@ The server successfully detected this situation and will download merged part fr \ M(ParallelReplicasHandleRequestMicroseconds, "Time spent processing requests for marks from replicas") \ M(ParallelReplicasHandleAnnouncementMicroseconds, "Time spent processing replicas announcements") \ + M(ParallelReplicasAnnouncementMicroseconds, "Time spent to send an announcement") \ + M(ParallelReplicasReadRequestMicroseconds, "Time spent for read requests") \ \ M(ParallelReplicasReadAssignedMarks, "Sum across all replicas of how many of scheduled marks were assigned by consistent hash") \ M(ParallelReplicasReadUnassignedMarks, "Sum across all replicas of how many unassigned marks were scheduled") \ M(ParallelReplicasReadAssignedForStealingMarks, "Sum across all replicas of how many of scheduled marks were assigned for stealing by consistent hash") \ + M(ParallelReplicasReadMarks, "How many marks were read by the given replica") \ \ M(ParallelReplicasStealingByHashMicroseconds, "Time spent collecting segments meant for stealing by hash") \ M(ParallelReplicasProcessingPartsMicroseconds, "Time spent processing data parts") \ @@ -508,6 +525,7 @@ The server successfully detected this situation and will download merged part fr M(CachedReadBufferReadFromCacheMicroseconds, "Time reading from filesystem cache") \ M(CachedReadBufferReadFromSourceBytes, "Bytes read from filesystem cache source (from remote fs, etc)") \ M(CachedReadBufferReadFromCacheBytes, "Bytes read from filesystem cache") \ + M(CachedReadBufferPredownloadedBytes, "Bytes read from filesystem cache source. Cache segments are read from left to right as a whole, it might be that we need to predownload some part of the segment irrelevant for the current task just to get to the needed data") \ M(CachedReadBufferCacheWriteBytes, "Bytes written from source (remote fs, etc) to filesystem cache") \ M(CachedReadBufferCacheWriteMicroseconds, "Time spent writing data into filesystem cache") \ M(CachedReadBufferCreateBufferMicroseconds, "Prepare buffer time") \ diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index c8f1ae999698..5dfefcbfccf1 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -27,6 +28,66 @@ namespace CurrentMetrics extern const Metric GlobalThreadScheduled; } +namespace ProfileEvents +{ + extern const Event GlobalThreadPoolExpansions; + extern const Event GlobalThreadPoolShrinks; + extern const Event GlobalThreadPoolThreadCreationMicroseconds; + extern const Event GlobalThreadPoolLockWaitMicroseconds; + extern const Event GlobalThreadPoolJobs; + extern const Event GlobalThreadPoolJobWaitTimeMicroseconds; + + extern const Event LocalThreadPoolExpansions; + extern const Event LocalThreadPoolShrinks; + extern const Event LocalThreadPoolThreadCreationMicroseconds; + extern const Event LocalThreadPoolLockWaitMicroseconds; + extern const Event LocalThreadPoolJobs; + extern const Event LocalThreadPoolBusyMicroseconds; + extern const Event LocalThreadPoolJobWaitTimeMicroseconds; + +} + +namespace +{ + struct ScopedDecrement + { + std::optional>> atomic_var; + + // Deleted copy constructor and copy assignment operator + ScopedDecrement(const ScopedDecrement&) = delete; + ScopedDecrement& operator=(const ScopedDecrement&) = delete; + + // Move constructor + ScopedDecrement(ScopedDecrement&& other) noexcept + : atomic_var(std::move(other.atomic_var)) + { + other.atomic_var.reset(); + } + + // Move assignment operator + ScopedDecrement& operator=(ScopedDecrement&& other) noexcept + { + if (this != &other) + { + atomic_var.swap(other.atomic_var); + } + return *this; + } + + explicit ScopedDecrement(std::atomic& var) + : atomic_var(var) + { + atomic_var->get().fetch_sub(1, std::memory_order_relaxed); + } + + ~ScopedDecrement() + { + if (atomic_var) + atomic_var->get().fetch_add(1, std::memory_order_relaxed); + } + }; +} + class JobWithPriority { public: @@ -35,17 +96,29 @@ class JobWithPriority Job job; Priority priority; CurrentMetrics::Increment metric_increment; + ScopedDecrement available_threads_decrement; + DB::OpenTelemetry::TracingContextOnThread thread_trace_context; /// Call stacks of all jobs' schedulings leading to this one std::vector frame_pointers; bool enable_job_stack_trace = false; + Stopwatch job_create_time; + + // Deleted copy constructor and copy assignment operator + JobWithPriority(const JobWithPriority&) = delete; + JobWithPriority& operator=(const JobWithPriority&) = delete; + + // Move constructor and move assignment operator + JobWithPriority(JobWithPriority&&) noexcept = default; + JobWithPriority& operator=(JobWithPriority&&) noexcept = default; JobWithPriority( Job job_, Priority priority_, CurrentMetrics::Metric metric, const DB::OpenTelemetry::TracingContextOnThread & thread_trace_context_, - bool capture_frame_pointers) + bool capture_frame_pointers, ScopedDecrement available_threads_decrement_) : job(job_), priority(priority_), metric_increment(metric), + available_threads_decrement(std::move(available_threads_decrement_)), thread_trace_context(thread_trace_context_), enable_job_stack_trace(capture_frame_pointers) { if (!capture_frame_pointers) @@ -59,6 +132,11 @@ class JobWithPriority { return priority > rhs.priority; // Reversed for `priority_queue` max-heap to yield minimum value (i.e. highest priority) first } + + UInt64 elapsedMicroseconds() const + { + return job_create_time.elapsedMicroseconds(); + } }; static constexpr auto DEFAULT_THREAD_NAME = "ThreadPool"; @@ -97,12 +175,19 @@ ThreadPoolImpl::ThreadPoolImpl( , queue_size(queue_size_ ? std::max(queue_size_, max_threads) : 0 /* zero means the queue is unlimited */) , shutdown_on_exception(shutdown_on_exception_) { + max_threads = std::min(max_threads, static_cast(MAX_THEORETICAL_THREAD_COUNT)); + max_free_threads = std::min(max_free_threads, static_cast(MAX_THEORETICAL_THREAD_COUNT)); + remaining_pool_capacity.store(max_threads, std::memory_order_relaxed); + available_threads.store(0, std::memory_order_relaxed); } template void ThreadPoolImpl::setMaxThreads(size_t value) { + value = std::min(value, static_cast(MAX_THEORETICAL_THREAD_COUNT)); std::lock_guard lock(mutex); + remaining_pool_capacity.fetch_add(value - max_threads, std::memory_order_relaxed); + bool need_start_threads = (value > max_threads); bool need_finish_free_threads = (value < max_free_threads); @@ -135,6 +220,7 @@ size_t ThreadPoolImpl::getMaxThreads() const template void ThreadPoolImpl::setMaxFreeThreads(size_t value) { + value = std::min(value, static_cast(MAX_THEORETICAL_THREAD_COUNT)); std::lock_guard lock(mutex); bool need_finish_free_threads = (value < max_free_threads); @@ -156,7 +242,6 @@ void ThreadPoolImpl::setQueueSize(size_t value) jobs.reserve(queue_size); } - template template ReturnType ThreadPoolImpl::scheduleImpl(Job job, Priority priority, std::optional wait_microseconds, bool propagate_opentelemetry_tracing_context) @@ -179,15 +264,52 @@ ReturnType ThreadPoolImpl::scheduleImpl(Job job, Priority priority, std: return false; }; + // Decrement available_threads, scoped to the job lifecycle. + // This ensures that available_threads decreases when a new job starts + // and automatically increments when the job completes or goes out of scope. + ScopedDecrement available_threads_decrement(available_threads); + + std::unique_ptr new_thread; + + // Load the current capacity + int64_t capacity = remaining_pool_capacity.load(std::memory_order_relaxed); + int64_t currently_available_threads = available_threads.load(std::memory_order_relaxed); + + while (currently_available_threads <= 0 && capacity > 0) { + if (remaining_pool_capacity.compare_exchange_weak(capacity, capacity - 1, std::memory_order_relaxed)) + { + try + { + new_thread = std::make_unique(*this); + break; // Exit the loop once a thread is successfully created. + } + catch (...) + { + // Failed to create the thread, restore capacity + remaining_pool_capacity.fetch_add(1, std::memory_order_relaxed); + std::lock_guard lock(mutex); // needed to change first_exception. + return on_error("failed to start the thread"); + } + } + // capacity gets reloaded by (unsuccessful) compare_exchange_weak + currently_available_threads = available_threads.load(std::memory_order_relaxed); + } + + { + Stopwatch watch; std::unique_lock lock(mutex); + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolLockWaitMicroseconds : ProfileEvents::LocalThreadPoolLockWaitMicroseconds, + watch.elapsedMicroseconds()); if (CannotAllocateThreadFaultInjector::injectFault()) return on_error("fault injected"); auto pred = [this] { return !queue_size || scheduled_jobs < queue_size || shutdown; }; - if (wait_microseconds) /// Check for optional. Condition is true if the optional is set and the value is zero. + /// Wait for available threads or timeout + if (wait_microseconds) /// Check for optional. Condition is true if the optional is set. Even if the value is zero. { if (!job_finished.wait_for(lock, std::chrono::microseconds(*wait_microseconds), pred)) return on_error(fmt::format("no free thread (timeout={})", *wait_microseconds)); @@ -198,47 +320,97 @@ ReturnType ThreadPoolImpl::scheduleImpl(Job job, Priority priority, std: if (shutdown) return on_error("shutdown"); - /// We must not to allocate any memory after we emplaced a job in a queue. - /// Because if an exception would be thrown, we won't notify a thread about job occurrence. - - /// Check if there are enough threads to process job. - if (threads.size() < std::min(max_threads, scheduled_jobs + 1)) + /// We must not allocate memory or perform operations that could throw exceptions after adding a job to the queue, + /// because if an exception occurs, it may leave the job in the queue without notifying any threads. + typename ThreadFromThreadPool::ThreadList::iterator thread_slot; + + /// The decision to start a new thread is made outside the locked section. + /// However, thread load and demand can change dynamically, and decisions based on + /// atomic variables outside the critical section might become outdated by the time we acquire the lock. + /// This can lead to two possible scenarios: + /// + /// 1) Relatively common: A new thread was started outside the lock, but by the time we acquire the lock, + /// demand for threads has decreased (e.g., other threads have finished their jobs and are now idle). + /// In this case, even though there are now enough threads, we still attempt to add the new thread + /// to the pool, provided it does not exceed the `max_threads` or `max_free_threads` limits. Keeping + /// an extra thread in the pool may help accommodate a sudden increase in demand without the need + /// to wait for thread creation. + /// + /// 2) Very unlikely (but possible): Outside the lock, it appeared there were enough threads + /// to handle the workload. However, after acquiring the lock, it turns out the new thread + /// is needed (possibly because one of the existing threads was removed or became unavailable). + /// In this case, we create the thread inside the critical section, even though this may introduce + /// a small delay. + + /// Check if we can add the thread created outside the critical section to the pool. + bool adding_new_thread = new_thread && threads.size() < std::min(max_threads, 1 /* current job */ + scheduled_jobs + max_free_threads); + + // If we didn't create a new thread initially but realize we actually need one (unlikely scenario). + if (unlikely(!adding_new_thread && threads.size() < std::min(max_threads, scheduled_jobs + 1))) { try { - threads.emplace_front(); + remaining_pool_capacity.fetch_sub(1, std::memory_order_relaxed); + new_thread = std::make_unique(*this); } catch (...) { - /// Most likely this is a std::bad_alloc exception - return on_error("cannot allocate thread slot"); + // If thread creation fails, restore the pool capacity and return an error. + remaining_pool_capacity.fetch_add(1, std::memory_order_relaxed); + return on_error("failed to start the thread"); } + adding_new_thread = true; + } + if (adding_new_thread) + { try { - threads.front() = Thread([this, it = threads.begin()] { worker(it); }); + threads.emplace_front(std::move(new_thread)); + thread_slot = threads.begin(); } catch (...) { - threads.pop_front(); - return on_error("cannot allocate thread"); + /// Most likely this is a std::bad_alloc exception + return on_error("cannot emplace the thread in the pool"); } } + else // we have a thread but there is no space for that in the pool. + { + new_thread.reset(); + } - jobs.emplace(std::move(job), - priority, - metric_scheduled_jobs, - /// Tracing context on this thread is used as parent context for the sub-thread that runs the job - propagate_opentelemetry_tracing_context ? DB::OpenTelemetry::CurrentContext() : DB::OpenTelemetry::TracingContextOnThread(), - /// capture_frame_pointers - DB::Exception::enable_job_stack_trace); + try + { + jobs.emplace(std::move(job), + priority, + metric_scheduled_jobs, + /// Tracing context on this thread is used as parent context for the sub-thread that runs the job + propagate_opentelemetry_tracing_context ? DB::OpenTelemetry::CurrentContext() : DB::OpenTelemetry::TracingContextOnThread(), + /// capture_frame_pointers + DB::Exception::enable_job_stack_trace, + std::move(available_threads_decrement)); - ++scheduled_jobs; + ++scheduled_jobs; + + if (adding_new_thread) + (*thread_slot)->start(thread_slot); + + } + catch (...) + { + if (adding_new_thread) + threads.pop_front(); + + return on_error("cannot start the job or thread"); + } } /// Wake up a free thread to run the new job. new_job_or_shutdown.notify_one(); + ProfileEvents::increment(std::is_same_v ? ProfileEvents::GlobalThreadPoolJobs : ProfileEvents::LocalThreadPoolJobs); + return static_cast(true); } @@ -251,23 +423,51 @@ void ThreadPoolImpl::startNewThreadsNoLock() /// Start new threads while there are more scheduled jobs in the queue and the limit `max_threads` is not reached. while (threads.size() < std::min(scheduled_jobs, max_threads)) { + std::unique_ptr new_thread; + + int64_t capacity = remaining_pool_capacity.load(std::memory_order_relaxed); + + while (capacity > 0) + { + if (remaining_pool_capacity.compare_exchange_weak(capacity, capacity - 1, std::memory_order_relaxed)) + { + try + { + // Successfully decremented, attempt to create a new thread + new_thread = std::make_unique(*this); + } + catch (...) + { + // Failed to create the thread, restore capacity + remaining_pool_capacity.fetch_add(1, std::memory_order_relaxed); + } + break; // Exit loop whether thread creation succeeded or not + } + } + + if (!new_thread) + break; /// failed to start more threads + + typename ThreadFromThreadPool::ThreadList::iterator thread_slot; + try { - threads.emplace_front(); + threads.emplace_front(std::move(new_thread)); + thread_slot = threads.begin(); } catch (...) { - break; /// failed to start more threads + break; } try { - threads.front() = Thread([this, it = threads.begin()] { worker(it); }); + (*thread_slot)->start(thread_slot); } catch (...) { threads.pop_front(); - break; /// failed to start more threads + break; } } } @@ -293,7 +493,11 @@ void ThreadPoolImpl::scheduleOrThrow(Job job, Priority priority, uint64_ template void ThreadPoolImpl::wait() { + Stopwatch watch; std::unique_lock lock(mutex); + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolLockWaitMicroseconds : ProfileEvents::LocalThreadPoolLockWaitMicroseconds, + watch.elapsedMicroseconds()); /// Signal here just in case. /// If threads are waiting on condition variables, but there are some jobs in the queue /// then it will prevent us from deadlock. @@ -325,17 +529,29 @@ void ThreadPoolImpl::finalize() { std::lock_guard lock(mutex); shutdown = true; - /// We don't want threads to remove themselves from `threads` anymore, otherwise `thread.join()` will go wrong below in this function. + + /// scheduleImpl doesn't check for shutdown outside the critical section, + /// so we set remaining_pool_capacity to a large negative value + /// (e.g., -MAX_THEORETICAL_THREAD_COUNT) to signal that no new threads are needed. + /// This effectively prevents any new threads from being started during shutdown. + remaining_pool_capacity.store(-MAX_THEORETICAL_THREAD_COUNT, std::memory_order_relaxed); + + /// Disable thread self-removal from `threads`. Otherwise, if threads remove themselves, + /// the thread.join() operation will fail later in this function. threads_remove_themselves = false; } - /// Wake up threads so they can finish themselves. + /// Notify all threads to wake them up, so they can complete their work and exit gracefully. new_job_or_shutdown.notify_all(); - /// Wait for all currently running jobs to finish (we don't wait for all scheduled jobs here like the function wait() does). - for (auto & thread : threads) - thread.join(); + /// Join all threads before clearing the list + for (auto& thread_ptr : threads) + { + if (thread_ptr) + thread_ptr->join(); + } + // now it's safe to clear the threads threads.clear(); } @@ -371,11 +587,88 @@ bool ThreadPoolImpl::finished() const return shutdown; } + +template +ThreadPoolImpl::ThreadFromThreadPool::ThreadFromThreadPool(ThreadPoolImpl& parent_pool_) + : parent_pool(parent_pool_) + , thread_state(ThreadState::Preparing) // Initial state is Preparing +{ + Stopwatch watch2; + + thread = Thread(&ThreadFromThreadPool::worker, this); + + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolThreadCreationMicroseconds : ProfileEvents::LocalThreadPoolThreadCreationMicroseconds, + watch2.elapsedMicroseconds()); + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolExpansions : ProfileEvents::LocalThreadPoolExpansions); + + parent_pool.available_threads.fetch_add(1, std::memory_order_relaxed); +} + + +template +void ThreadPoolImpl::ThreadFromThreadPool::start(typename ThreadList::iterator & it) +{ + /// the thread which created ThreadFromThreadPool should start it after adding it to the pool, or destroy it. + /// no parallelism is expected here. So the only valid transition for the start method is Preparing to Running. + chassert(thread_state.load(std::memory_order_relaxed) == ThreadState::Preparing); + thread_it = it; + thread_state.store(ThreadState::Running, std::memory_order_relaxed); /// now worker can start executing the main loop +} + +template +void ThreadPoolImpl::ThreadFromThreadPool::join() +{ + // Ensure the thread is joined before destruction if still joinable + if (thread.joinable()) + thread.join(); +} + template -void ThreadPoolImpl::worker(typename std::list::iterator thread_it) +void ThreadPoolImpl::ThreadFromThreadPool::removeSelfFromPoolNoPoolLock() +{ + if (thread.joinable()) + thread.detach(); + + parent_pool.threads.erase(thread_it); +} + +template +ThreadPoolImpl::ThreadFromThreadPool::~ThreadFromThreadPool() +{ + parent_pool.available_threads.fetch_sub(1, std::memory_order_relaxed); + + // The thread is being destructed, so the remaining pool capacity increases + parent_pool.remaining_pool_capacity.fetch_add(1, std::memory_order_relaxed); + + // If the worker was still waiting in the loop for thread initialization, + // signal it to terminate and be destroyed now. + thread_state.store(ThreadState::Destructing, std::memory_order_relaxed); + + join(); + + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolShrinks : ProfileEvents::LocalThreadPoolShrinks); +} + + +template +void ThreadPoolImpl::ThreadFromThreadPool::worker() { DENY_ALLOCATIONS_IN_SCOPE; - CurrentMetrics::Increment metric_pool_threads(metric_threads); + + // wait until the thread will be started + while (thread_state.load(std::memory_order_relaxed) == ThreadState::Preparing) + { + std::this_thread::yield(); // let's try to yield to avoid consuming too much CPU in the busy-loop + } + + // If the thread transitions to Destructing, exit + if (thread_state.load(std::memory_order_relaxed) == ThreadState::Destructing) + return; + + CurrentMetrics::Increment metric_pool_threads(parent_pool.metric_threads); bool job_is_done = false; std::exception_ptr exception_from_job; @@ -391,7 +684,11 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ std::optional job_data; { - std::unique_lock lock(mutex); + Stopwatch watch; + std::unique_lock lock(parent_pool.mutex); + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolLockWaitMicroseconds : ProfileEvents::LocalThreadPoolLockWaitMicroseconds, + watch.elapsedMicroseconds()); // Finish with previous job if any if (job_is_done) @@ -399,42 +696,55 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ job_is_done = false; if (exception_from_job) { - if (!first_exception) - first_exception = exception_from_job; - if (shutdown_on_exception) - shutdown = true; + if (!parent_pool.first_exception) + parent_pool.first_exception = exception_from_job; + if (parent_pool.shutdown_on_exception) + { + parent_pool.shutdown = true; + + // Prevent new thread creation, as explained in finalize. + parent_pool.remaining_pool_capacity.store(-MAX_THEORETICAL_THREAD_COUNT, std::memory_order_relaxed); + } exception_from_job = {}; } - --scheduled_jobs; + --parent_pool.scheduled_jobs; - job_finished.notify_all(); - if (shutdown) - new_job_or_shutdown.notify_all(); /// `shutdown` was set, wake up other threads so they can finish themselves. + parent_pool.job_finished.notify_all(); + if (parent_pool.shutdown) + parent_pool.new_job_or_shutdown.notify_all(); /// `shutdown` was set, wake up other threads so they can finish themselves. } - new_job_or_shutdown.wait(lock, [&] { return !jobs.empty() || shutdown || threads.size() > std::min(max_threads, scheduled_jobs + max_free_threads); }); + parent_pool.new_job_or_shutdown.wait(lock, [this] { + return !parent_pool.jobs.empty() + || parent_pool.shutdown + || parent_pool.threads.size() > std::min(parent_pool.max_threads, parent_pool.scheduled_jobs + parent_pool.max_free_threads); + }); - if (jobs.empty() || threads.size() > std::min(max_threads, scheduled_jobs + max_free_threads)) + + if (parent_pool.jobs.empty() || parent_pool.threads.size() > std::min(parent_pool.max_threads, parent_pool.scheduled_jobs + parent_pool.max_free_threads)) { // We enter here if: // - either this thread is not needed anymore due to max_free_threads excess; // - or shutdown happened AND all jobs are already handled. - if (threads_remove_themselves) - { - thread_it->detach(); - threads.erase(thread_it); - } + + if (parent_pool.threads_remove_themselves) + removeSelfFromPoolNoPoolLock(); // Detach and remove itself from the pool + return; } /// boost::priority_queue does not provide interface for getting non-const reference to an element /// to prevent us from modifying its priority. We have to use const_cast to force move semantics on JobWithPriority. - job_data = std::move(const_cast(jobs.top())); - jobs.pop(); + job_data = std::move(const_cast(parent_pool.jobs.top())); + parent_pool.jobs.pop(); + + ProfileEvents::increment( + std::is_same_v ? ProfileEvents::GlobalThreadPoolJobWaitTimeMicroseconds : ProfileEvents::LocalThreadPoolJobWaitTimeMicroseconds, + job_data->elapsedMicroseconds()); /// We don't run jobs after `shutdown` is set, but we have to properly dequeue all jobs and finish them. - if (shutdown) + if (parent_pool.shutdown) { { ALLOW_ALLOCATIONS_IN_SCOPE; @@ -457,9 +767,24 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ if (DB::Exception::enable_job_stack_trace) DB::Exception::setThreadFramePointers(std::move(job_data->frame_pointers)); - CurrentMetrics::Increment metric_active_pool_threads(metric_active_threads); + CurrentMetrics::Increment metric_active_pool_threads(parent_pool.metric_active_threads); + + if constexpr (!std::is_same_v) + { + Stopwatch watch; + job_data->job(); + // This metric is less relevant for the global thread pool, as it would show large values (time while + // a thread was used by local pools) and increment only when local pools are destroyed. + // + // In cases where global pool threads are used directly (without a local thread pool), distinguishing + // them is difficult. + ProfileEvents::increment(ProfileEvents::LocalThreadPoolBusyMicroseconds, watch.elapsedMicroseconds()); + } + else + { + job_data->job(); + } - job_data->job(); if (thread_trace_context.root_span.isTraceEnabled()) { @@ -495,7 +820,6 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ } } - template class ThreadPoolImpl; template class ThreadPoolImpl>; template class ThreadPoolImpl>; diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index a31d793264e9..7e497245acc3 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -32,7 +32,7 @@ class JobWithPriority; * * This thread pool can be used as a task queue. * For example, you can create a thread pool with 10 threads (and queue of size 10) and schedule 1000 tasks - * - in this case you will be blocked to keep 10 tasks in fly. + * - in this case you will be blocked to keep 10 tasks in flight. * * Thread: std::thread or something with identical interface. */ @@ -40,9 +40,57 @@ template class ThreadPoolImpl { public: + // used as 'unlimited' thread pool size + // on linux you can not have more threads even if the RAM is unlimited + // see https://docs.kernel.org/admin-guide/sysctl/kernel.html#threads-max + static constexpr int MAX_THEORETICAL_THREAD_COUNT = 0x3fffffff; // ~1 billion + using Job = std::function; using Metric = CurrentMetrics::Metric; + // Subclass that encapsulates the thread and has the ability to remove itself from the pool. + class ThreadFromThreadPool + { + public: + using ThreadList = std::list>; + + /// Constructor to initialize and start the thread (but not associate it with the pool) + explicit ThreadFromThreadPool(ThreadPoolImpl& parent_pool); + + // Shift the thread state from Preparing to Running to allow the worker to start. + void start(ThreadList::iterator& it); + + void join(); + + // Destructor to join the thread if needed (shift the state to Destructing if it was not running) + ~ThreadFromThreadPool(); + + private: + ThreadPoolImpl& parent_pool; + Thread thread; + + enum class ThreadState + { + Preparing, + Running, + Destructing + }; + + // Atomic state to track the thread's state + std::atomic thread_state; + + // Stores the position of the thread in the parent thread pool list + typename std::list>::iterator thread_it; + + // Remove itself from the parent pool + void removeSelfFromPoolNoPoolLock(); + + // Worker does a busy loop (with yield) while the state is Preparing. + // After that, immediately returns if the state changed to Destructing, + // or starts the main working loop if the state is Running. + void worker(); + }; + /// Maximum number of threads is based on the number of physical cores. ThreadPoolImpl(Metric metric_threads_, Metric metric_active_threads_, Metric metric_scheduled_jobs_); @@ -63,14 +111,14 @@ class ThreadPoolImpl size_t queue_size_, bool shutdown_on_exception_ = true); - /// Add new job. Locks until number of scheduled jobs is less than maximum or exception in one of threads was thrown. - /// If any thread was throw an exception, first exception will be rethrown from this method, - /// and exception will be cleared. + /// Add new job. Locks until the number of scheduled jobs is less than the maximum or an exception in one of the threads was thrown. + /// If any thread has thrown an exception, the first exception will be rethrown from this method, + /// and the exception will be cleared. /// Also throws an exception if cannot create thread. /// Priority: lower is higher. - /// NOTE: Probably you should call wait() if exception was thrown. If some previously scheduled jobs are using some objects, - /// located on stack of current thread, the stack must not be unwinded until all jobs finished. However, - /// if ThreadPool is a local object, it will wait for all scheduled jobs in own destructor. + /// NOTE: Probably you should call wait() if an exception was thrown. If some previously scheduled jobs are using some objects, + /// located on the stack of the current thread, the stack must not be unwound until all jobs are finished. However, + /// if ThreadPool is a local object, it will wait for all scheduled jobs in its own destructor. void scheduleOrThrowOnError(Job job, Priority priority = {}); /// Similar to scheduleOrThrowOnError(...). Wait for specified amount of time and schedule a job or return false. @@ -81,12 +129,12 @@ class ThreadPoolImpl /// Wait for all currently active jobs to be done. /// You may call schedule and wait many times in arbitrary order. - /// If any thread was throw an exception, first exception will be rethrown from this method, - /// and exception will be cleared. + /// If any thread has thrown an exception, the first exception will be rethrown from this method, + /// and the exception will be cleared. void wait(); /// Waits for all threads. Doesn't rethrow exceptions (use 'wait' method to rethrow exceptions). - /// You should not destroy object while calling schedule or wait methods from another threads. + /// You should not destroy the object while calling schedule or wait methods from other threads. ~ThreadPoolImpl(); /// Returns number of running and scheduled jobs. @@ -127,28 +175,40 @@ class ThreadPoolImpl size_t queue_size; size_t scheduled_jobs = 0; + + // Originally equals to max_threads, but changes dynamically. + // Decrements with every new thread started, increments when it finishes. + // If positive, then more threads can be started. + // When it comes to zero, it means that max_threads threads have already been started. + // it can be below zero when the threadpool is shutting down + std::atomic remaining_pool_capacity; + + // Increments every time a new thread joins the thread pool or a job finishes. + // Decrements every time a task is scheduled. + // If positive, it means that there are more threads than jobs (and some are idle). + // If zero, it means that every thread has a job. + // If negative, it means that we have more jobs than threads. + std::atomic available_threads; + bool shutdown = false; bool threads_remove_themselves = true; const bool shutdown_on_exception = true; - boost::heap::priority_queue jobs; - std::list threads; + boost::heap::priority_queue> jobs; + ThreadFromThreadPool::ThreadList threads; std::exception_ptr first_exception; std::stack on_destroy_callbacks; template ReturnType scheduleImpl(Job job, Priority priority, std::optional wait_microseconds, bool propagate_opentelemetry_tracing_context = true); - void worker(typename std::list::iterator thread_it); - - /// Tries to start new threads if there are scheduled jobs and the limit `max_threads` is not reached. Must be called with `mutex` locked. + /// Tries to start new threads if there are scheduled jobs and the limit `max_threads` is not reached. Must be called with the mutex locked. void startNewThreadsNoLock(); void finalize(); void onDestroy(); }; - /// ThreadPool with std::thread for threads. using FreeThreadPool = ThreadPoolImpl; diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index f09ea56391a6..95f64c817bd0 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -28,6 +28,16 @@ #include #include +#if USE_SSL +# include +# include +# include +# include +# include +# include +# include +#endif + #include #include #include @@ -44,10 +54,10 @@ namespace DB namespace ErrorCodes { extern const int RAFT_ERROR; - extern const int NO_ELEMENTS_IN_CONFIG; extern const int SUPPORT_IS_DISABLED; extern const int LOGICAL_ERROR; extern const int INVALID_CONFIG_PARAMETER; + extern const int OPENSSL_ERROR; } using namespace std::chrono_literals; @@ -56,31 +66,102 @@ namespace { #if USE_SSL -void setSSLParams(nuraft::asio_service::options & asio_opts) + +auto getSslContextProvider(const Poco::Util::AbstractConfiguration & config, std::string_view key) { - const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); - String certificate_file_property = "openSSL.server.certificateFile"; - String private_key_file_property = "openSSL.server.privateKeyFile"; - String root_ca_file_property = "openSSL.server.caConfig"; + String load_default_ca_file_property = fmt::format("openSSL.{}.loadDefaultCAFile", key); + String verification_mode_property = fmt::format("openSSL.{}.verificationMode", key); + String root_ca_file_property = fmt::format("openSSL.{}.caConfig", key); + String private_key_passphrase_property = fmt::format("openSSL.{}.privateKeyPassphraseHandler.options.password", key); + + Poco::Net::Context::Params params; + String certificate_file_property = fmt::format("openSSL.{}.certificateFile", key); + String private_key_file_property = fmt::format("openSSL.{}.privateKeyFile", key); + if (config.has(certificate_file_property)) + params.certificateFile = config.getString(certificate_file_property); + + if (config.has(private_key_file_property)) + params.privateKeyFile = config.getString(private_key_file_property); + + std::shared_ptr certificate_data; + if (config.has(private_key_passphrase_property)) + { + certificate_data = std::make_shared( + params.certificateFile, params.privateKeyFile, config.getString(private_key_passphrase_property)); + params.certificateFile.clear(); + params.privateKeyFile.clear(); + } - if (!config.has(certificate_file_property)) - throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server certificate file is not set."); + if (config.has(root_ca_file_property)) + params.caLocation = config.getString(root_ca_file_property); - if (!config.has(private_key_file_property)) - throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server private key file is not set."); + params.loadDefaultCAs = config.getBool(load_default_ca_file_property, false); + params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString(verification_mode_property, "none")); - asio_opts.enable_ssl_ = true; - asio_opts.server_cert_file_ = config.getString(certificate_file_property); - asio_opts.server_key_file_ = config.getString(private_key_file_property); + std::string disabled_protocols_list = config.getString(fmt::format("openSSL.{}.disableProtocols", key), ""); + Poco::StringTokenizer dp_tok(disabled_protocols_list, ";,", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY); + int disabled_protocols = 0; + for (const auto & token : dp_tok) + { + if (token == "sslv2") + disabled_protocols |= Poco::Net::Context::PROTO_SSLV2; + else if (token == "sslv3") + disabled_protocols |= Poco::Net::Context::PROTO_SSLV3; + else if (token == "tlsv1") + disabled_protocols |= Poco::Net::Context::PROTO_TLSV1; + else if (token == "tlsv1_1") + disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_1; + else if (token == "tlsv1_2") + disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_2; + } - if (config.has(root_ca_file_property)) - asio_opts.root_cert_file_ = config.getString(root_ca_file_property); + auto prefer_server_cypher = config.getBool(fmt::format("openSSL.{}.preferServerCiphers", key), false); + auto cache_sessions = config.getBool(fmt::format("openSSL.{}.cache_sessions", key), false); + return [params, disabled_protocols, prefer_server_cypher, cache_sessions, is_server = key == "server", certificate_data] + { + Poco::Net::Context context(is_server ? Poco::Net::Context::Usage::SERVER_USE : Poco::Net::Context::Usage::CLIENT_USE, params); + context.disableProtocols(disabled_protocols); + + if (prefer_server_cypher) + context.preferServerCiphers(); - if (config.getBool("openSSL.server.loadDefaultCAFile", false)) - asio_opts.load_default_ca_file_ = true; + if (cache_sessions) + context.enableSessionCache(); - if (config.getString("openSSL.server.verificationMode", "none") == "none") - asio_opts.skip_verification_ = true; + auto * ssl_ctx = context.sslContext(); + if (certificate_data) + { + if (auto err = SSL_CTX_clear_chain_certs(ssl_ctx); err != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Clear certificates {}", Poco::Net::Utility::getLastError()); + + if (auto err = SSL_CTX_use_certificate(ssl_ctx, const_cast(certificate_data->certs_chain[0].certificate())); err != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Use certificate {}", Poco::Net::Utility::getLastError()); + + for (auto cert = certificate_data->certs_chain.begin() + 1; cert != certificate_data->certs_chain.end(); cert++) + { + if (auto err = SSL_CTX_add1_chain_cert(ssl_ctx, const_cast(cert->certificate())); err != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Add certificate to chain {}", Poco::Net::Utility::getLastError()); + } + + if (auto err = SSL_CTX_use_PrivateKey(ssl_ctx, const_cast(static_cast(certificate_data->key))); err != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Use private key {}", Poco::Net::Utility::getLastError()); + + if (auto err = SSL_CTX_check_private_key(ssl_ctx); err != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Unusable key-pair {}", Poco::Net::Utility::getLastError()); + } + + + return context.takeSslContext(); + }; +} + +void setSSLParams(nuraft::asio_service::options & asio_opts) +{ + asio_opts.enable_ssl_ = true; + + const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); + asio_opts.ssl_context_provider_server_ = getSslContextProvider(config, "server"); + asio_opts.ssl_context_provider_client_ = getSslContextProvider(config, "client"); } #endif @@ -184,7 +265,7 @@ struct KeeperServer::KeeperRaftServer : public nuraft::raft_server } const size_t voting_members = get_num_voting_members(); - const auto not_responding_peers = get_not_responding_peers(); + const auto not_responding_peers = get_not_responding_peers_count(); const auto quorum_size = voting_members / 2 + 1; const auto max_not_responding_peers = voting_members - quorum_size; @@ -221,6 +302,11 @@ struct KeeperServer::KeeperRaftServer : public nuraft::raft_server return std::unique_lock(lock_); } + std::unique_lock lockCommit() + { + return std::unique_lock(commit_lock_); + } + bool isCommitInProgress() const { return sm_commit_exec_in_progress_; @@ -396,7 +482,6 @@ void KeeperServer::launchRaftServer(const Poco::Util::AbstractConfiguration & co throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSL support for NuRaft is disabled because ClickHouse was built without SSL support."); #endif } - if (is_recovering) enterRecoveryMode(params); @@ -1156,6 +1241,7 @@ Keeper4LWInfo KeeperServer::getPartiallyFilled4LWInfo() const uint64_t KeeperServer::createSnapshot() { + auto commit_lock = raft_instance->lockCommit(); uint64_t log_idx = raft_instance->create_snapshot(); if (log_idx != 0) LOG_INFO(log, "Snapshot creation scheduled with last committed log index {}.", log_idx); diff --git a/src/Core/ParallelReplicasMode.h b/src/Core/ParallelReplicasMode.h new file mode 100644 index 000000000000..dd0d9c05f0de --- /dev/null +++ b/src/Core/ParallelReplicasMode.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +namespace DB +{ + +/// Deprecated. Used only to define a type for the obsolete setting. +enum class ParallelReplicasCustomKeyFilterType : UInt8 +{ + DEFAULT, + RANGE, +}; + +enum class ParallelReplicasMode : UInt8 +{ + AUTO = 0, + READ_TASKS = 1, + CUSTOM_KEY_SAMPLING = 2, + CUSTOM_KEY_RANGE = 3, + SAMPLING_KEY = 4, +}; + + +} diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h index 02d54221ed3e..cd89ca013eea 100644 --- a/src/Core/ProtocolDefines.h +++ b/src/Core/ProtocolDefines.h @@ -33,7 +33,9 @@ static constexpr auto DBMS_MIN_REVISION_WITH_AGGREGATE_FUNCTIONS_VERSIONING = 54 static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION = 1; -static constexpr auto DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION = 3; +static constexpr auto DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION = 3; +static constexpr auto DBMS_PARALLEL_REPLICAS_MIN_VERSION_WITH_MARK_SEGMENT_SIZE_FIELD = 4; +static constexpr auto DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION = 4; static constexpr auto DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS = 54453; static constexpr auto DBMS_MERGE_TREE_PART_INFO_VERSION = 1; @@ -83,6 +85,14 @@ static constexpr auto DBMS_MIN_REVISION_WITH_SYSTEM_KEYWORDS_TABLE = 54468; static constexpr auto DBMS_MIN_REVISION_WITH_ROWS_BEFORE_AGGREGATION = 54469; +/// Packets size header +static constexpr auto DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS = 54470; + +static constexpr auto DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL = 54471; + +/// Push externally granted roles to other nodes +static constexpr auto DBMS_MIN_PROTOCOL_VERSION_WITH_INTERSERVER_EXTERNALLY_GRANTED_ROLES = 54472; + /// Version of ClickHouse TCP protocol. /// /// Should be incremented manually on protocol changes. @@ -90,6 +100,6 @@ static constexpr auto DBMS_MIN_REVISION_WITH_ROWS_BEFORE_AGGREGATION = 54469; /// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION, /// later is just a number for server version (one number instead of commit SHA) /// for simplicity (sometimes it may be more convenient in some use cases). -static constexpr auto DBMS_TCP_PROTOCOL_VERSION = 54469; +static constexpr auto DBMS_TCP_PROTOCOL_VERSION = 54472; } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 4f9abb306274..b82733f2a44c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -205,6 +205,7 @@ class IColumn; M(Bool, group_by_use_nulls, false, "Treat columns mentioned in ROLLUP, CUBE or GROUPING SETS as Nullable", 0) \ \ M(NonZeroUInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled. Should be always greater than 0", 0) \ + M(ParallelReplicasMode, parallel_replicas_mode, ParallelReplicasMode::READ_TASKS, "Type of filter to use with custom key for parallel replicas. default - use modulo operation on the custom key, range - use range filter on custom key using all possible values for the value type of custom key.", 0) \ \ M(Bool, skip_unavailable_shards, false, "If true, ClickHouse silently skips unavailable shards. Shard is marked as unavailable when: 1) The shard cannot be reached due to a connection failure. 2) Shard is unresolvable through DNS. 3) Table does not exist on the shard.", 0) \ \ @@ -898,6 +899,7 @@ class IColumn; M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \ M(Bool, restore_replace_external_engines_to_null, false, "Replace all the external table engines to Null on restore. Useful for testing purposes", 0) \ M(Bool, restore_replace_external_table_functions_to_null, false, "Replace all table functions to Null on restore. Useful for testing purposes", 0) \ + M(Bool, push_external_roles_in_interserver_queries, true, "Enable pushing user roles from originator to other nodes while performing a query.", 0) \ \ \ /* ###################################### */ \ @@ -930,18 +932,18 @@ class IColumn; M(UInt64, parallel_replicas_count, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the number of parallel replicas participating in query processing.", 0) \ M(UInt64, parallel_replica_offset, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the index of the replica participating in query processing among parallel replicas.", 0) \ M(String, parallel_replicas_custom_key, "", "Custom key assigning work to replicas when parallel replicas are used.", 0) \ - M(ParallelReplicasCustomKeyFilterType, parallel_replicas_custom_key_filter_type, ParallelReplicasCustomKeyFilterType::DEFAULT, "Type of filter to use with custom key for parallel replicas. default - use modulo operation on the custom key, range - use range filter on custom key using all possible values for the value type of custom key.", 0) \ M(UInt64, parallel_replicas_custom_key_range_lower, 0, "Lower bound for the universe that the parallel replicas custom range filter is calculated over", 0) \ M(UInt64, parallel_replicas_custom_key_range_upper, 0, "Upper bound for the universe that the parallel replicas custom range filter is calculated over. A value of 0 disables the upper bound, setting it to the max value of the custom key expression", 0) \ M(String, cluster_for_parallel_replicas, "", "Cluster for a shard in which current server is located", 0) \ - M(UInt64, allow_experimental_parallel_reading_from_replicas, 0, "Use all the replicas from a shard for SELECT query execution. Reading is parallelized and coordinated dynamically. 0 - disabled, 1 - enabled, silently disable them in case of failure, 2 - enabled, throw an exception in case of failure", 0) \ + M(UInt64, allow_experimental_parallel_reading_from_replicas, 0, "Use up to `max_parallel_replicas` the number of replicas from each shard for SELECT query execution. Reading is parallelized and coordinated dynamically. 0 - disabled, 1 - enabled, silently disable them in case of failure, 2 - enabled, throw an exception in case of failure", 0) ALIAS(enable_parallel_replicas)\ M(Bool, parallel_replicas_allow_in_with_subquery, true, "If true, subquery for IN will be executed on every follower replica.", 0) \ M(Float, parallel_replicas_single_task_marks_count_multiplier, 2, "A multiplier which will be added during calculation for minimal number of marks to retrieve from coordinator. This will be applied only for remote replicas.", 0) \ M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \ M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \ M(Bool, parallel_replicas_prefer_local_join, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN.", 0) \ - M(UInt64, parallel_replicas_mark_segment_size, 128, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing", 0) \ + M(UInt64, parallel_replicas_mark_segment_size, 0, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing. Value should be in range [128; 16384]", 0) \ M(Bool, allow_archive_path_syntax, true, "File/S3 engines/table function will parse paths with '::' as ' :: ' if archive has correct extension", 0) \ + M(Bool, parallel_replicas_local_plan, false, "Build local plan for local replica", 0) \ \ M(Bool, allow_experimental_inverted_index, false, "If it is set to true, allow to use experimental inverted index.", 0) \ M(Bool, allow_experimental_full_text_index, false, "If it is set to true, allow to use experimental full-text index.", 0) \ @@ -1035,6 +1037,7 @@ class IColumn; MAKE_OBSOLETE(M, UInt64, restore_threads, 16) \ MAKE_OBSOLETE(M, Bool, optimize_duplicate_order_by_and_distinct, false) \ MAKE_OBSOLETE(M, UInt64, parallel_replicas_min_number_of_granules_to_enable, 0) \ + MAKE_OBSOLETE(M, ParallelReplicasCustomKeyFilterType, parallel_replicas_custom_key_filter_type, ParallelReplicasCustomKeyFilterType::DEFAULT) \ MAKE_OBSOLETE(M, Bool, query_plan_optimize_projection, true) \ MAKE_OBSOLETE(M, Bool, query_cache_store_results_of_queries_with_nondeterministic_functions, false) \ MAKE_OBSOLETE(M, Bool, allow_experimental_annoy_index, false) \ @@ -1078,6 +1081,7 @@ class IColumn; M(Bool, input_format_parquet_preserve_order, false, "Avoid reordering rows when reading from Parquet files. Usually makes it much slower.", 0) \ M(Bool, input_format_parquet_filter_push_down, true, "When reading Parquet files, skip whole row groups based on the WHERE/PREWHERE expressions and min/max statistics in the Parquet metadata.", 0) \ M(Bool, input_format_parquet_use_native_reader, false, "When reading Parquet files, to use native reader instead of arrow reader.", 0) \ + M(Bool, input_format_parquet_bloom_filter_push_down, true, "When reading Parquet files, skip whole row groups based on the WHERE/PREWHERE expressions and bloom filter in the Parquet metadata.", 0) \ M(Bool, input_format_allow_seeks, true, "Allow seeks while reading in ORC/Parquet/Arrow input formats", 0) \ M(Bool, input_format_orc_allow_missing_columns, true, "Allow missing columns while reading ORC input formats", 0) \ M(Bool, input_format_orc_use_fast_decoder, true, "Use a faster ORC decoder implementation.", 0) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index b3e7a59c5e2f..4b3d5f60a8ef 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -63,6 +63,7 @@ static std::initializer_list #include #include +#include #include #include #include @@ -299,12 +300,6 @@ enum class Dialect : uint8_t DECLARE_SETTING_ENUM(Dialect) -enum class ParallelReplicasCustomKeyFilterType : uint8_t -{ - DEFAULT, - RANGE, -}; - DECLARE_SETTING_ENUM(ParallelReplicasCustomKeyFilterType) enum class LightweightMutationProjectionMode : uint8_t @@ -324,6 +319,8 @@ enum class DeduplicateMergeProjectionMode : uint8_t DECLARE_SETTING_ENUM(DeduplicateMergeProjectionMode) +DECLARE_SETTING_ENUM(ParallelReplicasMode) + DECLARE_SETTING_ENUM(LocalFSReadMethod) enum class ObjectStorageQueueMode : uint8_t diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp index 1a274c7f993a..62721ffa0f6c 100644 --- a/src/DataTypes/IDataType.cpp +++ b/src/DataTypes/IDataType.cpp @@ -322,6 +322,8 @@ bool isUInt8(TYPE data_type) { return WhichDataType(data_type).isUInt8(); } \ bool isUInt16(TYPE data_type) { return WhichDataType(data_type).isUInt16(); } \ bool isUInt32(TYPE data_type) { return WhichDataType(data_type).isUInt32(); } \ bool isUInt64(TYPE data_type) { return WhichDataType(data_type).isUInt64(); } \ +bool isUInt128(TYPE data_type) { return WhichDataType(data_type).isUInt128(); } \ +bool isUInt256(TYPE data_type) { return WhichDataType(data_type).isUInt256(); } \ bool isNativeUInt(TYPE data_type) { return WhichDataType(data_type).isNativeUInt(); } \ bool isUInt(TYPE data_type) { return WhichDataType(data_type).isUInt(); } \ \ @@ -329,6 +331,8 @@ bool isInt8(TYPE data_type) { return WhichDataType(data_type).isInt8(); } \ bool isInt16(TYPE data_type) { return WhichDataType(data_type).isInt16(); } \ bool isInt32(TYPE data_type) { return WhichDataType(data_type).isInt32(); } \ bool isInt64(TYPE data_type) { return WhichDataType(data_type).isInt64(); } \ +bool isInt128(TYPE data_type) { return WhichDataType(data_type).isInt128(); } \ +bool isInt256(TYPE data_type) { return WhichDataType(data_type).isInt256(); } \ bool isNativeInt(TYPE data_type) { return WhichDataType(data_type).isNativeInt(); } \ bool isInt(TYPE data_type) { return WhichDataType(data_type).isInt(); } \ \ diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index a7665e610ab4..2daaaf3caf98 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -459,7 +459,9 @@ struct WhichDataType bool isUInt8(TYPE data_type); \ bool isUInt16(TYPE data_type); \ bool isUInt32(TYPE data_type); \ -bool isUInt64(TYPE data_type); \ +bool isUInt64(TYPE data_type);\ +bool isUInt128(TYPE data_type);\ +bool isUInt256(TYPE data_type); \ bool isNativeUInt(TYPE data_type); \ bool isUInt(TYPE data_type); \ \ @@ -467,6 +469,8 @@ bool isInt8(TYPE data_type); \ bool isInt16(TYPE data_type); \ bool isInt32(TYPE data_type); \ bool isInt64(TYPE data_type); \ +bool isInt128(TYPE data_type); \ +bool isInt256(TYPE data_type); \ bool isNativeInt(TYPE data_type); \ bool isInt(TYPE data_type); \ \ diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index bf16f315ddf4..b36d53a61596 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -51,6 +51,8 @@ namespace configuration.db, configuration.user, configuration.password, + configuration.proto_send_chunked, + configuration.proto_recv_chunked, configuration.quota_key, "", /* cluster */ "", /* cluster_secret */ @@ -222,7 +224,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) { validateNamedCollection( *named_collection, {}, ValidateKeysMultiset{ - "secure", "host", "hostname", "port", "user", "username", "password", "quota_key", "name", + "secure", "host", "hostname", "port", "user", "username", "password", "proto_send_chunked", "proto_recv_chunked", "quota_key", "name", "db", "database", "table","query", "where", "invalidate_query", "update_field", "update_lag"}); const auto secure = named_collection->getOrDefault("secure", false); @@ -234,6 +236,8 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) .host = host, .user = named_collection->getAnyOrDefault({"user", "username"}, "default"), .password = named_collection->getOrDefault("password", ""), + .proto_send_chunked = named_collection->getOrDefault("proto_send_chunked", "notchunked"), + .proto_recv_chunked = named_collection->getOrDefault("proto_recv_chunked", "notchunked"), .quota_key = named_collection->getOrDefault("quota_key", ""), .db = named_collection->getAnyOrDefault({"db", "database"}, default_database), .table = named_collection->getOrDefault("table", ""), @@ -258,6 +262,8 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) .host = host, .user = config.getString(settings_config_prefix + ".user", "default"), .password = config.getString(settings_config_prefix + ".password", ""), + .proto_send_chunked = config.getString(settings_config_prefix + ".proto_caps.send", "notchunked"), + .proto_recv_chunked = config.getString(settings_config_prefix + ".proto_caps.recv", "notchunked"), .quota_key = config.getString(settings_config_prefix + ".quota_key", ""), .db = config.getString(settings_config_prefix + ".db", default_database), .table = config.getString(settings_config_prefix + ".table", ""), diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h index 3357514eab2b..faf9e5f80090 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.h +++ b/src/Dictionaries/ClickHouseDictionarySource.h @@ -23,6 +23,8 @@ class ClickHouseDictionarySource final : public IDictionarySource const std::string host; const std::string user; const std::string password; + const std::string proto_send_chunked; + const std::string proto_recv_chunked; const std::string quota_key; const std::string db; const std::string table; diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index b471f3fc58f9..6363c40a9ad2 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -28,6 +28,7 @@ extern const Event CachedReadBufferReadFromCacheMicroseconds; extern const Event CachedReadBufferCacheWriteMicroseconds; extern const Event CachedReadBufferReadFromSourceBytes; extern const Event CachedReadBufferReadFromCacheBytes; +extern const Event CachedReadBufferPredownloadedBytes; extern const Event CachedReadBufferCacheWriteBytes; extern const Event CachedReadBufferCreateBufferMicroseconds; @@ -644,6 +645,7 @@ void CachedOnDiskReadBufferFromFile::predownload(FileSegment & file_segment) size_t current_predownload_size = std::min(current_impl_buffer_size, bytes_to_predownload); ProfileEvents::increment(ProfileEvents::CachedReadBufferReadFromSourceBytes, current_impl_buffer_size); + ProfileEvents::increment(ProfileEvents::CachedReadBufferPredownloadedBytes, current_impl_buffer_size); bool continue_predownload = file_segment.reserve( current_predownload_size, settings.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds); diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index de78ce4594d5..4343d063e58e 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -118,6 +118,7 @@ std::unique_ptr getClient( .use_virtual_addressing = url.is_virtual_hosted_style, .disable_checksum = auth_settings.disable_checksum, .gcs_issue_compose_request = auth_settings.gcs_issue_compose_request, + .is_s3express_bucket = is_s3_express_bucket }; auto credentials_configuration = S3::CredentialsConfiguration diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index e47e2230e679..1d101d9b97ca 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -162,6 +162,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.parquet.case_insensitive_column_matching = settings.input_format_parquet_case_insensitive_column_matching; format_settings.parquet.preserve_order = settings.input_format_parquet_preserve_order; format_settings.parquet.filter_push_down = settings.input_format_parquet_filter_push_down; + format_settings.parquet.bloom_filter_push_down = settings.input_format_parquet_bloom_filter_push_down; format_settings.parquet.use_native_reader = settings.input_format_parquet_use_native_reader; format_settings.parquet.allow_missing_columns = settings.input_format_parquet_allow_missing_columns; format_settings.parquet.skip_columns_with_unsupported_types_in_schema_inference = settings.input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference; @@ -170,6 +171,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.parquet.max_block_size = settings.input_format_parquet_max_block_size; format_settings.parquet.prefer_block_bytes = settings.input_format_parquet_prefer_block_bytes; format_settings.parquet.output_compression_method = settings.output_format_parquet_compression_method; + format_settings.parquet.output_compression_level = context->getSettingsRef().output_format_compression_level; format_settings.parquet.output_compliant_nested_types = settings.output_format_parquet_compliant_nested_types; format_settings.parquet.use_custom_encoder = settings.output_format_parquet_use_custom_encoder; format_settings.parquet.parallel_encoding = settings.output_format_parquet_parallel_encoding; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 79bd7a60e405..ed68e36e41bd 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -271,6 +271,7 @@ struct FormatSettings bool case_insensitive_column_matching = false; bool filter_push_down = true; bool use_native_reader = false; + bool bloom_filter_push_down = true; std::unordered_set skip_row_groups = {}; bool output_string_as_string = false; bool output_fixed_string_as_fixed_byte_array = true; @@ -281,6 +282,7 @@ struct FormatSettings size_t prefer_block_bytes = DEFAULT_BLOCK_SIZE * 256; ParquetVersion output_version; ParquetCompression output_compression_method = ParquetCompression::SNAPPY; + uint64_t output_compression_level; bool output_compliant_nested_types = true; size_t data_page_size = 1024 * 1024; size_t write_batch_size = 1024; diff --git a/src/IO/NetUtils.h b/src/IO/NetUtils.h new file mode 100644 index 000000000000..12f09524ae7a --- /dev/null +++ b/src/IO/NetUtils.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +template +constexpr T netToHost(T value) noexcept +{ + if constexpr (std::endian::native != std::endian::big) + return std::byteswap(value); + return value; +} + +template +constexpr T hostToNet(T value) noexcept +{ + if constexpr (std::endian::native != std::endian::big) + return std::byteswap(value); + return value; +} + +template +constexpr T toLittleEndian(T value) noexcept +{ + if constexpr (std::endian::native == std::endian::big) + return std::byteswap(value); + return value; +} + +template +constexpr T toBigEndian(T value) noexcept +{ + if constexpr (std::endian::native != std::endian::big) + return std::byteswap(value); + return value; +} + +template +constexpr T fromLittleEndian(T value) noexcept +{ + if constexpr (std::endian::native == std::endian::big) + return std::byteswap(value); + return value; +} + +template +constexpr T fromBigEndian(T value) noexcept +{ + if constexpr (std::endian::native != std::endian::big) + return std::byteswap(value); + return value; +} + +} diff --git a/src/IO/ReadBufferFromPocoSocket.cpp b/src/IO/ReadBufferFromPocoSocket.cpp index 26cdee4140c8..bbf9f96404fc 100644 --- a/src/IO/ReadBufferFromPocoSocket.cpp +++ b/src/IO/ReadBufferFromPocoSocket.cpp @@ -32,7 +32,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -bool ReadBufferFromPocoSocket::nextImpl() +ssize_t ReadBufferFromPocoSocketBase::socketReceiveBytesImpl(char * ptr, size_t size) { ssize_t bytes_read = 0; Stopwatch watch; @@ -43,14 +43,11 @@ bool ReadBufferFromPocoSocket::nextImpl() ProfileEvents::increment(ProfileEvents::NetworkReceiveBytes, bytes_read); }); + CurrentMetrics::Increment metric_increment(CurrentMetrics::NetworkReceive); + /// Add more details to exceptions. try { - CurrentMetrics::Increment metric_increment(CurrentMetrics::NetworkReceive); - - if (internal_buffer.size() > INT_MAX) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Buffer overflow"); - /// If async_callback is specified, set socket to non-blocking mode /// and try to read data from it, if socket is not ready for reading, /// run async_callback and try again later. @@ -61,7 +58,7 @@ bool ReadBufferFromPocoSocket::nextImpl() socket.setBlocking(false); SCOPE_EXIT(socket.setBlocking(true)); bool secure = socket.secure(); - bytes_read = socket.impl()->receiveBytes(internal_buffer.begin(), static_cast(internal_buffer.size())); + bytes_read = socket.impl()->receiveBytes(ptr, static_cast(size)); /// Check EAGAIN and ERR_SSL_WANT_READ/ERR_SSL_WANT_WRITE for secure socket (reading from secure socket can write too). while (bytes_read < 0 && (errno == EAGAIN || (secure && (checkSSLWantRead(bytes_read) || checkSSLWantWrite(bytes_read))))) @@ -73,12 +70,12 @@ bool ReadBufferFromPocoSocket::nextImpl() async_callback(socket.impl()->sockfd(), socket.getReceiveTimeout(), AsyncEventTimeoutType::RECEIVE, socket_description, AsyncTaskExecutor::Event::READ | AsyncTaskExecutor::Event::ERROR); /// Try to read again. - bytes_read = socket.impl()->receiveBytes(internal_buffer.begin(), static_cast(internal_buffer.size())); + bytes_read = socket.impl()->receiveBytes(ptr, static_cast(size)); } } else { - bytes_read = socket.impl()->receiveBytes(internal_buffer.begin(), static_cast(internal_buffer.size())); + bytes_read = socket.impl()->receiveBytes(ptr, static_cast(size)); } } catch (const Poco::Net::NetException & e) @@ -99,6 +96,16 @@ bool ReadBufferFromPocoSocket::nextImpl() if (bytes_read < 0) throw NetException(ErrorCodes::CANNOT_READ_FROM_SOCKET, "Cannot read from socket (peer: {}, local: {})", peer_address.toString(), socket.address().toString()); + return bytes_read; +} + +bool ReadBufferFromPocoSocketBase::nextImpl() +{ + if (internal_buffer.size() > INT_MAX) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Buffer overflow"); + + ssize_t bytes_read = socketReceiveBytesImpl(internal_buffer.begin(), internal_buffer.size()); + if (read_event != ProfileEvents::end()) ProfileEvents::increment(read_event, bytes_read); @@ -110,7 +117,7 @@ bool ReadBufferFromPocoSocket::nextImpl() return true; } -ReadBufferFromPocoSocket::ReadBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size) +ReadBufferFromPocoSocketBase::ReadBufferFromPocoSocketBase(Poco::Net::Socket & socket_, size_t buf_size) : BufferWithOwnMemory(buf_size) , socket(socket_) , peer_address(socket.peerAddress()) @@ -119,19 +126,22 @@ ReadBufferFromPocoSocket::ReadBufferFromPocoSocket(Poco::Net::Socket & socket_, { } -ReadBufferFromPocoSocket::ReadBufferFromPocoSocket(Poco::Net::Socket & socket_, const ProfileEvents::Event & read_event_, size_t buf_size) - : ReadBufferFromPocoSocket(socket_, buf_size) +ReadBufferFromPocoSocketBase::ReadBufferFromPocoSocketBase(Poco::Net::Socket & socket_, const ProfileEvents::Event & read_event_, size_t buf_size) + : ReadBufferFromPocoSocketBase(socket_, buf_size) { read_event = read_event_; } -bool ReadBufferFromPocoSocket::poll(size_t timeout_microseconds) const +bool ReadBufferFromPocoSocketBase::poll(size_t timeout_microseconds) const { - if (available()) + /// For secure socket it is important to check if any remaining data available in underlying decryption buffer - + /// read always retrieves the whole encrypted frame from the wire and puts it into underlying buffer while returning only requested size - + /// further poll() can block though there is still data to read in the underlying decryption buffer. + if (available() || socket.impl()->available()) return true; Stopwatch watch; - bool res = socket.poll(timeout_microseconds, Poco::Net::Socket::SELECT_READ | Poco::Net::Socket::SELECT_ERROR); + bool res = socket.impl()->poll(timeout_microseconds, Poco::Net::Socket::SELECT_READ | Poco::Net::Socket::SELECT_ERROR); ProfileEvents::increment(ProfileEvents::NetworkReceiveElapsedMicroseconds, watch.elapsedMicroseconds()); return res; } diff --git a/src/IO/ReadBufferFromPocoSocket.h b/src/IO/ReadBufferFromPocoSocket.h index 76156612764b..912388adaac6 100644 --- a/src/IO/ReadBufferFromPocoSocket.h +++ b/src/IO/ReadBufferFromPocoSocket.h @@ -9,7 +9,7 @@ namespace DB { /// Works with the ready Poco::Net::Socket. Blocking operations. -class ReadBufferFromPocoSocket : public BufferWithOwnMemory +class ReadBufferFromPocoSocketBase : public BufferWithOwnMemory { protected: Poco::Net::Socket & socket; @@ -25,16 +25,29 @@ class ReadBufferFromPocoSocket : public BufferWithOwnMemory bool nextImpl() override; public: - explicit ReadBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); - explicit ReadBufferFromPocoSocket(Poco::Net::Socket & socket_, const ProfileEvents::Event & read_event_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); + explicit ReadBufferFromPocoSocketBase(Poco::Net::Socket & socket_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); + explicit ReadBufferFromPocoSocketBase(Poco::Net::Socket & socket_, const ProfileEvents::Event & read_event_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); bool poll(size_t timeout_microseconds) const; void setAsyncCallback(AsyncCallback async_callback_) { async_callback = std::move(async_callback_); } + ssize_t socketReceiveBytesImpl(char * ptr, size_t size); + private: AsyncCallback async_callback; std::string socket_description; }; +class ReadBufferFromPocoSocket : public ReadBufferFromPocoSocketBase +{ +public: + explicit ReadBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE) + : ReadBufferFromPocoSocketBase(socket_, buf_size) + {} + explicit ReadBufferFromPocoSocket(Poco::Net::Socket & socket_, const ProfileEvents::Event & read_event_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE) + : ReadBufferFromPocoSocketBase(socket_, read_event_, buf_size) + {} +}; + } diff --git a/src/IO/ReadBufferFromPocoSocketChunked.cpp b/src/IO/ReadBufferFromPocoSocketChunked.cpp new file mode 100644 index 000000000000..4a1e3732a557 --- /dev/null +++ b/src/IO/ReadBufferFromPocoSocketChunked.cpp @@ -0,0 +1,166 @@ +#include +#include +#include + + +namespace DB::ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace DB +{ + +ReadBufferFromPocoSocketChunked::ReadBufferFromPocoSocketChunked(Poco::Net::Socket & socket_, size_t buf_size) + : ReadBufferFromPocoSocketChunked(socket_, ProfileEvents::end(), buf_size) +{} + +ReadBufferFromPocoSocketChunked::ReadBufferFromPocoSocketChunked(Poco::Net::Socket & socket_, const ProfileEvents::Event & read_event_, size_t buf_size) + : ReadBufferFromPocoSocketBase( + socket_, read_event_, + std::min(buf_size, static_cast(std::numeric_limits::max()))), + our_address(socket_.address()), log(getLogger("Protocol")) +{} + +void ReadBufferFromPocoSocketChunked::enableChunked() +{ + if (chunked) + return; + chunked = 1; + data_end = buffer().end(); + /// Resize working buffer so any next read will call nextImpl + working_buffer.resize(offset()); + chunk_left = 0; + next_chunk = 0; +} + +bool ReadBufferFromPocoSocketChunked::hasBufferedData() const +{ + if (available()) + return true; + + return chunked && (static_cast(data_end - working_buffer.end()) > sizeof(next_chunk)); +} + +bool ReadBufferFromPocoSocketChunked::poll(size_t timeout_microseconds) const +{ + if (chunked) + if (available() || static_cast(data_end - working_buffer.end()) > sizeof(next_chunk)) + return true; + + return ReadBufferFromPocoSocketBase::poll(timeout_microseconds); +} + + +bool ReadBufferFromPocoSocketChunked::loadNextChunk(Position c_pos, bool cont) +{ + auto buffered = std::min(static_cast(data_end - c_pos), sizeof(next_chunk)); + + if (buffered) + std::memcpy(&next_chunk, c_pos, buffered); + if (buffered < sizeof(next_chunk)) + if (socketReceiveBytesImpl(reinterpret_cast(&next_chunk) + buffered, sizeof(next_chunk) - buffered) < static_cast(sizeof(next_chunk) - buffered)) + return false; + next_chunk = fromLittleEndian(next_chunk); + + if (next_chunk) + { + if (cont) + LOG_TEST(log, "{} <- {} Chunk receive continued. Size {}", ourAddress().toString(), peerAddress().toString(), next_chunk); + } + else + LOG_TEST(log, "{} <- {} Chunk receive ended.", ourAddress().toString(), peerAddress().toString()); + + return true; +} + +bool ReadBufferFromPocoSocketChunked::processChunkLeft(Position c_pos) +{ + if (data_end - c_pos < chunk_left) + { + working_buffer.resize(data_end - buffer().begin()); + nextimpl_working_buffer_offset = c_pos - buffer().begin(); + chunk_left -= (data_end - c_pos); + return true; + } + + nextimpl_working_buffer_offset = c_pos - buffer().begin(); + working_buffer.resize(nextimpl_working_buffer_offset + chunk_left); + + c_pos += chunk_left; + + if (!loadNextChunk(c_pos, true)) + return false; + + chunk_left = 0; + return true; +} + + +bool ReadBufferFromPocoSocketChunked::nextImpl() +{ + if (!chunked) + return ReadBufferFromPocoSocketBase::nextImpl(); + + auto * c_pos = pos; + + if (chunk_left == 0) + { + if (next_chunk == 0) + { + if (chunked == 1) + chunked = 2; // first chunked block - no end marker + else + c_pos = pos + sizeof(next_chunk); // bypass chunk end marker + + if (c_pos > data_end) + c_pos = data_end; + + if (!loadNextChunk(c_pos)) + return false; + + chunk_left = next_chunk; + next_chunk = 0; + + if (chunk_left == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Native protocol: empty chunk received"); + + c_pos += sizeof(next_chunk); + + if (c_pos >= data_end) + { + if (!ReadBufferFromPocoSocketBase::nextImpl()) + return false; + data_end = buffer().end(); + c_pos = buffer().begin(); + } + + LOG_TEST(log, "{} <- {} Chunk receive started. Message {}, size {}", ourAddress().toString(), peerAddress().toString(), static_cast(*c_pos), chunk_left); + } + else + { + c_pos += sizeof(next_chunk); + if (c_pos >= data_end) + { + if (!ReadBufferFromPocoSocketBase::nextImpl()) + return false; + data_end = buffer().end(); + c_pos = buffer().begin(); + } + + chunk_left = next_chunk; + next_chunk = 0; + } + } + else + { + if (!ReadBufferFromPocoSocketBase::nextImpl()) + return false; + data_end = buffer().end(); + c_pos = buffer().begin(); + } + + return processChunkLeft(c_pos); +} + +} diff --git a/src/IO/ReadBufferFromPocoSocketChunked.h b/src/IO/ReadBufferFromPocoSocketChunked.h new file mode 100644 index 000000000000..8bc4024b978a --- /dev/null +++ b/src/IO/ReadBufferFromPocoSocketChunked.h @@ -0,0 +1,109 @@ +#pragma once + +#include +#include + +/* + +Handshake +============= + | 'Hello' type + | handshake exchange + | chunked protocol negotiation + +============= + + +Basic chunk: + +============= +Chunk begins | 0x12345678 chunk size, 4 bytes little endian + +------------- + | Packet type always follows beginning of the chunk + | packet data + +------------- +Chunk ends | 0x00000000 4 zero bytes + +============= + + +Datastream chunk: + +============= +Chunk begins | 0x12345678 + +------------- + | Packet type + | packet data + +------------- + | Packet type + | packet data + +------------- +...arbitrary number ..... +of packets... ..... + +------------- + | Packet type + | packet data + +------------- +Chunk ends | 0x00000000 + +============= + + +Multipart chunk: + +============= +Chunk begins | 0x12345678 chunk part size, 4 bytes little endian + +------------- + | Packet type + | packet data + +------------- + | Packet type + | (partial) packet data + +============= +Chunk continues | 0x12345678 chunk next part size, 4 bytes little endian + +============= + | possibly previous packet's data + +------------- + | Packet type + | packet data + +------------- +...arbitrary number ..... +of chunk parts... ..... + +------------- + | Packet type + | packet data + +------------- +Chunk ends | 0x00000000 + +============= + +*/ + +namespace DB +{ + +class ReadBufferFromPocoSocketChunked: public ReadBufferFromPocoSocketBase +{ +public: + using ReadBufferFromPocoSocketBase::setAsyncCallback; + + explicit ReadBufferFromPocoSocketChunked(Poco::Net::Socket & socket_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); + explicit ReadBufferFromPocoSocketChunked(Poco::Net::Socket & socket_, const ProfileEvents::Event & read_event_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); + + void enableChunked(); + + bool hasBufferedData() const; + + bool poll(size_t timeout_microseconds) const; + + Poco::Net::SocketAddress peerAddress() { return peer_address; } + Poco::Net::SocketAddress ourAddress() { return our_address; } + +protected: + bool loadNextChunk(Position c_pos, bool cont = false); + bool processChunkLeft(Position c_pos); + bool nextImpl() override; + + Poco::Net::SocketAddress our_address; + +private: + LoggerPtr log; + Position data_end = nullptr; // end position of data in the internal_buffer + UInt32 chunk_left = 0; // chunk left to read from socket + UInt32 next_chunk = 0; // size of the next cnunk + UInt8 chunked = 0; // 0 - disabled; 1 - started; 2 - enabled; +}; + +} diff --git a/src/IO/WriteBuffer.h b/src/IO/WriteBuffer.h index 4759f96a2352..84b1079b8247 100644 --- a/src/IO/WriteBuffer.h +++ b/src/IO/WriteBuffer.h @@ -64,7 +64,8 @@ class WriteBuffer : public BufferBase } bytes += bytes_in_buffer; - pos = working_buffer.begin(); + pos = working_buffer.begin() + nextimpl_working_buffer_offset; + nextimpl_working_buffer_offset = 0; } /// Calling finalize() in the destructor of derived classes is a bad practice. @@ -164,6 +165,11 @@ class WriteBuffer : public BufferBase bool finalized = false; bool canceled = false; + /// The number of bytes to preserve from the initial position of `working_buffer` + /// buffer. Apparently this is an additional out-parameter for nextImpl(), + /// not a real field. + size_t nextimpl_working_buffer_offset = 0; + private: /** Write the data in the buffer (from the beginning of the buffer to the current position). * Throw an exception if something is wrong. diff --git a/src/IO/WriteBufferFromPocoSocket.cpp b/src/IO/WriteBufferFromPocoSocket.cpp index 5ed4dbdc7877..ffb38a384a06 100644 --- a/src/IO/WriteBufferFromPocoSocket.cpp +++ b/src/IO/WriteBufferFromPocoSocket.cpp @@ -183,6 +183,7 @@ WriteBufferFromPocoSocket::WriteBufferFromPocoSocket(Poco::Net::Socket & socket_ , socket(socket_) , peer_address(socket.peerAddress()) , our_address(socket.address()) + , write_event(ProfileEvents::end()) , socket_description("socket (" + peer_address.toString() + ")") { } diff --git a/src/IO/WriteBufferFromPocoSocketChunked.cpp b/src/IO/WriteBufferFromPocoSocketChunked.cpp new file mode 100644 index 000000000000..9da46ee2d10f --- /dev/null +++ b/src/IO/WriteBufferFromPocoSocketChunked.cpp @@ -0,0 +1,210 @@ +#include +#include +#include + + +namespace +{ + +template +void setValue(T * typed_ptr, std::type_identity_t val) +{ + memcpy(static_cast(typed_ptr), &val, sizeof(T)); +} + +} + +namespace DB +{ + +WriteBufferFromPocoSocketChunked::WriteBufferFromPocoSocketChunked(Poco::Net::Socket & socket_, size_t buf_size) + : WriteBufferFromPocoSocketChunked(socket_, ProfileEvents::end(), buf_size) +{} + +WriteBufferFromPocoSocketChunked::WriteBufferFromPocoSocketChunked(Poco::Net::Socket & socket_, const ProfileEvents::Event & write_event_, size_t buf_size) + : WriteBufferFromPocoSocket( + socket_, write_event_, + std::clamp(buf_size, sizeof(*chunk_size_ptr) + 1, static_cast(std::numeric_limits>::max()))), + log(getLogger("Protocol")) +{} + +void WriteBufferFromPocoSocketChunked::enableChunked() +{ + chunked = true; + /// Initialize next chunk + chunk_size_ptr = reinterpret_cast(pos); + pos += std::min(available(), sizeof(*chunk_size_ptr)); + /// Pretend finishChunk() was just called to prevent sending empty chunk if finishChunk() called immediately + last_finish_chunk = chunk_size_ptr; +} + +void WriteBufferFromPocoSocketChunked::finishChunk() +{ + if (!chunked) + return; + + if (pos <= reinterpret_cast(chunk_size_ptr) + sizeof(*chunk_size_ptr)) + { + /// Prevent duplicate finish chunk (and finish chunk right after enableChunked()) + if (chunk_size_ptr == last_finish_chunk) + return; + + /// If current chunk is empty it means we are finishing a chunk previously sent by next(), + /// we want to convert current chunk header into end-of-chunk marker and initialize next chunk. + /// We don't need to worry about if it's the end of the buffer because next() always sends the whole buffer + /// so it should be a beginning of the buffer. + + chassert(reinterpret_cast(chunk_size_ptr) == working_buffer.begin()); + + setValue(chunk_size_ptr, 0); + /// Initialize next chunk + chunk_size_ptr = reinterpret_cast(pos); + pos += std::min(available(), sizeof(*chunk_size_ptr)); + + last_finish_chunk = chunk_size_ptr; + + return; + } + + /// Previously finished chunk wasn't sent yet + if (last_finish_chunk == chunk_size_ptr) + { + chunk_started = false; + LOG_TEST(log, "{} -> {} Chunk send ended.", ourAddress().toString(), peerAddress().toString()); + } + + /// Fill up current chunk size + setValue(chunk_size_ptr, toLittleEndian(static_cast(pos - reinterpret_cast(chunk_size_ptr) - sizeof(*chunk_size_ptr)))); + + if (!chunk_started) + LOG_TEST(log, "{} -> {} Chunk send started. Message {}, size {}", + ourAddress().toString(), peerAddress().toString(), + static_cast(*(reinterpret_cast(chunk_size_ptr) + sizeof(*chunk_size_ptr))), + *chunk_size_ptr); + else + { + chunk_started = false; + LOG_TEST(log, "{} -> {} Chunk send continued. Size {}", ourAddress().toString(), peerAddress().toString(), *chunk_size_ptr); + } + + LOG_TEST(log, "{} -> {} Chunk send ended.", ourAddress().toString(), peerAddress().toString()); + + if (available() < sizeof(*chunk_size_ptr)) + { + finishing = available(); + pos += available(); + chunk_size_ptr = reinterpret_cast(pos); + last_finish_chunk = chunk_size_ptr; + return; + } + + /// Buffer end-of-chunk + setValue(reinterpret_cast(pos), 0); + pos += sizeof(*chunk_size_ptr); + /// Initialize next chunk + chunk_size_ptr = reinterpret_cast(pos); + pos += std::min(available(), sizeof(*chunk_size_ptr)); + + last_finish_chunk = chunk_size_ptr; +} + +WriteBufferFromPocoSocketChunked::~WriteBufferFromPocoSocketChunked() +{ + try + { + finalize(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } +} + +void WriteBufferFromPocoSocketChunked::nextImpl() +{ + if (!chunked) + { + WriteBufferFromPocoSocket::nextImpl(); + return; + } + + /// next() after finishChunk at the end of the buffer + if (finishing < sizeof(*chunk_size_ptr)) + { + pos -= finishing; + /// Send current chunk + WriteBufferFromPocoSocket::nextImpl(); + /// Send end-of-chunk directly + UInt32 s = 0; + socketSendBytes(reinterpret_cast(&s), sizeof(s)); + + finishing = sizeof(*chunk_size_ptr); + + /// Initialize next chunk + chunk_size_ptr = reinterpret_cast(working_buffer.begin()); + nextimpl_working_buffer_offset = sizeof(*chunk_size_ptr); + + last_finish_chunk = chunk_size_ptr; + + return; + } + + /// Prevent sending empty chunk + if (offset() == sizeof(*chunk_size_ptr)) + { + nextimpl_working_buffer_offset = sizeof(*chunk_size_ptr); + return; + } + + /// Finish chunk at the end of the buffer + if (working_buffer.end() - reinterpret_cast(chunk_size_ptr) <= static_cast(sizeof(*chunk_size_ptr))) + { + pos = reinterpret_cast(chunk_size_ptr); + /// Send current chunk + WriteBufferFromPocoSocket::nextImpl(); + /// Initialize next chunk + chunk_size_ptr = reinterpret_cast(working_buffer.begin()); + nextimpl_working_buffer_offset = sizeof(*chunk_size_ptr); + + last_finish_chunk = nullptr; + + return; + } + + bool initialize_last_finish_chunk = false; + if (pos - reinterpret_cast(chunk_size_ptr) == sizeof(*chunk_size_ptr)) // next() after finishChunk + { + pos -= sizeof(*chunk_size_ptr); + initialize_last_finish_chunk = true; + } + else // fill up current chunk size + { + setValue(chunk_size_ptr, toLittleEndian(static_cast(pos - reinterpret_cast(chunk_size_ptr) - sizeof(*chunk_size_ptr)))); + if (!chunk_started) + { + chunk_started = true; + LOG_TEST(log, "{} -> {} Chunk send started. Message {}, size {}", + ourAddress().toString(), peerAddress().toString(), + static_cast(*(reinterpret_cast(chunk_size_ptr) + sizeof(*chunk_size_ptr))), + *chunk_size_ptr); + } + else + LOG_TEST(log, "{} -> {} Chunk send continued. Size {}", ourAddress().toString(), peerAddress().toString(), *chunk_size_ptr); + } + /// Send current chunk + WriteBufferFromPocoSocket::nextImpl(); + /// Initialize next chunk + chunk_size_ptr = reinterpret_cast(working_buffer.begin()); + nextimpl_working_buffer_offset = sizeof(*chunk_size_ptr); + + last_finish_chunk = initialize_last_finish_chunk ? chunk_size_ptr : nullptr; +} + +void WriteBufferFromPocoSocketChunked::finalizeImpl() +{ + if (chunked && offset() == sizeof(*chunk_size_ptr)) + pos -= sizeof(*chunk_size_ptr); + WriteBufferFromPocoSocket::finalizeImpl(); +} + +} diff --git a/src/IO/WriteBufferFromPocoSocketChunked.h b/src/IO/WriteBufferFromPocoSocketChunked.h new file mode 100644 index 000000000000..13a277e3bfbd --- /dev/null +++ b/src/IO/WriteBufferFromPocoSocketChunked.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class WriteBufferFromPocoSocketChunked: public WriteBufferFromPocoSocket +{ +public: + explicit WriteBufferFromPocoSocketChunked(Poco::Net::Socket & socket_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); + explicit WriteBufferFromPocoSocketChunked(Poco::Net::Socket & socket_, const ProfileEvents::Event & write_event_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE); + + void enableChunked(); + void finishChunk(); + ~WriteBufferFromPocoSocketChunked() override; + +protected: + void nextImpl() override; + void finalizeImpl() override; + Poco::Net::SocketAddress peerAddress() const { return peer_address; } + Poco::Net::SocketAddress ourAddress() const { return our_address; } + +private: + LoggerPtr log; + bool chunked = false; + UInt32 * last_finish_chunk = nullptr; // pointer to the last chunk header created by finishChunk + bool chunk_started = false; // chunk started flag + UInt32 * chunk_size_ptr = nullptr; // pointer to the chunk size holder in the buffer + size_t finishing = sizeof(*chunk_size_ptr); // indicates not enough buffer for end-of-chunk marker +}; + +} diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index 59c98491c148..dd9e35834ebb 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -113,6 +113,9 @@ Cluster::Address::Address( secure = ConfigHelper::getBool(config, config_prefix + ".secure", false, /* empty_as */true) ? Protocol::Secure::Enable : Protocol::Secure::Disable; priority = Priority{config.getInt(config_prefix + ".priority", 1)}; + proto_send_chunked = config.getString(config_prefix + ".proto_caps.send", "notchunked"); + proto_recv_chunked = config.getString(config_prefix + ".proto_caps.recv", "notchunked"); + const char * port_type = secure == Protocol::Secure::Enable ? "tcp_port_secure" : "tcp_port"; auto default_port = config.getInt(port_type, 0); @@ -425,7 +428,9 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, auto pool = ConnectionPoolFactory::instance().get( static_cast(settings.distributed_connections_pool_size), address.host_name, address.port, - address.default_database, address.user, address.password, address.quota_key, + address.default_database, address.user, address.password, + address.proto_send_chunked, address.proto_recv_chunked, + address.quota_key, address.cluster, address.cluster_secret, "server", address.compression, address.secure, address.priority); @@ -589,6 +594,8 @@ void Cluster::addShard( replica.default_database, replica.user, replica.password, + replica.proto_send_chunked, + replica.proto_recv_chunked, replica.quota_key, replica.cluster, replica.cluster_secret, @@ -744,6 +751,8 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti address.default_database, address.user, address.password, + address.proto_send_chunked, + address.proto_recv_chunked, address.quota_key, address.cluster, address.cluster_secret, diff --git a/src/Interpreters/Cluster.h b/src/Interpreters/Cluster.h index dc5790ac339b..c69d77668abf 100644 --- a/src/Interpreters/Cluster.h +++ b/src/Interpreters/Cluster.h @@ -114,6 +114,8 @@ class Cluster UInt16 port{0}; String user; String password; + String proto_send_chunked = "notchunked"; + String proto_recv_chunked = "notchunked"; String quota_key; /// For inter-server authorization diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index d04a73e384e2..3d1181fe9da4 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include namespace DB @@ -38,6 +39,7 @@ namespace ErrorCodes extern const int TOO_LARGE_DISTRIBUTED_DEPTH; extern const int LOGICAL_ERROR; extern const int UNEXPECTED_CLUSTER; + extern const int INCONSISTENT_CLUSTER_DEFINITION; } namespace ClusterProxy @@ -240,7 +242,7 @@ getShardFilterGeneratorForCustomKey(const Cluster & cluster, ContextPtr context, return [my_custom_key_ast = std::move(custom_key_ast), column_description = columns, - custom_key_type = settings.parallel_replicas_custom_key_filter_type.value, + custom_key_type = settings.parallel_replicas_mode.value, custom_key_range_lower = settings.parallel_replicas_custom_key_range_lower.value, custom_key_range_upper = settings.parallel_replicas_custom_key_range_upper.value, query_context = context, @@ -286,7 +288,7 @@ void executeQuery( auto cluster = query_info.getCluster(); auto new_context = updateSettingsAndClientInfoForCluster(*cluster, is_remote_function, context, settings, main_table, query_info.additional_filter_ast, log, &distributed_settings); - if (context->getSettingsRef().allow_experimental_parallel_reading_from_replicas + if (context->getSettingsRef().allow_experimental_parallel_reading_from_replicas.value && context->getSettingsRef().allow_experimental_parallel_reading_from_replicas.value != new_context->getSettingsRef().allow_experimental_parallel_reading_from_replicas.value) { @@ -439,7 +441,8 @@ void executeQueryWithParallelReplicas( QueryProcessingStage::Enum processed_stage, const ASTPtr & query_ast, ContextPtr context, - std::shared_ptr storage_limits) + std::shared_ptr storage_limits, + QueryPlanStepPtr analyzed_read_from_merge_tree) { auto logger = getLogger("executeQueryWithParallelReplicas"); LOG_DEBUG(logger, "Executing read from {}, header {}, query ({}), stage {} with parallel replicas", @@ -459,14 +462,14 @@ void executeQueryWithParallelReplicas( { LOG_WARNING( getLogger("executeQueryWithParallelReplicas"), - "Setting 'use_hedged_requests' explicitly with enabled 'allow_experimental_parallel_reading_from_replicas' has no effect. " + "Setting 'use_hedged_requests' explicitly with enabled 'enable_parallel_replicas' has no effect. " "Hedged connections are not used for parallel reading from replicas"); } else { LOG_INFO( getLogger("executeQueryWithParallelReplicas"), - "Disabling 'use_hedged_requests' in favor of 'allow_experimental_parallel_reading_from_replicas'. Hedged connections are " + "Disabling 'use_hedged_requests' in favor of 'enable_parallel_replicas'. Hedged connections are " "not used for parallel reading from replicas"); } @@ -485,12 +488,12 @@ void executeQueryWithParallelReplicas( shard_num = column->getUInt(0); } - const auto shard_count = not_optimized_cluster->getShardCount(); ClusterPtr new_cluster = not_optimized_cluster; /// if got valid shard_num from query initiator, then parallel replicas scope is the specified shard /// shards are numbered in order of appearance in the cluster config if (shard_num > 0) { + const auto shard_count = not_optimized_cluster->getShardCount(); if (shard_num > shard_count) throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -516,21 +519,147 @@ void executeQueryWithParallelReplicas( "`cluster_for_parallel_replicas` setting refers to cluster with several shards. Expected a cluster with one shard"); } + const auto & shard = new_cluster->getShardsInfo().at(0); + size_t max_replicas_to_use = settings.max_parallel_replicas; + if (max_replicas_to_use > shard.getAllNodeCount()) + { + LOG_INFO( + getLogger("ReadFromParallelRemoteReplicasStep"), + "The number of replicas requested ({}) is bigger than the real number available in the cluster ({}). " + "Will use the latter number to execute the query.", + settings.max_parallel_replicas, + shard.getAllNodeCount()); + max_replicas_to_use = shard.getAllNodeCount(); + } + + auto coordinator = std::make_shared(max_replicas_to_use); + auto external_tables = new_context->getExternalTables(); - auto read_from_remote = std::make_unique( - query_ast, - new_cluster, - storage_id, - header, - processed_stage, - new_context, - getThrottler(new_context), - std::move(scalars), - std::move(external_tables), - getLogger("ReadFromParallelRemoteReplicasStep"), - std::move(storage_limits)); - - query_plan.addStep(std::move(read_from_remote)); + + std::vector shuffled_pool; + if (max_replicas_to_use < shard.getAllNodeCount()) + { + // will be shuffled according to `load_balancing` setting + shuffled_pool = shard.pool->getShuffledPools(settings); + } + else + { + /// If all replicas in cluster are used for query execution, + /// try to preserve replicas order as in cluster definition. + /// It's important for data locality during query execution + /// independently of the query initiator + auto priority_func = [](size_t i) { return Priority{static_cast(i)}; }; + shuffled_pool = shard.pool->getShuffledPools(settings, priority_func); + } + + std::vector pools_to_use; + pools_to_use.reserve(shuffled_pool.size()); + for (auto & pool : shuffled_pool) + pools_to_use.emplace_back(std::move(pool.pool)); + + /// do not build local plan for distributed queries for now (address it later) + if (settings.allow_experimental_analyzer && settings.parallel_replicas_local_plan && !shard_num) + { + /// find local replica index in pool + std::optional local_replica_index; + for (size_t i = 0, s = pools_to_use.size(); i < s; ++i) + { + const auto & hostname = pools_to_use[i]->getHost(); + const auto found = std::find_if( + begin(shard.local_addresses), + end(shard.local_addresses), + [&hostname](const Cluster::Address & local_addr) { return hostname == local_addr.host_name; }); + if (found != shard.local_addresses.end()) + { + local_replica_index = i; + break; + } + } + if (!local_replica_index) + throw Exception( + ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, + "Local replica is not found in '{}' cluster definition, see 'cluster_for_parallel_replicas' setting", + new_cluster->getName()); + + // resize the pool but keep local replicas in it (and update its index) + chassert(max_replicas_to_use <= pools_to_use.size()); + if (local_replica_index >= max_replicas_to_use) + { + std::swap(pools_to_use[max_replicas_to_use - 1], pools_to_use[local_replica_index.value()]); + local_replica_index = max_replicas_to_use - 1; + } + pools_to_use.resize(max_replicas_to_use); + + auto [local_plan, with_parallel_replicas] = createLocalPlanForParallelReplicas( + query_ast, + header, + new_context, + processed_stage, + coordinator, + std::move(analyzed_read_from_merge_tree), + local_replica_index.value()); + + if (!with_parallel_replicas) + { + query_plan = std::move(*local_plan); + return; + } + + LOG_DEBUG(logger, "Local replica got replica number {}", local_replica_index.value()); + + auto read_from_remote = std::make_unique( + query_ast, + new_cluster, + storage_id, + coordinator, + header, + processed_stage, + new_context, + getThrottler(new_context), + std::move(scalars), + std::move(external_tables), + getLogger("ReadFromParallelRemoteReplicasStep"), + std::move(storage_limits), + std::move(pools_to_use), + local_replica_index); + + auto remote_plan = std::make_unique(); + remote_plan->addStep(std::move(read_from_remote)); + + DataStreams input_streams; + input_streams.reserve(2); + input_streams.emplace_back(local_plan->getCurrentDataStream()); + input_streams.emplace_back(remote_plan->getCurrentDataStream()); + + std::vector plans; + plans.emplace_back(std::move(local_plan)); + plans.emplace_back(std::move(remote_plan)); + + auto union_step = std::make_unique(std::move(input_streams)); + query_plan.unitePlans(std::move(union_step), std::move(plans)); + } + else + { + chassert(max_replicas_to_use <= pools_to_use.size()); + pools_to_use.resize(max_replicas_to_use); + + auto read_from_remote = std::make_unique( + query_ast, + new_cluster, + storage_id, + std::move(coordinator), + header, + processed_stage, + new_context, + getThrottler(new_context), + std::move(scalars), + std::move(external_tables), + getLogger("ReadFromParallelRemoteReplicasStep"), + std::move(storage_limits), + std::move(pools_to_use)); + + query_plan.addStep(std::move(read_from_remote)); + } } void executeQueryWithParallelReplicas( @@ -540,7 +669,8 @@ void executeQueryWithParallelReplicas( const QueryTreeNodePtr & query_tree, const PlannerContextPtr & planner_context, ContextPtr context, - std::shared_ptr storage_limits) + std::shared_ptr storage_limits, + QueryPlanStepPtr analyzed_read_from_merge_tree) { QueryTreeNodePtr modified_query_tree = query_tree->clone(); rewriteJoinToGlobalJoin(modified_query_tree, context); @@ -550,7 +680,8 @@ void executeQueryWithParallelReplicas( = InterpreterSelectQueryAnalyzer::getSampleBlock(modified_query_tree, context, SelectQueryOptions(processed_stage).analyze()); auto modified_query_ast = queryNodeToDistributedSelectQuery(modified_query_tree); - executeQueryWithParallelReplicas(query_plan, storage_id, header, processed_stage, modified_query_ast, context, storage_limits); + executeQueryWithParallelReplicas( + query_plan, storage_id, header, processed_stage, modified_query_ast, context, storage_limits, std::move(analyzed_read_from_merge_tree)); } void executeQueryWithParallelReplicas( @@ -646,6 +777,49 @@ void executeQueryWithParallelReplicasCustomKey( context, query_info.query, storage_id.getDatabaseName(), storage_id.getTableName(), /*table_function_ptr=*/nullptr); executeQueryWithParallelReplicasCustomKey(query_plan, storage_id, query_info, columns, snapshot, processed_stage, header, context); } + +bool canUseParallelReplicasOnInitiator(const ContextPtr & context) +{ + if (!context->canUseParallelReplicasOnInitiator()) + return false; + + auto cluster = context->getClusterForParallelReplicas(); + if (cluster->getShardCount() == 1) + return cluster->getShardsInfo()[0].getAllNodeCount() > 1; + + /// parallel replicas with distributed table + auto scalars = context->hasQueryContext() ? context->getQueryContext()->getScalars() : Scalars{}; + UInt64 shard_num = 0; /// shard_num is 1-based, so 0 - no shard specified + const auto it = scalars.find("_shard_num"); + if (it != scalars.end()) + { + const Block & block = it->second; + const auto & column = block.safeGetByPosition(0).column; + shard_num = column->getUInt(0); + } + if (shard_num > 0) + { + const auto shard_count = cluster->getShardCount(); + if (shard_num > shard_count) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Shard number is greater than shard count: shard_num={} shard_count={} cluster={}", + shard_num, + shard_count, + cluster->getName()); + + return cluster->getShardsInfo().at(shard_num - 1).getAllNodeCount() > 1; + } + + if (cluster->getShardCount() > 1) + throw DB::Exception( + ErrorCodes::UNEXPECTED_CLUSTER, + "`cluster_for_parallel_replicas` setting refers to cluster with {} shards. Expected a cluster with one shard", + cluster->getShardCount()); + + return false; +} + } } diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index c22fcd24f03c..2a21f3e82551 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -35,6 +35,9 @@ using QueryTreeNodePtr = std::shared_ptr; class PlannerContext; using PlannerContextPtr = std::shared_ptr; +class IQueryPlanStep; +using QueryPlanStepPtr = std::unique_ptr; + namespace ClusterProxy { @@ -55,6 +58,8 @@ using AdditionalShardFilterGenerator = std::function; AdditionalShardFilterGenerator getShardFilterGeneratorForCustomKey(const Cluster & cluster, ContextPtr context, const ColumnsDescription & columns); +bool canUseParallelReplicasOnInitiator(const ContextPtr & context); + /// Execute a distributed query, creating a query plan, from which the query pipeline can be built. /// `stream_factory` object encapsulates the logic of creating plans for a different type of query /// (currently SELECT, DESCRIBE). @@ -81,7 +86,8 @@ void executeQueryWithParallelReplicas( QueryProcessingStage::Enum processed_stage, const ASTPtr & query_ast, ContextPtr context, - std::shared_ptr storage_limits); + std::shared_ptr storage_limits, + QueryPlanStepPtr read_from_merge_tree = nullptr); void executeQueryWithParallelReplicas( QueryPlan & query_plan, @@ -98,7 +104,8 @@ void executeQueryWithParallelReplicas( const QueryTreeNodePtr & query_tree, const PlannerContextPtr & planner_context, ContextPtr context, - std::shared_ptr storage_limits); + std::shared_ptr storage_limits, + QueryPlanStepPtr read_from_merge_tree); void executeQueryWithParallelReplicasCustomKey( QueryPlan & query_plan, diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index a967503dfb56..d12ff92b0f1b 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1367,7 +1367,7 @@ ConfigurationPtr Context::getUsersConfig() return shared->users_config; } -void Context::setUser(const UUID & user_id_) +void Context::setUser(const UUID & user_id_, const std::vector & external_roles_) { /// Prepare lists of user's profiles, constraints, settings, roles. /// NOTE: AccessControl::read() and other AccessControl's functions may require some IO work, @@ -1382,7 +1382,6 @@ void Context::setUser(const UUID & user_id_) const auto & database = user->default_database; /// Apply user's profiles, constraints, settings, roles. - std::lock_guard lock(mutex); setUserIDWithLock(user_id_, lock); @@ -1392,6 +1391,7 @@ void Context::setUser(const UUID & user_id_) setCurrentProfilesWithLock(*enabled_profiles, /* check_constraints= */ false, lock); setCurrentRolesWithLock(default_roles, lock); + setExternalRolesWithLock(external_roles_, lock); /// It's optional to specify the DEFAULT DATABASE in the user's definition. if (!database.empty()) @@ -1435,6 +1435,18 @@ void Context::setCurrentRolesWithLock(const std::vector & new_current_role need_recalculate_access = true; } +void Context::setExternalRolesWithLock(const std::vector & new_external_roles, const std::lock_guard &) +{ + if (!new_external_roles.empty()) + { + if (current_roles) + current_roles->insert(current_roles->end(), new_external_roles.begin(), new_external_roles.end()); + else + current_roles = std::make_shared>(new_external_roles); + need_recalculate_access = true; + } +} + void Context::setCurrentRolesImpl(const std::vector & new_current_roles, bool throw_if_not_granted, bool skip_if_not_granted, const std::shared_ptr & user) { if (skip_if_not_granted) @@ -1548,7 +1560,8 @@ std::shared_ptr Context::getAccess() const /// If setUserID() was never called then this must be the global context with the full access. bool full_access = !user_id; - return ContextAccessParams{user_id, full_access, /* use_default_roles= */ false, current_roles, *settings, current_database, client_info}; + return ContextAccessParams{ + user_id, full_access, /* use_default_roles= */ false, current_roles, external_roles, *settings, current_database, client_info}; }; /// Check if the current access rights are still valid, otherwise get parameters for recalculating access rights. @@ -5594,24 +5607,13 @@ std::shared_ptr Context::getAsyncReadCounters() const return async_read_counters; } -Context::ParallelReplicasMode Context::getParallelReplicasMode() const -{ - const auto & settings_ref = getSettingsRef(); - - using enum Context::ParallelReplicasMode; - if (!settings_ref.parallel_replicas_custom_key.value.empty()) - return CUSTOM_KEY; - - if (settings_ref.allow_experimental_parallel_reading_from_replicas > 0) - return READ_TASKS; - - return SAMPLE_KEY; -} - bool Context::canUseTaskBasedParallelReplicas() const { const auto & settings_ref = getSettingsRef(); - return getParallelReplicasMode() == ParallelReplicasMode::READ_TASKS && settings_ref.max_parallel_replicas > 1; + + return settings_ref.allow_experimental_parallel_reading_from_replicas > 0 + && settings_ref.parallel_replicas_mode == ParallelReplicasMode::READ_TASKS + && settings_ref.max_parallel_replicas > 1; } bool Context::canUseParallelReplicasOnInitiator() const @@ -5626,7 +5628,15 @@ bool Context::canUseParallelReplicasOnFollower() const bool Context::canUseParallelReplicasCustomKey() const { - return settings->max_parallel_replicas > 1 && getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY; + const auto & settings_ref = getSettingsRef(); + + const bool has_enough_servers = settings_ref.max_parallel_replicas > 1; + const bool parallel_replicas_enabled = settings_ref.allow_experimental_parallel_reading_from_replicas > 0; + const bool is_parallel_replicas_with_custom_key = + settings_ref.parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_SAMPLING || + settings_ref.parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_RANGE; + + return has_enough_servers && parallel_replicas_enabled && is_parallel_replicas_with_custom_key; } bool Context::canUseParallelReplicasCustomKeyForCluster(const Cluster & cluster) const @@ -5636,8 +5646,23 @@ bool Context::canUseParallelReplicasCustomKeyForCluster(const Cluster & cluster) bool Context::canUseOffsetParallelReplicas() const { - return offset_parallel_replicas_enabled && settings->max_parallel_replicas > 1 - && getParallelReplicasMode() != Context::ParallelReplicasMode::READ_TASKS; + const auto & settings_ref = getSettingsRef(); + + /** + * Offset parallel replicas algorithm is not only the one which relies on native SAMPLING KEY, + * but also those which rely on customer-provided "custom" key. + * We combine them together into one group for convenience. + */ + const bool has_enough_servers = settings_ref.max_parallel_replicas > 1; + const bool parallel_replicas_enabled = settings_ref.allow_experimental_parallel_reading_from_replicas > 0; + const bool is_parallel_replicas_with_custom_key_or_native_sampling_key = + settings_ref.parallel_replicas_mode == ParallelReplicasMode::SAMPLING_KEY || + settings_ref.parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_SAMPLING || + settings_ref.parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_RANGE; + return offset_parallel_replicas_enabled && + has_enough_servers && + parallel_replicas_enabled && + is_parallel_replicas_with_custom_key_or_native_sampling_key; } void Context::disableOffsetParallelReplicas() diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 4b52d28dd9ed..a4fa924d89b8 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -277,6 +277,7 @@ class ContextData std::optional user_id; std::shared_ptr> current_roles; + std::shared_ptr> external_roles; std::shared_ptr settings_constraints_and_current_profiles; mutable std::shared_ptr access; mutable bool need_recalculate_access = true; @@ -613,7 +614,7 @@ class Context: public ContextData, public std::enable_shared_from_this /// Sets the current user assuming that he/she is already authenticated. /// WARNING: This function doesn't check password! - void setUser(const UUID & user_id_); + void setUser(const UUID & user_id_, const std::vector & external_roles_ = {}); UserPtr getUser() const; std::optional getUserID() const; @@ -1332,15 +1333,6 @@ class Context: public ContextData, public std::enable_shared_from_this ClusterPtr getClusterForParallelReplicas() const; - enum class ParallelReplicasMode : uint8_t - { - SAMPLE_KEY, - CUSTOM_KEY, - READ_TASKS, - }; - - ParallelReplicasMode getParallelReplicasMode() const; - void setPreparedSetsCache(const PreparedSetsCachePtr & cache); PreparedSetsCachePtr getPreparedSetsCache() const; @@ -1357,6 +1349,8 @@ class Context: public ContextData, public std::enable_shared_from_this void setCurrentRolesWithLock(const std::vector & new_current_roles, const std::lock_guard & lock); + void setExternalRolesWithLock(const std::vector & new_external_roles, const std::lock_guard & lock); + void setSettingWithLock(std::string_view name, const String & value, const std::lock_guard & lock); void setSettingWithLock(std::string_view name, const Field & value, const std::lock_guard & lock); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index d3334548bc63..9741519801b7 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -597,7 +597,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( settings.parallel_replicas_count, settings.parallel_replica_offset, std::move(custom_key_ast), - {settings.parallel_replicas_custom_key_filter_type, + {settings.parallel_replicas_mode, settings.parallel_replicas_custom_key_range_lower, settings.parallel_replicas_custom_key_range_upper}, storage->getInMemoryMetadataPtr()->columns, @@ -2314,12 +2314,12 @@ void InterpreterSelectQuery::addPrewhereAliasActions() } /// Based on the query analysis, check if using a trivial count (storage or partition metadata) is possible -std::optional InterpreterSelectQuery::getTrivialCount(UInt64 max_parallel_replicas) +std::optional InterpreterSelectQuery::getTrivialCount(UInt64 allow_experimental_parallel_reading_from_replicas) { const Settings & settings = context->getSettingsRef(); bool optimize_trivial_count = syntax_analyzer_result->optimize_trivial_count - && (max_parallel_replicas <= 1) + && (allow_experimental_parallel_reading_from_replicas == 0) && !settings.allow_experimental_query_deduplication && !settings.empty_result_for_aggregation_by_empty_set && storage @@ -2403,7 +2403,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc std::optional num_rows; /// Optimization for trivial query like SELECT count() FROM table. - if (processing_stage == QueryProcessingStage::FetchColumns && (num_rows = getTrivialCount(settings.max_parallel_replicas))) + if (processing_stage == QueryProcessingStage::FetchColumns && (num_rows = getTrivialCount(settings.allow_experimental_parallel_reading_from_replicas))) { const auto & desc = query_analyzer->aggregates()[0]; const auto & func = desc.function; diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index a01a390f3c76..d70d9ea3eea0 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -195,7 +195,7 @@ class InterpreterSelectQuery : public IInterpreterUnionOrSelectQuery void executeExtremes(QueryPlan & query_plan); void executeSubqueriesInSetsAndJoins(QueryPlan & query_plan); bool autoFinalOnQuery(ASTSelectQuery & select_query); - std::optional getTrivialCount(UInt64 max_parallel_replicas); + std::optional getTrivialCount(UInt64 allow_experimental_parallel_reading_from_replicas); /// Check if we can limit block size to read based on LIMIT clause UInt64 maxBlockSizeByLimit() const; diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 58e9c4695686..61a432cde292 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -147,6 +147,7 @@ ColumnDependencies getAllColumnDependencies( bool isStorageTouchedByMutations( MergeTreeData::DataPartPtr source_part, + MergeTreeData::MutationsSnapshotPtr mutations_snapshot, const StorageMetadataPtr & metadata_snapshot, const std::vector & commands, ContextPtr context) @@ -154,7 +155,7 @@ bool isStorageTouchedByMutations( if (commands.empty()) return false; - auto storage_from_part = std::make_shared(source_part); + auto storage_from_part = std::make_shared(source_part, mutations_snapshot); bool all_commands_can_be_skipped = true; for (const auto & command : commands) @@ -285,8 +286,13 @@ MutationsInterpreter::Source::Source(StoragePtr storage_) : storage(std::move(st { } -MutationsInterpreter::Source::Source(MergeTreeData & storage_, MergeTreeData::DataPartPtr source_part_) - : data(&storage_), part(std::move(source_part_)) +MutationsInterpreter::Source::Source( + MergeTreeData & storage_, + MergeTreeData::DataPartPtr source_part_, + AlterConversionsPtr alter_conversions_) + : data(&storage_) + , part(std::move(source_part_)) + , alter_conversions(std::move(alter_conversions_)) { } @@ -386,13 +392,14 @@ MutationsInterpreter::MutationsInterpreter( MutationsInterpreter::MutationsInterpreter( MergeTreeData & storage_, MergeTreeData::DataPartPtr source_part_, + AlterConversionsPtr alter_conversions_, StorageMetadataPtr metadata_snapshot_, MutationCommands commands_, Names available_columns_, ContextPtr context_, Settings settings_) : MutationsInterpreter( - Source(storage_, std::move(source_part_)), + Source(storage_, std::move(source_part_), std::move(alter_conversions_)), std::move(metadata_snapshot_), std::move(commands_), std::move(available_columns_), std::move(context_), std::move(settings_)) { @@ -1212,7 +1219,7 @@ void MutationsInterpreter::Source::read( createReadFromPartStep( MergeTreeSequentialSourceType::Mutation, plan, *data, storage_snapshot, - part, required_columns, + part, alter_conversions, required_columns, apply_deleted_mask_, std::move(filter), context_, getLogger("MutationsInterpreter")); } diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h index 57863e9ae737..8601558b7882 100644 --- a/src/Interpreters/MutationsInterpreter.h +++ b/src/Interpreters/MutationsInterpreter.h @@ -20,6 +20,7 @@ using QueryPipelineBuilderPtr = std::unique_ptr; /// Return false if the data isn't going to be changed by mutations. bool isStorageTouchedByMutations( MergeTreeData::DataPartPtr source_part, + MergeTreeData::MutationsSnapshotPtr mutations_snapshot, const StorageMetadataPtr & metadata_snapshot, const std::vector & commands, ContextPtr context @@ -70,6 +71,7 @@ class MutationsInterpreter MutationsInterpreter( MergeTreeData & storage_, MergeTreeData::DataPartPtr source_part_, + AlterConversionsPtr alter_conversions_, StorageMetadataPtr metadata_snapshot_, MutationCommands commands_, Names available_columns_, @@ -137,7 +139,7 @@ class MutationsInterpreter bool can_execute_) const; explicit Source(StoragePtr storage_); - Source(MergeTreeData & storage_, MergeTreeData::DataPartPtr source_part_); + Source(MergeTreeData & storage_, MergeTreeData::DataPartPtr source_part_, AlterConversionsPtr alter_conversions_); private: StoragePtr storage; @@ -145,6 +147,7 @@ class MutationsInterpreter /// Special case for *MergeTree. MergeTreeData * data = nullptr; MergeTreeData::DataPartPtr part; + AlterConversionsPtr alter_conversions; }; private: diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 9b74ae2aa930..e7e3d1f84b28 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -25,7 +27,6 @@ #include #include - namespace DB { @@ -286,7 +287,7 @@ void Session::shutdownNamedSessions() Session::Session(const ContextPtr & global_context_, ClientInfo::Interface interface_, bool is_secure, const std::string & certificate) : auth_id(UUIDHelpers::generateV4()), global_context(global_context_), - log(getLogger(String{magic_enum::enum_name(interface_)} + "-Session")) + log(getLogger(String{magic_enum::enum_name(interface_)} + "-Session-" + toString(auth_id))) { prepared_client_info.emplace(); prepared_client_info->interface = interface_; @@ -331,12 +332,12 @@ AuthenticationType Session::getAuthenticationTypeOrLogInFailure(const String & u } } -void Session::authenticate(const String & user_name, const String & password, const Poco::Net::SocketAddress & address) +void Session::authenticate(const String & user_name, const String & password, const Poco::Net::SocketAddress & address, const Strings & external_roles_) { - authenticate(BasicCredentials{user_name, password}, address); + authenticate(BasicCredentials{user_name, password}, address, external_roles_); } -void Session::authenticate(const Credentials & credentials_, const Poco::Net::SocketAddress & address_) +void Session::authenticate(const Credentials & credentials_, const Poco::Net::SocketAddress & address_, const Strings & external_roles_) { if (session_context) throw Exception(ErrorCodes::LOGICAL_ERROR, "If there is a session context it must be created after authentication"); @@ -348,8 +349,8 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So if ((address == Poco::Net::SocketAddress{}) && (prepared_client_info->interface == ClientInfo::Interface::LOCAL)) address = Poco::Net::SocketAddress{"127.0.0.1", 0}; - LOG_DEBUG(log, "{} Authenticating user '{}' from {}", - toString(auth_id), credentials_.getUserName(), address.toString()); + LOG_DEBUG(log, "Authenticating user '{}' from {}", + credentials_.getUserName(), address.toString()); try { @@ -358,6 +359,14 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So settings_from_auth_server = auth_result.settings; LOG_DEBUG(log, "{} Authenticated with global context as user {}", toString(auth_id), toString(*user_id)); + + if (!external_roles_.empty() && global_context->getSettingsRef().push_external_roles_in_interserver_queries) + { + external_roles = global_context->getAccessControl().find(external_roles_); + + LOG_DEBUG(log, "User {} has external_roles applied: [{}] ({})", + toString(*user_id), fmt::join(external_roles_, ", "), external_roles_.size()); + } } catch (const Exception & e) { @@ -382,7 +391,7 @@ void Session::checkIfUserIsStillValid() void Session::onAuthenticationFailure(const std::optional & user_name, const Poco::Net::SocketAddress & address_, const Exception & e) { - LOG_DEBUG(log, "{} Authentication failed with error: {}", toString(auth_id), e.what()); + LOG_DEBUG(log, "Authentication failed with error: {}", e.what()); if (auto session_log = getSessionLog()) { /// Add source address to the log @@ -508,8 +517,8 @@ ContextMutablePtr Session::makeSessionContext() if (session_tracker_handle) throw Exception(ErrorCodes::LOGICAL_ERROR, "Session tracker handle was created before making session"); - LOG_DEBUG(log, "{} Creating session context with user_id: {}", - toString(auth_id), toString(*user_id)); + LOG_DEBUG(log, "Creating session context with user_id: {}", + toString(*user_id)); /// Make a new session context. ContextMutablePtr new_session_context; new_session_context = Context::createCopy(global_context); @@ -520,7 +529,7 @@ ContextMutablePtr Session::makeSessionContext() prepared_client_info.reset(); /// Set user information for the new context: current profiles, roles, access rights. - new_session_context->setUser(*user_id); + new_session_context->setUser(*user_id, external_roles); /// Session context is ready. session_context = new_session_context; @@ -551,8 +560,8 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std: if (session_tracker_handle) throw Exception(ErrorCodes::LOGICAL_ERROR, "Session tracker handle was created before making session"); - LOG_DEBUG(log, "{} Creating named session context with name: {}, user_id: {}", - toString(auth_id), session_name_, toString(*user_id)); + LOG_DEBUG(log, "Creating named session context with name: {}, user_id: {}", + session_name_, toString(*user_id)); /// Make a new session context OR /// if the `session_id` and `user_id` were used before then just get a previously created session context. @@ -575,7 +584,7 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std: /// Set user information for the new context: current profiles, roles, access rights. if (!access->tryGetUser()) { - new_session_context->setUser(*user_id); + new_session_context->setUser(*user_id, external_roles); max_sessions_for_user = new_session_context->getSettingsRef().max_sessions_for_user; } else @@ -627,7 +636,7 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t throw Exception(ErrorCodes::LOGICAL_ERROR, "Query context must be created after authentication"); /// We can create a query context either from a session context or from a global context. - bool from_session_context = static_cast(session_context); + const bool from_session_context = static_cast(session_context); /// Create a new query context. ContextMutablePtr query_context = Context::createCopy(from_session_context ? session_context : global_context); @@ -667,7 +676,7 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t /// Set user information for the new context: current profiles, roles, access rights. if (user_id && !query_context->getAccess()->tryGetUser()) - query_context->setUser(*user_id); + query_context->setUser(*user_id, external_roles); /// Query context is ready. query_context_created = true; diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index fc41c78e666d..800aadb001e1 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace Poco::Net { class SocketAddress; } @@ -50,8 +51,11 @@ class Session /// Sets the current user, checks the credentials and that the specified address is allowed to connect from. /// The function throws an exception if there is no such user or password is wrong. - void authenticate(const String & user_name, const String & password, const Poco::Net::SocketAddress & address); - void authenticate(const Credentials & credentials_, const Poco::Net::SocketAddress & address_); + void authenticate(const String & user_name, const String & password, const Poco::Net::SocketAddress & address, const Strings & external_roles_ = {}); + + /// `external_roles_` names of the additional roles (over what is granted via local access control mechanisms) that would be granted to user during this session. + /// Role is not granted if it can't be found by name via AccessControl (i.e. doesn't exist on this instance). + void authenticate(const Credentials & credentials_, const Poco::Net::SocketAddress & address_, const Strings & external_roles_ = {}); // Verifies whether the user's validity extends beyond the current time. // Throws an exception if the user's validity has expired. @@ -114,6 +118,8 @@ class Session mutable UserPtr user; std::optional user_id; + std::vector external_roles; + ContextMutablePtr session_context; mutable bool query_context_created = false; diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h index 8b44f36b278d..45c570a9bed7 100644 --- a/src/Interpreters/Set.h +++ b/src/Interpreters/Set.h @@ -239,6 +239,8 @@ class MergeTreeSetIndex const Columns & getOrderedSet() const { return ordered_set; } + const std::vector & getIndexesMapping() const { return indexes_mapping; } + private: // If all arguments in tuple are key columns, we can optimize NOT IN when there is only one element. bool has_all_keys; diff --git a/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp b/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp index 6f902c7cd7e9..7b17cecefb08 100644 --- a/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp +++ b/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp @@ -31,7 +31,8 @@ ASTPtr getCustomKeyFilterForParallelReplica( const ContextPtr & context) { chassert(replicas_count > 1); - if (filter.filter_type == ParallelReplicasCustomKeyFilterType::DEFAULT) + chassert(filter.filter_type == ParallelReplicasMode::CUSTOM_KEY_SAMPLING || filter.filter_type == ParallelReplicasMode::CUSTOM_KEY_RANGE); + if (filter.filter_type == ParallelReplicasMode::CUSTOM_KEY_SAMPLING) { // first we do modulo with replica count auto modulo_function = makeASTFunction("positiveModulo", custom_key_ast, std::make_shared(replicas_count)); @@ -42,7 +43,7 @@ ASTPtr getCustomKeyFilterForParallelReplica( return equals_function; } - chassert(filter.filter_type == ParallelReplicasCustomKeyFilterType::RANGE); + chassert(filter.filter_type == ParallelReplicasMode::CUSTOM_KEY_RANGE); KeyDescription custom_key_description = KeyDescription::getKeyFromAST(custom_key_ast, columns, context); diff --git a/src/Interpreters/getCustomKeyFilterForParallelReplicas.h b/src/Interpreters/getCustomKeyFilterForParallelReplicas.h index dfee5123ecb1..28fa1e45ebbf 100644 --- a/src/Interpreters/getCustomKeyFilterForParallelReplicas.h +++ b/src/Interpreters/getCustomKeyFilterForParallelReplicas.h @@ -12,7 +12,7 @@ namespace DB { struct ParallelReplicasCustomKeyFilter { - ParallelReplicasCustomKeyFilterType filter_type; + ParallelReplicasMode filter_type; UInt64 range_lower; UInt64 range_upper; }; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index bc31af32a20c..1754b2102f63 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -3,6 +3,7 @@ #include #include +#include #include @@ -290,14 +291,21 @@ bool applyTrivialCountIfPossible( if (!num_rows) return false; - if (settings.max_parallel_replicas > 1) + if (settings.allow_experimental_parallel_reading_from_replicas > 0 && settings.max_parallel_replicas > 1) { - if (!settings.parallel_replicas_custom_key.value.empty() || settings.allow_experimental_parallel_reading_from_replicas == 0) + /// Imagine the situation when we have a query with parallel replicas and + /// this code executed on the remote server. + /// If we will apply trivial count optimization, then each remote server will do the same + /// and we will have N times more rows as the result on the initiator. + /// TODO: This condition seems unneeded when we will make the parallel replicas with custom key + /// to work on top of MergeTree instead of Distributed. + if (settings.parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_RANGE || + settings.parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_SAMPLING || + settings.parallel_replicas_mode == ParallelReplicasMode::SAMPLING_KEY) return false; /// The query could use trivial count if it didn't use parallel replicas, so let's disable it query_context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); - query_context->setSetting("max_parallel_replicas", UInt64{1}); LOG_TRACE(getLogger("Planner"), "Disabling parallel replicas to be able to use a trivial count optimization"); } @@ -504,7 +512,7 @@ std::optional buildCustomKeyFilterIfNeeded(const StoragePtr & sto settings.parallel_replicas_count, settings.parallel_replica_offset, std::move(custom_key_ast), - {settings.parallel_replicas_custom_key_filter_type, + {settings.parallel_replicas_mode, settings.parallel_replicas_custom_key_range_lower, settings.parallel_replicas_custom_key_range_upper}, storage->getInMemoryMetadataPtr()->columns, @@ -893,31 +901,6 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres /// query_plan can be empty if there is nothing to read if (query_plan.isInitialized() && parallel_replicas_enabled_for_storage(storage, settings)) { - // (1) find read step - QueryPlan::Node * node = query_plan.getRootNode(); - ReadFromMergeTree * reading = nullptr; - while (node) - { - reading = typeid_cast(node->step.get()); - if (reading) - break; - - QueryPlan::Node * prev_node = node; - if (!node->children.empty()) - { - chassert(node->children.size() == 1); - node = node->children.at(0); - } - else - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Step is expected to be ReadFromMergeTree but it's {}", - prev_node->step->getName()); - } - } - - chassert(reading); if (query_context->canUseParallelReplicasCustomKey() && query_context->getClientInfo().distributed_depth == 0) { if (auto cluster = query_context->getClusterForParallelReplicas(); @@ -940,21 +923,50 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres query_plan = std::move(query_plan_parallel_replicas); } } - else if (query_context->canUseParallelReplicasOnInitiator()) + else if (ClusterProxy::canUseParallelReplicasOnInitiator(query_context)) { + // (1) find read step + QueryPlan::Node * node = query_plan.getRootNode(); + ReadFromMergeTree * reading = nullptr; + while (node) + { + reading = typeid_cast(node->step.get()); + if (reading) + break; + + QueryPlan::Node * prev_node = node; + if (!node->children.empty()) + { + chassert(node->children.size() == 1); + node = node->children.at(0); + } + else + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Step is expected to be ReadFromMergeTree but it's {}", + prev_node->step->getName()); + } + } + + chassert(reading); + // (2) if it's ReadFromMergeTree - run index analysis and check number of rows to read if (settings.parallel_replicas_min_number_of_rows_per_replica > 0) { auto result_ptr = reading->selectRangesToRead(); - UInt64 rows_to_read = result_ptr->selected_rows; + + reading->setAnalyzedResult(std::move(result_ptr)); + if (table_expression_query_info.trivial_limit > 0 && table_expression_query_info.trivial_limit < rows_to_read) rows_to_read = table_expression_query_info.trivial_limit; if (max_block_size_limited && (max_block_size_limited < rows_to_read)) rows_to_read = max_block_size_limited; - const size_t number_of_replicas_to_use = rows_to_read / settings.parallel_replicas_min_number_of_rows_per_replica; + const size_t number_of_replicas_to_use + = rows_to_read / settings.parallel_replicas_min_number_of_rows_per_replica; LOG_TRACE( getLogger("Planner"), "Estimated {} rows to read. It is enough work for {} parallel replicas", @@ -980,6 +992,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres { from_stage = QueryProcessingStage::WithMergeableState; QueryPlan query_plan_parallel_replicas; + QueryPlanStepPtr reading_step = std::move(node->step); ClusterProxy::executeQueryWithParallelReplicas( query_plan_parallel_replicas, storage->getStorageID(), @@ -987,9 +1000,24 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres table_expression_query_info.query_tree, table_expression_query_info.planner_context, query_context, - table_expression_query_info.storage_limits); + table_expression_query_info.storage_limits, + std::move(reading_step)); query_plan = std::move(query_plan_parallel_replicas); } + else + { + QueryPlan query_plan_no_parallel_replicas; + storage->read( + query_plan_no_parallel_replicas, + columns_names, + storage_snapshot, + table_expression_query_info, + query_context, + from_stage, + max_block_size, + max_streams); + query_plan = std::move(query_plan_no_parallel_replicas); + } } } diff --git a/src/Planner/findParallelReplicasQuery.cpp b/src/Planner/findParallelReplicasQuery.cpp index 39edb1e6516b..25481d06670a 100644 --- a/src/Planner/findParallelReplicasQuery.cpp +++ b/src/Planner/findParallelReplicasQuery.cpp @@ -52,7 +52,13 @@ std::stack getSupportingParallelReplicasQuery(const IQueryTre const auto & storage = table_node.getStorage(); /// Here we check StorageDummy as well, to support a query tree with replaced storages. if (std::dynamic_pointer_cast(storage) || typeid_cast(storage.get())) + { + /// parallel replicas is not supported with FINAL + if (table_node.getTableExpressionModifiers() && table_node.getTableExpressionModifiers()->hasFinal()) + return {}; + return res; + } return {}; } diff --git a/src/Processors/Formats/Impl/ArrowFieldIndexUtil.h b/src/Processors/Formats/Impl/ArrowFieldIndexUtil.h index 24ffdc105811..fca954b6cfb4 100644 --- a/src/Processors/Formats/Impl/ArrowFieldIndexUtil.h +++ b/src/Processors/Formats/Impl/ArrowFieldIndexUtil.h @@ -15,6 +15,7 @@ #include #include #include +#include namespace arrow @@ -65,11 +66,22 @@ class ArrowFieldIndexUtil return result; } + // For a parquet schema {x: {i: int, j: int}}, this should be populated as follows + // clickhouse_index = 0, parquet_indexes = {0, 1} + struct ClickHouseIndexToParquetIndex + { + std::size_t clickhouse_index; + std::vector parquet_indexes; + }; + /// Only collect the required fields' indices. Eg. when just read a field of a struct, /// don't need to collect the whole indices in this struct. - std::vector findRequiredIndices(const Block & header, const arrow::Schema & schema) + std::vector findRequiredIndices( + const Block & header, + const arrow::Schema & schema, + const parquet::FileMetaData & file) { - std::vector required_indices; + std::vector required_indices; std::unordered_set added_indices; /// Flat all named fields' index information into a map. auto fields_indices = calculateFieldIndices(schema); @@ -79,7 +91,7 @@ class ArrowFieldIndexUtil std::string col_name = named_col.name; if (ignore_case) boost::to_lower(col_name); - findRequiredIndices(col_name, named_col.type, fields_indices, added_indices, required_indices); + findRequiredIndices(col_name, i, named_col.type, fields_indices, added_indices, required_indices, file); } return required_indices; } @@ -169,10 +181,12 @@ class ArrowFieldIndexUtil void findRequiredIndices( const String & name, + std::size_t header_index, DataTypePtr data_type, const std::unordered_map> & field_indices, std::unordered_set & added_indices, - std::vector & required_indices) + std::vector & required_indices, + const parquet::FileMetaData & file) { auto nested_type = removeNullable(data_type); if (const DB::DataTypeTuple * type_tuple = typeid_cast(nested_type.get())) @@ -187,20 +201,20 @@ class ArrowFieldIndexUtil if (ignore_case) boost::to_lower(field_name); const auto & field_type = field_types[i]; - findRequiredIndices(Nested::concatenateName(name, field_name), field_type, field_indices, added_indices, required_indices); + findRequiredIndices(Nested::concatenateName(name, field_name), header_index, field_type, field_indices, added_indices, required_indices, file); } return; } } else if (const auto * type_array = typeid_cast(nested_type.get())) { - findRequiredIndices(name, type_array->getNestedType(), field_indices, added_indices, required_indices); + findRequiredIndices(name, header_index, type_array->getNestedType(), field_indices, added_indices, required_indices, file); return; } else if (const auto * type_map = typeid_cast(nested_type.get())) { - findRequiredIndices(name, type_map->getKeyType(), field_indices, added_indices, required_indices); - findRequiredIndices(name, type_map->getValueType(), field_indices, added_indices, required_indices); + findRequiredIndices(name, header_index, type_map->getKeyType(), field_indices, added_indices, required_indices, file); + findRequiredIndices(name, header_index, type_map->getValueType(), field_indices, added_indices, required_indices, file); return; } auto it = field_indices.find(name); @@ -211,14 +225,18 @@ class ArrowFieldIndexUtil } else { + ClickHouseIndexToParquetIndex index_mapping; + index_mapping.clickhouse_index = header_index; for (int j = 0; j < it->second.second; ++j) { auto index = it->second.first + j; if (added_indices.insert(index).second) { - required_indices.emplace_back(index); + index_mapping.parquet_indexes.emplace_back(index); } } + + required_indices.emplace_back(index_mapping); } } }; diff --git a/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.cpp b/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.cpp new file mode 100644 index 000000000000..75eeb15a5190 --- /dev/null +++ b/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.cpp @@ -0,0 +1,525 @@ +#include +#include + +#if USE_PARQUET + +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace +{ + +bool isParquetStringTypeSupportedForBloomFilters( + const std::shared_ptr & logical_type, + parquet::ConvertedType::type converted_type) +{ + if (logical_type && + !logical_type->is_none() + && !(logical_type->is_string() || logical_type->is_BSON() || logical_type->is_JSON())) + { + return false; + } + + if (parquet::ConvertedType::type::NONE != converted_type && + !(converted_type == parquet::ConvertedType::JSON || converted_type == parquet::ConvertedType::UTF8 + || converted_type == parquet::ConvertedType::BSON)) + { + return false; + } + + return true; +} + +bool isParquetIntegerTypeSupportedForBloomFilters(const std::shared_ptr & logical_type, parquet::ConvertedType::type converted_type) +{ + if (logical_type && !logical_type->is_none() && !logical_type->is_int()) + { + return false; + } + + if (parquet::ConvertedType::type::NONE != converted_type && !(converted_type == parquet::ConvertedType::INT_8 || converted_type == parquet::ConvertedType::INT_16 + || converted_type == parquet::ConvertedType::INT_32 || converted_type == parquet::ConvertedType::INT_64 + || converted_type == parquet::ConvertedType::UINT_8 || converted_type == parquet::ConvertedType::UINT_16 + || converted_type == parquet::ConvertedType::UINT_32 || converted_type == parquet::ConvertedType::UINT_64)) + { + return false; + } + + return true; +} + +template +uint64_t hashSpecialFLBATypes(const Field & field) +{ + const T & value = field.safeGet(); + + parquet::FLBA flba(reinterpret_cast(&value)); + + parquet::XxHasher hasher; + + return hasher.Hash(&flba, sizeof(T)); +}; + +std::optional tryHashStringWithoutCompatibilityCheck(const Field & field) +{ + const auto field_type = field.getType(); + + if (field_type != Field::Types::Which::String) + { + return std::nullopt; + } + + parquet::XxHasher hasher; + parquet::ByteArray ba { field.safeGet() }; + + return hasher.Hash(&ba); +} + +std::optional tryHashString( + const Field & field, + const std::shared_ptr & logical_type, + parquet::ConvertedType::type converted_type) +{ + if (!isParquetStringTypeSupportedForBloomFilters(logical_type, converted_type)) + { + return std::nullopt; + } + + return tryHashStringWithoutCompatibilityCheck(field); +} + +std::optional tryHashFLBA( + const Field & field, + const std::shared_ptr & logical_type, + parquet::ConvertedType::type converted_type, + std::size_t parquet_column_length) +{ + if (!isParquetStringTypeSupportedForBloomFilters(logical_type, converted_type)) + { + return std::nullopt; + } + + const auto field_type = field.getType(); + + if (field_type == Field::Types::Which::IPv6 && parquet_column_length == sizeof(IPv6)) + { + return hashSpecialFLBATypes(field); + } + + return tryHashStringWithoutCompatibilityCheck(field); +} + +template +std::optional tryHashInt(const Field & field, const std::shared_ptr & logical_type, parquet::ConvertedType::type converted_type) +{ + if (!isParquetIntegerTypeSupportedForBloomFilters(logical_type, converted_type)) + { + return std::nullopt; + } + + parquet::XxHasher hasher; + + if (field.getType() == Field::Types::Which::Int64) + { + return hasher.Hash(static_cast(field.safeGet())); + } + else if (field.getType() == Field::Types::Which::UInt64) + { + return hasher.Hash(static_cast(field.safeGet())); + } + else if (field.getType() == Field::Types::IPv4) + { + /* + * In theory, we could accept IPv4 over 64 bits variables. It would only be a problem in case it was hashed using the byte array api + * with a zero-ed buffer that had a 32 bits variable copied into it. + * + * To be on the safe side, accept only in case physical type is 32 bits. + * */ + if constexpr (std::is_same_v) + { + return hasher.Hash(static_cast(field.safeGet())); + } + } + + return std::nullopt; +} + +std::optional tryHash(const Field & field, const parquet::ColumnDescriptor * parquet_column_descriptor) +{ + const auto physical_type = parquet_column_descriptor->physical_type(); + const auto & logical_type = parquet_column_descriptor->logical_type(); + const auto converted_type = parquet_column_descriptor->converted_type(); + + switch (physical_type) + { + case parquet::Type::type::INT32: + return tryHashInt(field, logical_type, converted_type); + case parquet::Type::type::INT64: + return tryHashInt(field, logical_type, converted_type); + case parquet::Type::type::BYTE_ARRAY: + return tryHashString(field, logical_type, converted_type); + case parquet::Type::type::FIXED_LEN_BYTE_ARRAY: + return tryHashFLBA(field, logical_type, converted_type, parquet_column_descriptor->type_length()); + default: + return std::nullopt; + } +} + +std::optional> hash(const IColumn * data_column, const parquet::ColumnDescriptor * parquet_column_descriptor) +{ + std::vector hashes; + + for (size_t i = 0u; i < data_column->size(); i++) + { + Field f; + data_column->get(i, f); + + auto hashed_value = tryHash(f, parquet_column_descriptor); + + if (!hashed_value) + { + return std::nullopt; + } + + hashes.emplace_back(*hashed_value); + } + + return hashes; +} + +bool maybeTrueOnBloomFilter(const std::vector & hashes, const std::unique_ptr & bloom_filter) +{ + for (const auto hash : hashes) + { + if (bloom_filter->FindHash(hash)) + { + return true; + } + } + + return false; +} + +const parquet::ColumnDescriptor * getColumnDescriptorIfBloomFilterIsPresent( + const std::unique_ptr & parquet_rg_metadata, + const std::vector & clickhouse_column_index_to_parquet_index, + std::size_t clickhouse_column_index) +{ + if (clickhouse_column_index_to_parquet_index.size() <= clickhouse_column_index) + { + return nullptr; + } + + const auto & parquet_indexes = clickhouse_column_index_to_parquet_index[clickhouse_column_index].parquet_indexes; + + // complex types like structs, tuples and maps will have more than one index. + // we don't support those for now + if (parquet_indexes.size() > 1) + { + return nullptr; + } + + if (parquet_indexes.empty()) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Something bad happened, raise an issue and try the query with `input_format_parquet_bloom_filter_push_down=false`"); + } + + auto parquet_column_index = parquet_indexes[0]; + + const auto * parquet_column_descriptor = parquet_rg_metadata->schema()->Column(parquet_column_index); + + bool column_has_bloom_filter = parquet_rg_metadata->ColumnChunk(parquet_column_index)->bloom_filter_offset().has_value(); + if (!column_has_bloom_filter) + { + return nullptr; + } + + return parquet_column_descriptor; +} + +} + +ParquetBloomFilterCondition::ParquetBloomFilterCondition(const std::vector & condition_, const Block & header_) + : condition(condition_), header(header_) +{ +} + +bool ParquetBloomFilterCondition::mayBeTrueOnRowGroup(const ColumnIndexToBF & column_index_to_column_bf) const +{ + using Function = ConditionElement::Function; + std::vector rpn_stack; + + for (const auto & element : condition) + { + if (element.function == Function::FUNCTION_IN + || element.function == Function::FUNCTION_NOT_IN) + { + bool maybe_true = true; + for (auto column_index = 0u; column_index < element.hashes_per_column.size(); column_index++) + { + // in case bloom filter is not present for this row group + // https://github.com/ClickHouse/ClickHouse/pull/62966#discussion_r1722361237 + if (!column_index_to_column_bf.contains(element.key_columns[column_index])) + { + rpn_stack.emplace_back(true, true); + continue; + } + + bool column_maybe_contains = maybeTrueOnBloomFilter( + element.hashes_per_column[column_index], + column_index_to_column_bf.at(element.key_columns[column_index])); + + if (!column_maybe_contains) + { + maybe_true = false; + break; + } + } + + rpn_stack.emplace_back(maybe_true, true); + if (element.function == Function::FUNCTION_NOT_IN) + rpn_stack.back() = !rpn_stack.back(); + } + else if (element.function == Function::FUNCTION_NOT) + { + rpn_stack.back() = !rpn_stack.back(); + } + else if (element.function == Function::FUNCTION_OR) + { + auto arg1 = rpn_stack.back(); + rpn_stack.pop_back(); + auto arg2 = rpn_stack.back(); + rpn_stack.back() = arg1 | arg2; + } + else if (element.function == Function::FUNCTION_AND) + { + auto arg1 = rpn_stack.back(); + rpn_stack.pop_back(); + auto arg2 = rpn_stack.back(); + rpn_stack.back() = arg1 & arg2; + } + else if (element.function == Function::ALWAYS_TRUE) + { + rpn_stack.emplace_back(true, false); + } + else if (element.function == Function::ALWAYS_FALSE) + { + rpn_stack.emplace_back(false, true); + } + else + { + rpn_stack.emplace_back(true, true); + } + } + + if (rpn_stack.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected stack size in KeyCondition::mayBeTrueOnRowGroup"); + + return rpn_stack[0].can_be_true; +} + +std::unordered_set ParquetBloomFilterCondition::getFilteringColumnKeys() const +{ + std::unordered_set column_keys; + + for (const auto & element : condition) + { + for (const auto index : element.key_columns) + { + column_keys.insert(index); + } + } + + return column_keys; +} + +/* + * `KeyCondition::rpn` is overly complex for bloom filters, some operations are not even supported. Not only that, but to avoid hashing each time + * we loop over a rpn element, we need to store hashes instead of where predicate values. To address this, we loop over `KeyCondition::rpn` + * and build a simplified RPN that holds hashes instead of values. + * + * `KeyCondition::RPNElement::FUNCTION_IN_RANGE` becomes: + * `FUNCTION_IN` + * `FUNCTION_UNKNOWN` when range limits are different + * `KeyCondition::RPNElement::FUNCTION_IN_SET` becomes + * `FUNCTION_IN` + * + * Complex types and structs are not supported. + * There are two sources of data types being analyzed, and they need to be compatible: DB::Field type and parquet type. + * This is determined by the `isColumnSupported` method. + * + * Some interesting examples: + * 1. file(..., 'str_column UInt64') where str_column = 50; Field.type == UInt64. Parquet type string. Not supported. + * 2. file(...) where str_column = 50; Field.type == String (conversion already taken care by `KeyCondition`). Parquet type string. + * 3. file(...) where uint32_column = toIPv4(5). Field.type == IPv4. Incompatible column types, resolved by `KeyCondition` itself. + * 4. file(...) where toIPv4(uint32_column) = toIPv4(5). Field.type == IPv4. We know it is safe to hash it using an int32 API. + * */ +std::vector keyConditionRPNToParquetBloomFilterCondition( + const std::vector & rpn, + const std::vector & clickhouse_column_index_to_parquet_index, + const std::unique_ptr & parquet_rg_metadata) +{ + std::vector condition_elements; + + using RPNElement = KeyCondition::RPNElement; + using Function = ParquetBloomFilterCondition::ConditionElement::Function; + + for (const auto & rpn_element : rpn) + { + // this would be a problem for `where negate(x) = -58`. + // It would perform a bf search on `-58`, and possibly miss row groups containing this data. + if (!rpn_element.monotonic_functions_chain.empty()) + { + condition_elements.emplace_back(Function::FUNCTION_UNKNOWN); + continue; + } + + ParquetBloomFilterCondition::ConditionElement::HashesForColumns hashes; + + if (rpn_element.function == RPNElement::FUNCTION_IN_RANGE + || rpn_element.function == RPNElement::FUNCTION_NOT_IN_RANGE) + { + // Only FUNCTION_EQUALS is supported and for that extremes need to be the same + if (rpn_element.range.left != rpn_element.range.right) + { + condition_elements.emplace_back(Function::FUNCTION_UNKNOWN); + continue; + } + + const auto * parquet_column_descriptor = + getColumnDescriptorIfBloomFilterIsPresent(parquet_rg_metadata, clickhouse_column_index_to_parquet_index, rpn_element.key_column); + + if (!parquet_column_descriptor) + { + condition_elements.emplace_back(Function::FUNCTION_UNKNOWN); + continue; + } + + auto hashed_value = tryHash(rpn_element.range.left, parquet_column_descriptor); + + if (!hashed_value) + { + condition_elements.emplace_back(Function::FUNCTION_UNKNOWN); + continue; + } + + std::vector hashes_for_column; + hashes_for_column.emplace_back(*hashed_value); + + hashes.emplace_back(std::move(hashes_for_column)); + + auto function = rpn_element.function == RPNElement::FUNCTION_IN_RANGE + ? ParquetBloomFilterCondition::ConditionElement::Function::FUNCTION_IN + : ParquetBloomFilterCondition::ConditionElement::Function::FUNCTION_NOT_IN; + + std::vector key_columns; + key_columns.emplace_back(rpn_element.key_column); + + condition_elements.emplace_back(function, std::move(hashes), std::move(key_columns)); + } + else if (rpn_element.function == RPNElement::FUNCTION_IN_SET + || rpn_element.function == RPNElement::FUNCTION_NOT_IN_SET) + { + const auto & set_index = rpn_element.set_index; + const auto & ordered_set = set_index->getOrderedSet(); + const auto & indexes_mapping = set_index->getIndexesMapping(); + bool found_empty_column = false; + + std::vector key_columns; + + for (auto i = 0u; i < ordered_set.size(); i++) + { + const auto & set_column = ordered_set[i]; + + const auto * parquet_column_descriptor = getColumnDescriptorIfBloomFilterIsPresent( + parquet_rg_metadata, + clickhouse_column_index_to_parquet_index, + indexes_mapping[i].key_index); + + if (!parquet_column_descriptor) + { + continue; + } + + auto column = set_column; + + if (column->empty()) + { + found_empty_column = true; + break; + } + + if (const auto & nullable_column = checkAndGetColumn(set_column.get())) + { + column = nullable_column->getNestedColumnPtr(); + } + + auto hashes_for_column_opt = hash(column.get(), parquet_column_descriptor); + + if (!hashes_for_column_opt) + { + continue; + } + + auto & hashes_for_column = *hashes_for_column_opt; + + if (hashes_for_column.empty()) + { + continue; + } + + hashes.emplace_back(hashes_for_column); + + key_columns.push_back(indexes_mapping[i].key_index); + } + + if (found_empty_column) + { + condition_elements.emplace_back(Function::ALWAYS_FALSE); + continue; + } + + if (hashes.empty()) + { + condition_elements.emplace_back(Function::FUNCTION_UNKNOWN); + continue; + } + + auto function = RPNElement::FUNCTION_IN_SET == rpn_element.function ? Function::FUNCTION_IN : Function::FUNCTION_NOT_IN; + + condition_elements.emplace_back(function, hashes, key_columns); + } + else if (rpn_element.function == RPNElement::FUNCTION_NOT) + { + condition_elements.emplace_back(Function::FUNCTION_NOT); + } + else if (rpn_element.function == RPNElement::FUNCTION_OR) + { + condition_elements.emplace_back(Function::FUNCTION_OR); + } + else if (rpn_element.function == RPNElement::FUNCTION_AND) + { + condition_elements.emplace_back(Function::FUNCTION_AND); + } + else + { + condition_elements.emplace_back(Function::ALWAYS_TRUE); + } + } + + return condition_elements; +} + +} + +#endif diff --git a/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.h b/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.h new file mode 100644 index 000000000000..6de6030b23cb --- /dev/null +++ b/src/Processors/Formats/Impl/Parquet/ParquetBloomFilterCondition.h @@ -0,0 +1,73 @@ +#pragma once + +#include + +#if USE_PARQUET + +#include +#include +#include + +namespace parquet +{ +class BloomFilter; +} + +namespace DB +{ + +class ParquetBloomFilterCondition +{ +public: + + struct ConditionElement + { + enum Function + { + /// Atoms of a Boolean expression. + FUNCTION_IN, + FUNCTION_NOT_IN, + /// Can take any value. + FUNCTION_UNKNOWN, + /// Operators of the logical expression. + FUNCTION_NOT, + FUNCTION_AND, + FUNCTION_OR, + /// Constants + ALWAYS_FALSE, + ALWAYS_TRUE, + }; + + using ColumnPtr = IColumn::Ptr; + using HashesForColumns = std::vector>; + using KeyColumns = std::vector; + + Function function; + // each entry represents a list of hashes per column + // suppose there are three columns with 2 rows each + // hashes_per_column.size() == 3 and hashes_per_column[0].size() == 2 + HashesForColumns hashes_per_column; + KeyColumns key_columns; + }; + + using RPNElement = KeyCondition::RPNElement; + using ColumnIndexToBF = std::unordered_map>; + + explicit ParquetBloomFilterCondition(const std::vector & condition_, const Block & header_); + + bool mayBeTrueOnRowGroup(const ColumnIndexToBF & column_index_to_column_bf) const; + std::unordered_set getFilteringColumnKeys() const; + +private: + std::vector condition; + Block header; +}; + +std::vector keyConditionRPNToParquetBloomFilterCondition( + const std::vector & rpn, + const std::vector & clickhouse_column_index_to_parquet_index, + const std::unique_ptr & parquet_rg_metadata); + +} + +#endif diff --git a/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.cpp b/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.cpp index b8e4db8700cc..b471989076b4 100644 --- a/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.cpp +++ b/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.cpp @@ -296,6 +296,31 @@ void ParquetPlainValuesReader::readBatch( ); } +template +void ParquetBitPlainReader::readBatch( + MutableColumnPtr & col_ptr, LazyNullMap & null_map, UInt32 num_values) +{ + auto cursor = col_ptr->size(); + auto * column_data = getResizedPrimitiveData(*assert_cast(col_ptr.get()), cursor + num_values); + + def_level_reader->visitNullableValues( + cursor, + num_values, + max_def_level, + null_map, + /* individual_visitor */ [&](size_t nest_cursor) + { + uint8_t byte; + bit_reader->GetValue(1, &byte); + column_data[nest_cursor] = byte; + }, + /* repeated_visitor */ [&](size_t nest_cursor, UInt32 count) + { + bit_reader->GetBatch(1, &column_data[nest_cursor], count); + } + ); +} + template <> void ParquetPlainValuesReader, ParquetReaderTypes::TimestampInt96>::readBatch( @@ -561,6 +586,9 @@ template class ParquetPlainValuesReader>; template class ParquetPlainValuesReader>; template class ParquetPlainValuesReader>; template class ParquetPlainValuesReader; +template class ParquetPlainValuesReader; + +template class ParquetBitPlainReader; template class ParquetFixedLenPlainReader>; template class ParquetFixedLenPlainReader>; @@ -569,6 +597,7 @@ template class ParquetRleLCReader; template class ParquetRleLCReader; template class ParquetRleLCReader; +template class ParquetRleDictReader; template class ParquetRleDictReader; template class ParquetRleDictReader; template class ParquetRleDictReader; diff --git a/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.h b/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.h index fbccb612b3c0..db55f7e2d6ab 100644 --- a/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.h +++ b/src/Processors/Formats/Impl/Parquet/ParquetDataValuesReader.h @@ -172,6 +172,27 @@ class ParquetPlainValuesReader : public ParquetDataValuesReader ParquetDataBuffer plain_data_buffer; }; +template +class ParquetBitPlainReader : public ParquetDataValuesReader +{ +public: + ParquetBitPlainReader( + Int32 max_def_level_, + std::unique_ptr def_level_reader_, + std::unique_ptr bit_reader_) + : max_def_level(max_def_level_) + , def_level_reader(std::move(def_level_reader_)) + , bit_reader(std::move(bit_reader_)) + {} + + void readBatch(MutableColumnPtr & col_ptr, LazyNullMap & null_map, UInt32 num_values) override; + +private: + Int32 max_def_level; + std::unique_ptr def_level_reader; + std::unique_ptr bit_reader; +}; + /** * The data and definition level encoding are same as ParquetPlainValuesReader. * But the element size is const and bigger than primitive data type. diff --git a/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.cpp b/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.cpp index 9e1cae9bb657..c3ced4a47b0f 100644 --- a/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.cpp +++ b/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.cpp @@ -370,6 +370,7 @@ template void ParquetLeafColReader::readPage() { // refer to: ColumnReaderImplBase::ReadNewPage in column_reader.cc + // this is where decompression happens auto cur_page = parquet_page_reader->NextPage(); switch (cur_page->type()) { @@ -409,46 +410,13 @@ void ParquetLeafColReader::readPage() } template -void ParquetLeafColReader::readPageV1(const parquet::DataPageV1 & page) +void ParquetLeafColReader::initDataReader( + parquet::Encoding::type enconding_type, + const uint8_t * buffer, + std::size_t max_size, + std::unique_ptr && def_level_reader) { - static parquet::LevelDecoder repetition_level_decoder; - - cur_page_values = page.num_values(); - - // refer to: VectorizedColumnReader::readPageV1 in Spark and LevelDecoder::SetData in column_reader.cc - if (page.definition_level_encoding() != parquet::Encoding::RLE && col_descriptor.max_definition_level() != 0) - { - throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unsupported encoding: {}", page.definition_level_encoding()); - } - const auto * buffer = page.data(); - auto max_size = page.size(); - - if (col_descriptor.max_repetition_level() > 0) - { - auto rep_levels_bytes = repetition_level_decoder.SetData( - page.repetition_level_encoding(), col_descriptor.max_repetition_level(), 0, buffer, max_size); - buffer += rep_levels_bytes; - max_size -= rep_levels_bytes; - } - - assert(col_descriptor.max_definition_level() >= 0); - std::unique_ptr def_level_reader; - if (col_descriptor.max_definition_level() > 0) - { - auto bit_width = arrow::bit_util::Log2(col_descriptor.max_definition_level() + 1); - auto num_bytes = ::arrow::util::SafeLoadAs(buffer); - auto bit_reader = std::make_unique(buffer + 4, num_bytes); - num_bytes += 4; - buffer += num_bytes; - max_size -= num_bytes; - def_level_reader = std::make_unique(std::move(bit_reader), bit_width); - } - else - { - def_level_reader = std::make_unique(page.num_values()); - } - - switch (page.encoding()) + switch (enconding_type) { case parquet::Encoding::PLAIN: { @@ -458,16 +426,29 @@ void ParquetLeafColReader::readPageV1(const parquet::DataPageV1 & page) degradeDictionary(); } - ParquetDataBuffer parquet_buffer = [&]() + if (col_descriptor.physical_type() == parquet::Type::BOOLEAN) { - if constexpr (!std::is_same_v, TColumn>) - return ParquetDataBuffer(buffer, max_size); - - auto scale = assert_cast(*base_data_type).getScale(); - return ParquetDataBuffer(buffer, max_size, scale); - }(); - data_values_reader = createPlainReader( - col_descriptor, std::move(def_level_reader), std::move(parquet_buffer)); + if constexpr (std::is_same_v) + { + auto bit_reader = std::make_unique(buffer, max_size); + data_values_reader = std::make_unique>(col_descriptor.max_definition_level(), + std::move(def_level_reader), + std::move(bit_reader)); + } + } + else + { + ParquetDataBuffer parquet_buffer = [&]() + { + if constexpr (!std::is_same_v, TColumn>) + return ParquetDataBuffer(buffer, max_size); + + auto scale = assert_cast(*base_data_type).getScale(); + return ParquetDataBuffer(buffer, max_size, scale); + }(); + data_values_reader = createPlainReader( + col_descriptor, std::move(def_level_reader), std::move(parquet_buffer)); + } break; } case parquet::Encoding::RLE_DICTIONARY: @@ -489,17 +470,144 @@ void ParquetLeafColReader::readPageV1(const parquet::DataPageV1 & page) case parquet::Encoding::DELTA_BINARY_PACKED: case parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: case parquet::Encoding::DELTA_BYTE_ARRAY: - throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unsupported encoding: {}", page.encoding()); + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unsupported encoding: {}", enconding_type); default: - throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unknown encoding type: {}", page.encoding()); + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unknown encoding type: {}", enconding_type); } } template -void ParquetLeafColReader::readPageV2(const parquet::DataPageV2 & /*page*/) +void ParquetLeafColReader::readPageV1(const parquet::DataPageV1 & page) { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "read page V2 is not implemented yet"); + cur_page_values = page.num_values(); + + // refer to: VectorizedColumnReader::readPageV1 in Spark and LevelDecoder::SetData in column_reader.cc + if (page.definition_level_encoding() != parquet::Encoding::RLE && col_descriptor.max_definition_level() != 0) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Unsupported encoding: {}", page.definition_level_encoding()); + } + + const auto * buffer = page.data(); + auto max_size = static_cast(page.size()); + + if (col_descriptor.max_repetition_level() > 0) + { + if (max_size < sizeof(int32_t)) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Not enough bytes in parquet page buffer, corrupt?"); + } + + auto num_bytes = ::arrow::util::SafeLoadAs(buffer); + + if (num_bytes < 0) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Number of bytes for dl is negative, corrupt?"); + } + + if (num_bytes + 4u > max_size) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Not enough bytes in parquet page buffer, corrupt?"); + } + + // not constructing level reader because we are not using it atm + num_bytes += 4; + buffer += num_bytes; + max_size -= num_bytes; + } + + assert(col_descriptor.max_definition_level() >= 0); + std::unique_ptr def_level_reader; + if (col_descriptor.max_definition_level() > 0) + { + auto bit_width = arrow::bit_util::Log2(col_descriptor.max_definition_level() + 1); + + if (max_size < sizeof(int32_t)) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Not enough bytes in parquet page buffer, corrupt?"); + } + + auto num_bytes = ::arrow::util::SafeLoadAs(buffer); + + if (num_bytes < 0) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Number of bytes for dl is negative, corrupt?"); + } + + if (num_bytes + 4u > max_size) + { + throw Exception(ErrorCodes::PARQUET_EXCEPTION, "Not enough bytes in parquet page buffer, corrupt?"); + } + + auto bit_reader = std::make_unique(buffer + 4, num_bytes); + num_bytes += 4; + buffer += num_bytes; + max_size -= num_bytes; + def_level_reader = std::make_unique(std::move(bit_reader), bit_width); + } + else + { + def_level_reader = std::make_unique(page.num_values()); + } + + initDataReader(page.encoding(), buffer, max_size, std::move(def_level_reader)); +} + +/* + * As far as I understand, the difference between page v1 and page v2 lies primarily on the below: + * 1. repetition and definition levels are not compressed; + * 2. size of repetition and definition levels is present in the header; + * 3. the encoding is always RLE + * + * Therefore, this method leverages the existing `parquet::LevelDecoder::SetDataV2` method to build the repetition level decoder. + * The data buffer is "offset-ed" by rl bytes length and then dl decoder is built using RLE decoder. Since dl bytes length was present in the header, + * there is no need to read it and apply an offset like in page v1. + * */ +template +void ParquetLeafColReader::readPageV2(const parquet::DataPageV2 & page) +{ + cur_page_values = page.num_values(); + + const auto * buffer = page.data(); + + if (page.repetition_levels_byte_length() < 0 || page.definition_levels_byte_length() < 0) + { + throw Exception( + ErrorCodes::PARQUET_EXCEPTION, "Either RL or DL is negative, this should not happen. Most likely corrupt file or parsing issue"); + } + + const int64_t total_levels_length = + static_cast(page.repetition_levels_byte_length()) + + page.definition_levels_byte_length(); + + if (total_levels_length > page.size()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "Data page too small for levels (corrupt header?)"); + } + + // ARROW-17453: Even if max_rep_level_ is 0, there may still be + // repetition level bytes written and/or reported in the header by + // some writers (e.g. Athena) + buffer += page.repetition_levels_byte_length(); + + assert(col_descriptor.max_definition_level() >= 0); + std::unique_ptr def_level_reader; + if (col_descriptor.max_definition_level() > 0) + { + auto bit_width = arrow::bit_util::Log2(col_descriptor.max_definition_level() + 1); + auto num_bytes = page.definition_levels_byte_length(); + auto bit_reader = std::make_unique(buffer, num_bytes); + def_level_reader = std::make_unique(std::move(bit_reader), bit_width); + } + else + { + def_level_reader = std::make_unique(page.num_values()); + } + + buffer += page.definition_levels_byte_length(); + + initDataReader(page.encoding(), buffer, page.size() - total_levels_length, std::move(def_level_reader)); } template @@ -518,6 +626,12 @@ std::unique_ptr ParquetLeafColReader::createDi }); return res; } + + if (col_descriptor.physical_type() == parquet::Type::type::BOOLEAN) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Dictionary encoding for booleans is not supported"); + } + return std::make_unique>( col_descriptor.max_definition_level(), std::move(def_level_reader), @@ -526,6 +640,7 @@ std::unique_ptr ParquetLeafColReader::createDi } +template class ParquetLeafColReader; template class ParquetLeafColReader; template class ParquetLeafColReader; template class ParquetLeafColReader; diff --git a/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.h b/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.h index c5b14132f176..e1eb7702def1 100644 --- a/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.h +++ b/src/Processors/Formats/Impl/Parquet/ParquetLeafColReader.h @@ -54,6 +54,10 @@ class ParquetLeafColReader : public ParquetColumnReader void readPage(); void readPageV1(const parquet::DataPageV1 & page); void readPageV2(const parquet::DataPageV2 & page); + void initDataReader(parquet::Encoding::type enconding_type, + const uint8_t * buffer, + std::size_t max_size, + std::unique_ptr && def_level_reader); std::unique_ptr createDictReader( std::unique_ptr def_level_reader, std::unique_ptr rle_data_reader); diff --git a/src/Processors/Formats/Impl/Parquet/ParquetRecordReader.cpp b/src/Processors/Formats/Impl/Parquet/ParquetRecordReader.cpp index 18a0db7484ec..74107df4e26b 100644 --- a/src/Processors/Formats/Impl/Parquet/ParquetRecordReader.cpp +++ b/src/Processors/Formats/Impl/Parquet/ParquetRecordReader.cpp @@ -265,7 +265,7 @@ std::unique_ptr ColReaderFactory::makeReader() switch (col_descriptor.physical_type()) { case parquet::Type::BOOLEAN: - break; + return makeLeafReader(); case parquet::Type::INT32: return fromInt32(); case parquet::Type::INT64: diff --git a/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp b/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp index ce859b38b3c4..2e2ff216a72d 100644 --- a/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp +++ b/src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp @@ -232,6 +232,7 @@ void preparePrimitiveColumn(ColumnPtr column, DataTypePtr type, const std::strin auto & state = states.emplace_back(); state.primitive_column = column; state.compression = options.compression; + state.compression_level = options.compression_level; state.column_chunk.__isset.meta_data = true; state.column_chunk.meta_data.__set_path_in_schema({name}); diff --git a/src/Processors/Formats/Impl/Parquet/Write.cpp b/src/Processors/Formats/Impl/Parquet/Write.cpp index b1e231d7749a..d0f68148ff19 100644 --- a/src/Processors/Formats/Impl/Parquet/Write.cpp +++ b/src/Processors/Formats/Impl/Parquet/Write.cpp @@ -390,7 +390,7 @@ struct ConverterDecimal }; /// Returns either `source` or `scratch`. -PODArray & compress(PODArray & source, PODArray & scratch, CompressionMethod method) +PODArray & compress(PODArray & source, PODArray & scratch, CompressionMethod method, int level) { /// We could use wrapWriteBufferWithCompressionMethod() for everything, but I worry about the /// overhead of creating a bunch of WriteBuffers on each page (thousands of values). @@ -447,7 +447,7 @@ PODArray & compress(PODArray & source, PODArray & scratch, Com auto compressed_buf = wrapWriteBufferWithCompressionMethod( std::move(dest_buf), method, - /*level*/ 3, + level, /*zstd_window_log*/ 0, source.size(), /*existing_memory*/ source.data()); @@ -575,7 +575,7 @@ void writeColumnImpl( throw Exception(ErrorCodes::CANNOT_COMPRESS, "Uncompressed page is too big: {}", encoded.size()); size_t uncompressed_size = encoded.size(); - auto & compressed = compress(encoded, compressed_maybe, s.compression); + auto & compressed = compress(encoded, compressed_maybe, s.compression, s.compression_level); if (compressed.size() > INT32_MAX) throw Exception(ErrorCodes::CANNOT_COMPRESS, "Compressed page is too big: {}", compressed.size()); @@ -626,7 +626,7 @@ void writeColumnImpl( encoded.resize(static_cast(dict_size)); dict_encoder->WriteDict(reinterpret_cast(encoded.data())); - auto & compressed = compress(encoded, compressed_maybe, s.compression); + auto & compressed = compress(encoded, compressed_maybe, s.compression, s.compression_level); if (compressed.size() > INT32_MAX) throw Exception(ErrorCodes::CANNOT_COMPRESS, "Compressed dictionary page is too big: {}", compressed.size()); diff --git a/src/Processors/Formats/Impl/Parquet/Write.h b/src/Processors/Formats/Impl/Parquet/Write.h index f162984fd5ed..1a0a60a5b5b6 100644 --- a/src/Processors/Formats/Impl/Parquet/Write.h +++ b/src/Processors/Formats/Impl/Parquet/Write.h @@ -19,6 +19,7 @@ struct WriteOptions bool output_fixed_string_as_fixed_byte_array = true; CompressionMethod compression = CompressionMethod::Lz4; + int compression_level = 3; size_t data_page_size = 1024 * 1024; size_t write_batch_size = 1024; @@ -43,6 +44,7 @@ struct ColumnChunkWriteState ColumnPtr primitive_column; CompressionMethod compression; // must match what's inside column_chunk + int compression_level = 3; Int64 datetime64_multiplier = 1; // for converting e.g. seconds to milliseconds bool is_bool = false; // bool vs UInt8 have the same column type but are encoded differently diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index 4231ef3c7ab5..bfab82148d32 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include "ArrowBufferedStreams.h" @@ -25,6 +27,7 @@ #include #include #include +#include #include namespace CurrentMetrics @@ -264,6 +267,50 @@ static Field decodePlainParquetValueSlow(const std::string & data, parquet::Type return field; } +static ParquetBloomFilterCondition::ColumnIndexToBF buildColumnIndexToBF( + parquet::BloomFilterReader & bf_reader, + int row_group, + const std::vector & clickhouse_column_index_to_parquet_index, + const std::unordered_set & filtering_columns +) +{ + auto rg_bf = bf_reader.RowGroup(row_group); + + if (!rg_bf) + { + return {}; + } + + ParquetBloomFilterCondition::ColumnIndexToBF index_to_column_bf; + + for (const auto & [clickhouse_index, parquet_indexes] : clickhouse_column_index_to_parquet_index) + { + if (!filtering_columns.contains(clickhouse_index)) + { + continue; + } + + // Complex / nested types contain more than one index. We don't support those. + if (parquet_indexes.size() > 1) + { + continue; + } + + auto parquet_index = parquet_indexes[0]; + + auto bf = rg_bf->GetColumnBloomFilter(parquet_index); + + if (!bf) + { + continue; + } + + index_to_column_bf[clickhouse_index] = std::move(bf); + } + + return index_to_column_bf; +} + /// Range of values for each column, based on statistics in the Parquet metadata. /// This is lower/upper bounds, not necessarily exact min and max, e.g. the min/max can be just /// missing in the metadata. @@ -471,9 +518,27 @@ void ParquetBlockInputFormat::initializeIfNeeded() ArrowFieldIndexUtil field_util( format_settings.parquet.case_insensitive_column_matching, format_settings.parquet.allow_missing_columns); - column_indices = field_util.findRequiredIndices(getPort().getHeader(), *schema); + + auto index_mapping = field_util.findRequiredIndices(getPort().getHeader(), *schema, *metadata); + + for (const auto & [clickhouse_header_index, parquet_indexes] : index_mapping) + { + for (auto parquet_index : parquet_indexes) + { + column_indices.push_back(parquet_index); + } + } int num_row_groups = metadata->num_row_groups(); + + if (num_row_groups == 0) + { + return; + } + + const auto bf_reader_properties = parquet::default_reader_properties(); + std::unique_ptr bf_reader; + row_group_batches.reserve(num_row_groups); auto adaptive_chunk_size = [&](int row_group_idx) -> size_t @@ -494,11 +559,38 @@ void ParquetBlockInputFormat::initializeIfNeeded() return std::min(std::max(preferred_num_rows, MIN_ROW_NUM), static_cast(format_settings.parquet.max_block_size)); }; + std::unique_ptr parquet_bloom_filter_condition; + + std::unordered_set filtering_columns; + + if (format_settings.parquet.bloom_filter_push_down && key_condition) + { + bf_reader = parquet::BloomFilterReader::Make(arrow_file, metadata, bf_reader_properties, nullptr); + + const auto parquet_conditions = keyConditionRPNToParquetBloomFilterCondition( + key_condition->getRPN(), + index_mapping, + metadata->RowGroup(0)); + parquet_bloom_filter_condition = std::make_unique(parquet_conditions, getPort().getHeader()); + + filtering_columns = parquet_bloom_filter_condition->getFilteringColumnKeys(); + } + for (int row_group = 0; row_group < num_row_groups; ++row_group) { if (skip_row_groups.contains(row_group)) continue; + if (parquet_bloom_filter_condition) + { + const auto column_index_to_bf = buildColumnIndexToBF(*bf_reader, row_group, index_mapping, filtering_columns); + + if (!parquet_bloom_filter_condition->mayBeTrueOnRowGroup(column_index_to_bf)) + { + continue; + } + } + if (format_settings.parquet.filter_push_down && key_condition && !key_condition ->checkInHyperrectangle( diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 645f16c6abe5..96d47b2ffb96 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -95,6 +95,7 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Blo case C::GZIP: options.compression = CompressionMethod::Gzip; break; case C::BROTLI: options.compression = CompressionMethod::Brotli; break; } + options.compression_level = static_cast(format_settings.parquet.output_compression_level); options.output_string_as_string = format_settings.parquet.output_string_as_string; options.output_fixed_string_as_fixed_byte_array = format_settings.parquet.output_fixed_string_as_fixed_byte_array; options.data_page_size = format_settings.parquet.data_page_size; @@ -322,7 +323,13 @@ void ParquetBlockOutputFormat::writeUsingArrow(std::vector chunks) parquet::WriterProperties::Builder builder; builder.version(getParquetVersion(format_settings)); - builder.compression(getParquetCompression(format_settings.parquet.output_compression_method)); + auto compression_codec = getParquetCompression(format_settings.parquet.output_compression_method); + builder.compression(compression_codec); + + if (arrow::util::Codec::SupportsCompressionLevel(compression_codec)) + { + builder.compression_level(static_cast(format_settings.parquet.output_compression_level)); + } // write page index is disable at default. if (format_settings.parquet.write_page_index) builder.enable_write_page_index(); diff --git a/src/Processors/QueryPlan/ConvertingActions.cpp b/src/Processors/QueryPlan/ConvertingActions.cpp new file mode 100644 index 000000000000..b9703ef59cd5 --- /dev/null +++ b/src/Processors/QueryPlan/ConvertingActions.cpp @@ -0,0 +1,32 @@ +#include +#include +#include + +namespace DB +{ + +void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects) +{ + if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header)) + return; + + auto mode = has_missing_objects ? ActionsDAG::MatchColumnsMode::Position : ActionsDAG::MatchColumnsMode::Name; + + auto get_converting_dag = [mode](const Block & block_, const Block & header_) + { + /// Convert header structure to expected. + /// Also we ignore constants from result and replace it with constants from header. + /// It is needed for functions like `now64()` or `randConstant()` because their values may be different. + return ActionsDAG::makeConvertingActions( + block_.getColumnsWithTypeAndName(), + header_.getColumnsWithTypeAndName(), + mode, + true); + }; + + auto convert_actions_dag = get_converting_dag(plan.getCurrentDataStream().header, header); + auto converting = std::make_unique(plan.getCurrentDataStream(), std::move(convert_actions_dag)); + plan.addStep(std::move(converting)); +} + +} diff --git a/src/Processors/QueryPlan/ConvertingActions.h b/src/Processors/QueryPlan/ConvertingActions.h new file mode 100644 index 000000000000..6bdf9b8af9a6 --- /dev/null +++ b/src/Processors/QueryPlan/ConvertingActions.h @@ -0,0 +1,9 @@ +#pragma once + +namespace DB +{ +class QueryPlan; +class Block; + +void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects); +} diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index d8624a1c99b7..eb699858bdf5 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -2,43 +2,13 @@ #include #include -#include #include #include -#include +#include namespace DB { -namespace -{ - -void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects) -{ - if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header)) - return; - - auto mode = has_missing_objects ? ActionsDAG::MatchColumnsMode::Position : ActionsDAG::MatchColumnsMode::Name; - - auto get_converting_dag = [mode](const Block & block_, const Block & header_) - { - /// Convert header structure to expected. - /// Also we ignore constants from result and replace it with constants from header. - /// It is needed for functions like `now64()` or `randConstant()` because their values may be different. - return ActionsDAG::makeConvertingActions( - block_.getColumnsWithTypeAndName(), - header_.getColumnsWithTypeAndName(), - mode, - true); - }; - - auto convert_actions_dag = get_converting_dag(plan.getCurrentDataStream().header, header); - auto converting = std::make_unique(plan.getCurrentDataStream(), std::move(convert_actions_dag)); - plan.addStep(std::move(converting)); -} - -} - std::unique_ptr createLocalPlan( const ASTPtr & query_ast, const Block & header, diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index 5df7d7b4e82d..2eac18960669 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -899,6 +900,18 @@ AggregationInputOrder buildInputOrderInfo(AggregatingStep & aggregating, QueryPl return {}; } +static bool readingFromParallelReplicas(const QueryPlan::Node * node) +{ + IQueryPlanStep * step = node->step.get(); + while (!node->children.empty()) + { + step = node->children.front()->step.get(); + node = node->children.front(); + } + + return typeid_cast(step); +} + void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) { if (node.children.size() != 1) @@ -924,6 +937,16 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) std::vector infos; infos.reserve(node.children.size()); + for (const auto * child : union_node->children) + { + /// in case of parallel replicas + /// avoid applying read-in-order optimization for local replica + /// since it will lead to different parallel replicas modes + /// between local and remote nodes + if (readingFromParallelReplicas(child)) + return; + } + for (auto * child : union_node->children) { infos.push_back(buildInputOrderInfo(*sorting, *child, steps_to_update)); diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp index b31ee7ea53cf..35ac2c220ffb 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp @@ -757,7 +757,7 @@ std::optional optimizeUseAggregateProjections(QueryPlan::Node & node, Qu projection_reading = reader.readFromParts( /* parts = */ {}, - /* alter_conversions = */ {}, + reading->getMutationsSnapshot()->cloneEmpty(), best_candidate->dag.getRequiredColumnsNames(), proj_snapshot, projection_query_info, diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp index b15f913fc195..654046cda899 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp @@ -199,7 +199,7 @@ std::optional optimizeUseNormalProjections(Stack & stack, QueryPlan::Nod auto projection_reading = reader.readFromParts( /*parts=*/ {}, - /*alter_conversions=*/ {}, + reading->getMutationsSnapshot()->cloneEmpty(), required_columns, proj_snapshot, query_info_copy, diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp index 7414d479cc93..998b606ec57a 100644 --- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp +++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp @@ -41,12 +41,19 @@ bool canUseProjectionForReadingStep(ReadFromMergeTree * reading) if (reading->readsInOrder()) return false; + const auto & query_settings = reading->getContext()->getSettingsRef(); + // Currently projection don't support deduplication when moving parts between shards. - if (reading->getContext()->getSettingsRef().allow_experimental_query_deduplication) + if (query_settings.allow_experimental_query_deduplication) return false; // Currently projection don't support settings which implicitly modify aggregate functions. - if (reading->getContext()->getSettingsRef().aggregate_functions_null_for_empty) + if (query_settings.aggregate_functions_null_for_empty) + return false; + + /// Don't use projections if have mutations to apply + /// because we need to apply them on original data. + if (query_settings.apply_mutations_on_fly && reading->getMutationsSnapshot()->hasDataMutations()) return false; return true; @@ -215,20 +222,15 @@ bool analyzeProjectionCandidate( { MergeTreeData::DataPartsVector projection_parts; MergeTreeData::DataPartsVector normal_parts; - std::vector alter_conversions; + for (const auto & part_with_ranges : parts_with_ranges) { const auto & created_projections = part_with_ranges.data_part->getProjectionParts(); auto it = created_projections.find(candidate.projection->name); if (it != created_projections.end() && !it->second->is_broken) - { projection_parts.push_back(it->second); - } else - { normal_parts.push_back(part_with_ranges.data_part); - alter_conversions.push_back(part_with_ranges.alter_conversions); - } } if (projection_parts.empty()) @@ -241,6 +243,7 @@ bool analyzeProjectionCandidate( auto projection_result_ptr = reader.estimateNumMarksToRead( std::move(projection_parts), + reading.getMutationsSnapshot()->cloneEmpty(), required_column_names, candidate.projection->metadata, projection_query_info, @@ -254,7 +257,7 @@ bool analyzeProjectionCandidate( if (!normal_parts.empty()) { /// TODO: We can reuse existing analysis_result by filtering out projection parts - auto normal_result_ptr = reading.selectRangesToRead(std::move(normal_parts), std::move(alter_conversions)); + auto normal_result_ptr = reading.selectRangesToRead(std::move(normal_parts)); if (normal_result_ptr->selected_marks != 0) { diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp new file mode 100644 index 000000000000..050044edd3ae --- /dev/null +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -0,0 +1,97 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +std::pair, bool> createLocalPlanForParallelReplicas( + const ASTPtr & query_ast, + const Block & header, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + ParallelReplicasReadingCoordinatorPtr coordinator, + QueryPlanStepPtr analyzed_read_from_merge_tree, + size_t replica_number) +{ + checkStackSize(); + + auto query_plan = std::make_unique(); + auto new_context = Context::createCopy(context); + + /// Do not push down limit to local plan, as it will break `rows_before_limit_at_least` counter. + if (processed_stage == QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit) + processed_stage = QueryProcessingStage::WithMergeableStateAfterAggregation; + + /// Do not apply AST optimizations, because query + /// is already optimized and some optimizations + /// can be applied only for non-distributed tables + /// and we can produce query, inconsistent with remote plans. + auto select_query_options = SelectQueryOptions(processed_stage).ignoreASTOptimizations(); + + /// For Analyzer, identifier in GROUP BY/ORDER BY/LIMIT BY lists has been resolved to + /// ConstantNode in QueryTree if it is an alias of a constant, so we should not replace + /// ConstantNode with ProjectionNode again(https://github.com/ClickHouse/ClickHouse/issues/62289). + new_context->setSetting("enable_positional_arguments", Field(false)); + new_context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); + auto interpreter = InterpreterSelectQueryAnalyzer(query_ast, new_context, select_query_options); + query_plan = std::make_unique(std::move(interpreter).extractQueryPlan()); + + QueryPlan::Node * node = query_plan->getRootNode(); + ReadFromMergeTree * reading = nullptr; + while (node) + { + reading = typeid_cast(node->step.get()); + if (reading) + break; + + if (!node->children.empty()) + node = node->children.at(0); + else + node = nullptr; + } + + if (!reading) + /// it can happened if merge tree table is empty, - it'll be replaced with ReadFromPreparedSource + return {std::move(query_plan), false}; + + ReadFromMergeTree::AnalysisResultPtr analyzed_result_ptr; + if (analyzed_read_from_merge_tree.get()) + { + auto * analyzed_merge_tree = typeid_cast(analyzed_read_from_merge_tree.get()); + if (analyzed_merge_tree) + analyzed_result_ptr = analyzed_merge_tree->getAnalyzedResult(); + } + + MergeTreeAllRangesCallback all_ranges_cb = [coordinator](InitialAllRangesAnnouncement announcement) + { coordinator->handleInitialAllRangesAnnouncement(std::move(announcement)); }; + + MergeTreeReadTaskCallback read_task_cb = [coordinator](ParallelReadRequest req) -> std::optional + { return coordinator->handleRequest(std::move(req)); }; + + auto read_from_merge_tree_parallel_replicas = reading->createLocalParallelReplicasReadingStep( + analyzed_result_ptr, std::move(all_ranges_cb), std::move(read_task_cb), replica_number); + node->step = std::move(read_from_merge_tree_parallel_replicas); + + addConvertingActions(*query_plan, header, /*has_missing_objects=*/false); + + return {std::move(query_plan), true}; +} + +} diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h new file mode 100644 index 000000000000..ab0bbeaeeff1 --- /dev/null +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +std::pair, bool> createLocalPlanForParallelReplicas( + const ASTPtr & query_ast, + const Block & header, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + ParallelReplicasReadingCoordinatorPtr coordinator, + QueryPlanStepPtr read_from_merge_tree, + size_t replica_number); +} diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index 935af9148273..aaa1f53b5eeb 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -222,7 +222,6 @@ class RangesInDataPartsBuilder { ranges_in_data_parts.emplace_back( initial_ranges_in_data_parts[part_index].data_part, - initial_ranges_in_data_parts[part_index].alter_conversions, initial_ranges_in_data_parts[part_index].part_index_in_query, MarkRanges{mark_range}); part_index_to_initial_ranges_in_data_parts_index[it->second] = part_index; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 348019d7d107..142f4be119d9 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1,6 +1,8 @@ #include +#include #include +#include #include #include #include @@ -8,6 +10,8 @@ #include #include #include +#include +#include #include #include #include @@ -16,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -24,10 +29,11 @@ #include #include #include -#include #include -#include +#include +#include #include +#include #include #include #include @@ -41,11 +47,6 @@ #include #include #include -#include -#include -#include -#include -#include #include #include @@ -266,7 +267,7 @@ void ReadFromMergeTree::AnalysisResult::checkLimits(const Settings & settings, c ReadFromMergeTree::ReadFromMergeTree( MergeTreeData::DataPartsVector parts_, - std::vector alter_conversions_, + MergeTreeData::MutationsSnapshotPtr mutations_, Names all_column_names_, const MergeTreeData & data_, const SelectQueryInfo & query_info_, @@ -277,13 +278,16 @@ ReadFromMergeTree::ReadFromMergeTree( std::shared_ptr max_block_numbers_to_read_, LoggerPtr log_, AnalysisResultPtr analyzed_result_ptr_, - bool enable_parallel_reading) + bool enable_parallel_reading_, + std::optional all_ranges_callback_, + std::optional read_task_callback_, + std::optional number_of_current_replica_) : SourceStepWithFilter(DataStream{.header = MergeTreeSelectProcessor::transformHeader( storage_snapshot_->getSampleBlockForColumns(all_column_names_), query_info_.prewhere_info)}, all_column_names_, query_info_, storage_snapshot_, context_) , reader_settings(getMergeTreeReaderSettings(context_, query_info_)) , prepared_parts(std::move(parts_)) - , alter_conversions_for_parts(std::move(alter_conversions_)) + , mutations_snapshot(std::move(mutations_)) , all_column_names(std::move(all_column_names_)) , data(data_) , actions_settings(ExpressionActionsSettings::fromContext(context_)) @@ -295,13 +299,21 @@ ReadFromMergeTree::ReadFromMergeTree( , max_block_numbers_to_read(std::move(max_block_numbers_to_read_)) , log(std::move(log_)) , analyzed_result_ptr(analyzed_result_ptr_) - , is_parallel_reading_from_replicas(enable_parallel_reading) + , is_parallel_reading_from_replicas(enable_parallel_reading_) , enable_remove_parts_from_snapshot_optimization(query_info_.merge_tree_enable_remove_parts_from_snapshot_optimization) + , number_of_current_replica(number_of_current_replica_) { if (is_parallel_reading_from_replicas) { - all_ranges_callback = context->getMergeTreeAllRangesCallback(); - read_task_callback = context->getMergeTreeReadTaskCallback(); + if (all_ranges_callback_.has_value()) + all_ranges_callback = all_ranges_callback_.value(); + else + all_ranges_callback = context->getMergeTreeAllRangesCallback(); + + if (read_task_callback_.has_value()) + read_task_callback = read_task_callback_.value(); + else + read_task_callback = context->getMergeTreeReadTaskCallback(); } const auto & settings = context->getSettingsRef(); @@ -335,20 +347,38 @@ ReadFromMergeTree::ReadFromMergeTree( enable_vertical_final); } +std::unique_ptr ReadFromMergeTree::createLocalParallelReplicasReadingStep( + AnalysisResultPtr analyzed_result_ptr_, + MergeTreeAllRangesCallback all_ranges_callback_, + MergeTreeReadTaskCallback read_task_callback_, + size_t replica_number) +{ + const bool enable_parallel_reading = true; + return std::make_unique( + prepared_parts, + mutations_snapshot, + all_column_names, + data, + getQueryInfo(), + getStorageSnapshot(), + getContext(), + block_size.max_block_size_rows, + requested_num_streams, + max_block_numbers_to_read, + log, + std::move(analyzed_result_ptr_), + enable_parallel_reading, + all_ranges_callback_, + read_task_callback_, + replica_number); +} -Pipe ReadFromMergeTree::readFromPoolParallelReplicas( - RangesInDataParts parts_with_range, - Names required_columns, - PoolSettings pool_settings) +Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_with_range, Names required_columns, PoolSettings pool_settings) { const auto & client_info = context->getClientInfo(); - auto extension = ParallelReadingExtension - { - .all_callback = all_ranges_callback.value(), - .callback = read_task_callback.value(), - .number_of_current_replica = client_info.number_of_current_replica, - }; + auto extension = ParallelReadingExtension{ + all_ranges_callback.value(), read_task_callback.value(), number_of_current_replica.value_or(client_info.number_of_current_replica), context->getClusterForParallelReplicas()->getShardsInfo().at(0).getAllNodeCount()}; /// We have a special logic for local replica. It has to read less data, because in some cases it should /// merge states of aggregate functions or do some other important stuff other than reading from Disk. @@ -364,6 +394,7 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas( auto pool = std::make_shared( std::move(extension), std::move(parts_with_range), + mutations_snapshot, shared_virtual_fields, storage_snapshot, prewhere_info, @@ -445,6 +476,7 @@ Pipe ReadFromMergeTree::readFromPool( { pool = std::make_shared( std::move(parts_with_range), + mutations_snapshot, shared_virtual_fields, storage_snapshot, prewhere_info, @@ -458,6 +490,7 @@ Pipe ReadFromMergeTree::readFromPool( { pool = std::make_shared( std::move(parts_with_range), + mutations_snapshot, shared_virtual_fields, storage_snapshot, prewhere_info, @@ -514,12 +547,10 @@ Pipe ReadFromMergeTree::readInOrder( if (is_parallel_reading_from_replicas) { const auto & client_info = context->getClientInfo(); - ParallelReadingExtension extension - { - .all_callback = all_ranges_callback.value(), - .callback = read_task_callback.value(), - .number_of_current_replica = client_info.number_of_current_replica, - }; + ParallelReadingExtension extension{ + all_ranges_callback.value(), + read_task_callback.value(), + number_of_current_replica.value_or(client_info.number_of_current_replica), context->getClusterForParallelReplicas()->getShardsInfo().at(0).getAllNodeCount()}; const auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; if (auto result = pool_settings.min_marks_for_concurrent_read * multiplier; canConvertTo(result)) @@ -538,6 +569,7 @@ Pipe ReadFromMergeTree::readInOrder( std::move(extension), mode, parts_with_ranges, + mutations_snapshot, shared_virtual_fields, storage_snapshot, prewhere_info, @@ -553,6 +585,7 @@ Pipe ReadFromMergeTree::readInOrder( has_limit_below_one_block, read_type, parts_with_ranges, + mutations_snapshot, shared_virtual_fields, storage_snapshot, prewhere_info, @@ -563,11 +596,12 @@ Pipe ReadFromMergeTree::readInOrder( context); } - /// Actually it means that parallel reading from replicas enabled - /// and we have to collaborate with initiator. - /// In this case we won't set approximate rows, because it will be accounted multiple times. - const auto in_order_limit = query_info.input_order_info ? query_info.input_order_info->limit : 0; - const bool set_total_rows_approx = !is_parallel_reading_from_replicas; + /// If parallel replicas enabled, set total rows in progress here only on initiator with local plan + /// Otherwise rows will counted multiple times + const UInt64 in_order_limit = query_info.input_order_info ? query_info.input_order_info->limit : 0; + const bool parallel_replicas_local_plan_for_initiator = is_parallel_reading_from_replicas + && context->getSettingsRef().parallel_replicas_local_plan && context->canUseParallelReplicasOnInitiator(); + const bool set_total_rows_approx = !is_parallel_reading_from_replicas || parallel_replicas_local_plan_for_initiator; Pipes pipes; for (size_t i = 0; i < parts_with_ranges.size(); ++i) @@ -1022,7 +1056,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( } ranges_to_get_from_part = split_ranges(ranges_to_get_from_part, input_order_info->direction); - new_parts.emplace_back(part.data_part, part.alter_conversions, part.part_index_in_query, std::move(ranges_to_get_from_part)); + new_parts.emplace_back(part.data_part, part.part_index_in_query, std::move(ranges_to_get_from_part)); } splitted_parts_and_ranges.emplace_back(std::move(new_parts)); @@ -1249,7 +1283,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( RangesInDataParts new_parts; for (auto part_it = parts_to_merge_ranges[range_index]; part_it != parts_to_merge_ranges[range_index + 1]; ++part_it) - new_parts.emplace_back(part_it->data_part, part_it->alter_conversions, part_it->part_index_in_query, part_it->ranges); + new_parts.emplace_back(part_it->data_part, part_it->part_index_in_query, part_it->ranges); if (new_parts.empty()) continue; @@ -1362,15 +1396,14 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead(bool find_exact_ranges) const { - return selectRangesToRead(prepared_parts, alter_conversions_for_parts, find_exact_ranges); + return selectRangesToRead(prepared_parts, find_exact_ranges); } -ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( - MergeTreeData::DataPartsVector parts, std::vector alter_conversions, bool find_exact_ranges) const +ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead(MergeTreeData::DataPartsVector parts, bool find_exact_ranges) const { return selectRangesToRead( std::move(parts), - std::move(alter_conversions), + mutations_snapshot, storage_snapshot->metadata, query_info, context, @@ -1388,9 +1421,11 @@ static void buildIndexes( const ActionsDAG * filter_actions_dag, const MergeTreeData & data, const MergeTreeData::DataPartsVector & parts, + const MergeTreeData::MutationsSnapshotPtr & mutations_snapshot, const ContextPtr & context, const SelectQueryInfo & query_info, - const StorageMetadataPtr & metadata_snapshot) + const StorageMetadataPtr & metadata_snapshot, + const LoggerPtr & log) { indexes.reset(); @@ -1400,11 +1435,8 @@ static void buildIndexes( const auto & settings = context->getSettingsRef(); - indexes.emplace(ReadFromMergeTree::Indexes{{ - filter_actions_dag, - context, - primary_key_column_names, - primary_key.expression}, {}, {}, {}, {}, false, {}}); + indexes.emplace( + ReadFromMergeTree::Indexes{KeyCondition{filter_actions_dag, context, primary_key_column_names, primary_key.expression}}); if (metadata_snapshot->hasPartitionKey()) { @@ -1416,19 +1448,21 @@ static void buildIndexes( indexes->partition_pruner.emplace(metadata_snapshot, filter_actions_dag, context, false /* strict */); } - indexes->part_values - = MergeTreeDataSelectExecutor::filterPartsByVirtualColumns(metadata_snapshot, data, parts, filter_actions_dag, context); + indexes->part_values = MergeTreeDataSelectExecutor::filterPartsByVirtualColumns(metadata_snapshot, data, parts, filter_actions_dag, context); MergeTreeDataSelectExecutor::buildKeyConditionFromPartOffset(indexes->part_offset_condition, filter_actions_dag, context); indexes->use_skip_indexes = settings.use_skip_indexes; - bool final = query_info.isFinal(); - - if (final && !settings.use_skip_indexes_if_final) + if (query_info.isFinal() && !settings.use_skip_indexes_if_final) indexes->use_skip_indexes = false; if (!indexes->use_skip_indexes) return; + const auto & all_indexes = metadata_snapshot->getSecondaryIndices(); + + if (all_indexes.empty()) + return; + std::unordered_set ignored_index_names; if (settings.ignore_data_skipping_indices.changed) @@ -1453,47 +1487,68 @@ static void buildIndexes( throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "Cannot parse ignore_data_skipping_indices ('{}')", indices); } + auto all_updated_columns = mutations_snapshot->getAllUpdatedColumns(); + UsefulSkipIndexes skip_indexes; using Key = std::pair; std::map merged; - for (const auto & index : metadata_snapshot->getSecondaryIndices()) + for (const auto & index : all_indexes) { - if (!ignored_index_names.contains(index.name)) + if (ignored_index_names.contains(index.name)) + continue; + + auto index_helper = MergeTreeIndexFactory::instance().get(index); + + if (!all_updated_columns.empty()) { - auto index_helper = MergeTreeIndexFactory::instance().get(index); - if (index_helper->isMergeable()) + auto required_columns = index_helper->getColumnsRequiredForIndexCalc(); + auto it = std::ranges::find_if(required_columns, [&](const auto & column_name) { - auto [it, inserted] = merged.emplace(Key{index_helper->index.type, index_helper->getGranularity()}, skip_indexes.merged_indices.size()); - if (inserted) - { - skip_indexes.merged_indices.emplace_back(); - skip_indexes.merged_indices.back().condition = index_helper->createIndexMergedCondition(query_info, metadata_snapshot); - } + return all_updated_columns.contains(column_name); + }); - skip_indexes.merged_indices[it->second].addIndex(index_helper); + if (it != required_columns.end()) + { + LOG_TRACE(log, "Index {} is not used because it depends on column {} which will be updated on fly", index.name, *it); + continue; } - else + } + + if (index_helper->isMergeable()) + { + auto [it, inserted] = merged.emplace(Key{index_helper->index.type, index_helper->getGranularity()}, skip_indexes.merged_indices.size()); + if (inserted) { - MergeTreeIndexConditionPtr condition; - if (index_helper->isVectorSimilarityIndex()) - { + skip_indexes.merged_indices.emplace_back(); + skip_indexes.merged_indices.back().condition = index_helper->createIndexMergedCondition(query_info, metadata_snapshot); + } + + skip_indexes.merged_indices[it->second].addIndex(index_helper); + continue; + } + + MergeTreeIndexConditionPtr condition; + + if (index_helper->isVectorSimilarityIndex()) + { #if USE_USEARCH - if (const auto * vector_similarity_index = typeid_cast(index_helper.get())) - condition = vector_similarity_index->createIndexCondition(query_info, context); + if (const auto * vector_similarity_index = typeid_cast(index_helper.get())) + condition = vector_similarity_index->createIndexCondition(query_info, context); #endif - if (const auto * legacy_vector_similarity_index = typeid_cast(index_helper.get())) - condition = legacy_vector_similarity_index->createIndexCondition(query_info, context); - if (!condition) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown vector search index {}", index_helper->index.name); - } - else - condition = index_helper->createIndexCondition(filter_actions_dag, context); + if (const auto * legacy_vector_similarity_index = typeid_cast(index_helper.get())) + condition = legacy_vector_similarity_index->createIndexCondition(query_info, context); - if (!condition->alwaysUnknownOrTrue()) - skip_indexes.useful_indices.emplace_back(index_helper, condition); - } + if (!condition) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown vector search index {}", index_helper->index.name); + } + else + { + condition = index_helper->createIndexCondition(filter_actions_dag, context); } + + if (!condition->alwaysUnknownOrTrue()) + skip_indexes.useful_indices.emplace_back(index_helper, condition); } // move minmax indices to first positions, so they will be applied first as cheapest ones @@ -1531,15 +1586,17 @@ void ReadFromMergeTree::applyFilters(ActionDAGNodes added_filter_nodes) query_info.filter_actions_dag.get(), data, prepared_parts, + mutations_snapshot, context, query_info, - storage_snapshot->metadata); + storage_snapshot->metadata, + log); } } ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( MergeTreeData::DataPartsVector parts, - std::vector alter_conversions, + MergeTreeData::MutationsSnapshotPtr mutations_snapshot, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info_, ContextPtr context_, @@ -1570,7 +1627,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( const Names & primary_key_column_names = primary_key.column_names; if (!indexes) - buildIndexes(indexes, query_info_.filter_actions_dag.get(), data, parts, context_, query_info_, metadata_snapshot); + buildIndexes(indexes, query_info_.filter_actions_dag.get(), data, parts, mutations_snapshot, context_, query_info_, metadata_snapshot, log); if (indexes->part_values && indexes->part_values->empty()) return std::make_shared(std::move(result)); @@ -1601,10 +1658,9 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( { MergeTreeDataSelectExecutor::filterPartsByPartition( + parts, indexes->partition_pruner, indexes->minmax_idx_condition, - parts, - alter_conversions, indexes->part_values, metadata_snapshot, data, @@ -1633,7 +1689,6 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( auto reader_settings = getMergeTreeReaderSettings(context_, query_info_); result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( std::move(parts), - std::move(alter_conversions), metadata_snapshot, context_, indexes->key_condition, diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index f12da5d10bce..60707c406b31 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -110,7 +110,7 @@ class ReadFromMergeTree final : public SourceStepWithFilter ReadFromMergeTree( MergeTreeData::DataPartsVector parts_, - std::vector alter_conversions_, + MergeTreeData::MutationsSnapshotPtr mutations_snapshot_, Names all_column_names_, const MergeTreeData & data_, const SelectQueryInfo & query_info_, @@ -121,7 +121,16 @@ class ReadFromMergeTree final : public SourceStepWithFilter std::shared_ptr max_block_numbers_to_read_, LoggerPtr log_, AnalysisResultPtr analyzed_result_ptr_, - bool enable_parallel_reading); + bool enable_parallel_reading_, + std::optional all_ranges_callback_ = std::nullopt, + std::optional read_task_callback_ = std::nullopt, + std::optional number_of_current_replica_ = std::nullopt); + + std::unique_ptr createLocalParallelReplicasReadingStep( + AnalysisResultPtr analyzed_result_ptr_, + MergeTreeAllRangesCallback all_ranges_callback_, + MergeTreeReadTaskCallback read_task_callback_, + size_t replica_number); static constexpr auto name = "ReadFromMergeTree"; String getName() const override { return name; } @@ -143,6 +152,11 @@ class ReadFromMergeTree final : public SourceStepWithFilter struct Indexes { + explicit Indexes(KeyCondition key_condition_) + : key_condition(std::move(key_condition_)) + , use_skip_indexes(false) + {} + KeyCondition key_condition; std::optional partition_pruner; std::optional minmax_idx_condition; @@ -154,7 +168,7 @@ class ReadFromMergeTree final : public SourceStepWithFilter static AnalysisResultPtr selectRangesToRead( MergeTreeData::DataPartsVector parts, - std::vector alter_conversions, + MergeTreeData::MutationsSnapshotPtr mutations_snapshot, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, ContextPtr context, @@ -166,8 +180,7 @@ class ReadFromMergeTree final : public SourceStepWithFilter std::optional & indexes, bool find_exact_ranges); - AnalysisResultPtr selectRangesToRead( - MergeTreeData::DataPartsVector parts, std::vector alter_conversions, bool find_exact_ranges = false) const; + AnalysisResultPtr selectRangesToRead(MergeTreeData::DataPartsVector parts, bool find_exact_ranges = false) const; AnalysisResultPtr selectRangesToRead(bool find_exact_ranges = false) const; @@ -188,7 +201,7 @@ class ReadFromMergeTree final : public SourceStepWithFilter void setAnalyzedResult(AnalysisResultPtr analyzed_result_ptr_) { analyzed_result_ptr = std::move(analyzed_result_ptr_); } const MergeTreeData::DataPartsVector & getParts() const { return prepared_parts; } - const std::vector & getAlterConvertionsForParts() const { return alter_conversions_for_parts; } + MergeTreeData::MutationsSnapshotPtr getMutationsSnapshot() const { return mutations_snapshot; } const MergeTreeData & getMergeTreeData() const { return data; } size_t getMaxBlockSize() const { return block_size.max_block_size_rows; } @@ -209,7 +222,7 @@ class ReadFromMergeTree final : public SourceStepWithFilter MergeTreeReaderSettings reader_settings; MergeTreeData::DataPartsVector prepared_parts; - std::vector alter_conversions_for_parts; + MergeTreeData::MutationsSnapshotPtr mutations_snapshot; Names all_column_names; @@ -269,6 +282,7 @@ class ReadFromMergeTree final : public SourceStepWithFilter std::optional read_task_callback; bool enable_vertical_final = false; bool enable_remove_parts_from_snapshot_optimization = true; + std::optional number_of_current_replica; }; } diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index cf11052cd594..3df46eb19878 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -21,8 +21,8 @@ #include #include #include -#include -#include + +#include namespace DB { @@ -362,6 +362,7 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( ASTPtr query_ast_, ClusterPtr cluster_, const StorageID & storage_id_, + ParallelReplicasReadingCoordinatorPtr coordinator_, Block header_, QueryProcessingStage::Enum stage_, ContextMutablePtr context_, @@ -369,11 +370,14 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( Scalars scalars_, Tables external_tables_, LoggerPtr log_, - std::shared_ptr storage_limits_) + std::shared_ptr storage_limits_, + std::vector pools_to_use_, + std::optional exclude_pool_index_) : ISourceStep(DataStream{.header = std::move(header_)}) , cluster(cluster_) , query_ast(query_ast_) , storage_id(storage_id_) + , coordinator(std::move(coordinator_)) , stage(std::move(stage_)) , context(context_) , throttler(throttler_) @@ -381,16 +385,24 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( , external_tables{external_tables_} , storage_limits(std::move(storage_limits_)) , log(log_) + , pools_to_use(std::move(pools_to_use_)) + , exclude_pool_index(exclude_pool_index_) { chassert(cluster->getShardCount() == 1); - std::vector description; - description.push_back(fmt::format("query: {}", formattedAST(query_ast))); + std::vector replicas; + replicas.reserve(pools_to_use.size()); - for (const auto & pool : cluster->getShardsInfo().front().per_replica_pools) - description.push_back(fmt::format("Replica: {}", pool->getHost())); + for (size_t i = 0, l = pools_to_use.size(); i < l; ++i) + { + if (exclude_pool_index.has_value() && i == exclude_pool_index) + continue; - setStepDescription(boost::algorithm::join(description, ", ")); + replicas.push_back(pools_to_use[i]->getAddress()); + } + + auto description = fmt::format("Query: {} Replicas: {}", formattedAST(query_ast), fmt::join(replicas, ", ")); + setStepDescription(std::move(description)); } void ReadFromParallelRemoteReplicasStep::enforceSorting(SortDescription output_sort_description) @@ -406,48 +418,29 @@ void ReadFromParallelRemoteReplicasStep::enforceAggregationInOrder() void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { Pipes pipes; - const Settings & current_settings = context->getSettingsRef(); - auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings); - const auto & shard = cluster->getShardsInfo().at(0); - size_t max_replicas_to_use = current_settings.max_parallel_replicas; - if (max_replicas_to_use > shard.getAllNodeCount()) + std::vector addresses; + addresses.reserve(pools_to_use.size()); + for (size_t i = 0, l = pools_to_use.size(); i < l; ++i) { - LOG_INFO( - getLogger("ReadFromParallelRemoteReplicasStep"), - "The number of replicas requested ({}) is bigger than the real number available in the cluster ({}). " - "Will use the latter number to execute the query.", - current_settings.max_parallel_replicas, - shard.getAllNodeCount()); - max_replicas_to_use = shard.getAllNodeCount(); - } + if (exclude_pool_index.has_value() && i == exclude_pool_index) + continue; - std::vector shuffled_pool; - if (max_replicas_to_use < shard.getAllNodeCount()) - { - shuffled_pool = shard.pool->getShuffledPools(current_settings); - shuffled_pool.resize(max_replicas_to_use); - } - else - { - /// try to preserve replicas order if all replicas in cluster are used for query execution - /// it's important for data locality during query execution - auto priority_func = [](size_t i) { return Priority{static_cast(i)}; }; - shuffled_pool = shard.pool->getShuffledPools(current_settings, priority_func); + addresses.emplace_back(pools_to_use[i]->getAddress()); } + LOG_DEBUG(getLogger("ReadFromParallelRemoteReplicasStep"), "Addresses to use: {}", fmt::join(addresses, ", ")); - coordinator - = std::make_shared(max_replicas_to_use, current_settings.parallel_replicas_mark_segment_size); - - for (size_t i=0; i < max_replicas_to_use; ++i) + for (size_t i = 0, l = pools_to_use.size(); i < l; ++i) { - IConnections::ReplicaInfo replica_info - { + if (exclude_pool_index.has_value() && i == exclude_pool_index) + continue; + + IConnections::ReplicaInfo replica_info{ /// we should use this number specifically because efficiency of data distribution by consistent hash depends on it. .number_of_current_replica = i, }; - addPipeForSingeReplica(pipes, shuffled_pool[i].pool, replica_info); + addPipeForSingeReplica(pipes, pools_to_use[i], replica_info); } auto pipe = Pipe::unitePipes(std::move(pipes)); diff --git a/src/Processors/QueryPlan/ReadFromRemote.h b/src/Processors/QueryPlan/ReadFromRemote.h index 1adb26b2915a..74389c8f9eb8 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.h +++ b/src/Processors/QueryPlan/ReadFromRemote.h @@ -70,6 +70,7 @@ class ReadFromParallelRemoteReplicasStep : public ISourceStep ASTPtr query_ast_, ClusterPtr cluster_, const StorageID & storage_id_, + ParallelReplicasReadingCoordinatorPtr coordinator_, Block header_, QueryProcessingStage::Enum stage_, ContextMutablePtr context_, @@ -77,7 +78,9 @@ class ReadFromParallelRemoteReplicasStep : public ISourceStep Scalars scalars_, Tables external_tables_, LoggerPtr log_, - std::shared_ptr storage_limits_); + std::shared_ptr storage_limits_, + std::vector pools_to_use, + std::optional exclude_pool_index_ = std::nullopt); String getName() const override { return "ReadFromRemoteParallelReplicas"; } @@ -100,6 +103,8 @@ class ReadFromParallelRemoteReplicasStep : public ISourceStep Tables external_tables; std::shared_ptr storage_limits; LoggerPtr log; + std::vector pools_to_use; + std::optional exclude_pool_index; }; } diff --git a/src/QueryPipeline/RemoteInserter.cpp b/src/QueryPipeline/RemoteInserter.cpp index 134c169e35ff..91cc25a6a17d 100644 --- a/src/QueryPipeline/RemoteInserter.cpp +++ b/src/QueryPipeline/RemoteInserter.cpp @@ -51,8 +51,9 @@ RemoteInserter::RemoteInserter( /** Send query and receive "header", that describes table structure. * Header is needed to know, what structure is required for blocks to be passed to 'write' method. */ + /// TODO (vnemkov): figure out should we pass additional roles in this case or not. connection.sendQuery( - timeouts, query, /* query_parameters */ {}, "", QueryProcessingStage::Complete, &settings, &modified_client_info, false, {}); + timeouts, query, /* query_parameters */ {}, "", QueryProcessingStage::Complete, &settings, &modified_client_info, false, /* external_roles */ {}, {}); while (true) { diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 6f8b3931803b..8d7925a34350 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -22,8 +22,12 @@ #include #include #include -#include #include +#include + +#include +#include +#include namespace ProfileEvents @@ -390,7 +394,25 @@ void RemoteQueryExecutor::sendQueryUnlocked(ClientInfo::QueryKind query_kind, As if (!duplicated_part_uuids.empty()) connections->sendIgnoredPartUUIDs(duplicated_part_uuids); - connections->sendQuery(timeouts, query, query_id, stage, modified_client_info, true); + // Collect all roles granted on this node and pass those to the remote node + std::vector local_granted_roles; + if (context->getSettingsRef().push_external_roles_in_interserver_queries && !modified_client_info.initial_user.empty()) + { + auto user = context->getAccessControl().read(modified_client_info.initial_user, false); + boost::container::flat_set granted_roles; + if (user) + { + const auto & access_control = context->getAccessControl(); + for (const auto & e : user->granted_roles.getElements()) + { + auto names = access_control.readNames(e.ids); + granted_roles.insert(names.begin(), names.end()); + } + } + local_granted_roles.insert(local_granted_roles.end(), granted_roles.begin(), granted_roles.end()); + } + + connections->sendQuery(timeouts, query, query_id, stage, modified_client_info, true, local_granted_roles); established = false; sent_query = true; diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index df7b6e7fbd7d..5b981fc7a875 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -34,8 +34,12 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu auto current = pdata->data.get(); if (!current) return -1; + return setCertificateCallback(ssl, current.get(), log); +} - if (current->certs_chain.empty()) +int setCertificateCallback(SSL * ssl, const CertificateReloader::Data * current_data, LoggerPtr log) +{ + if (current_data->certs_chain.empty()) return -1; if (auto err = SSL_clear_chain_certs(ssl); err != 1) @@ -43,12 +47,12 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu LOG_ERROR(log, "Clear certificates {}", Poco::Net::Utility::getLastError()); return -1; } - if (auto err = SSL_use_certificate(ssl, const_cast(current->certs_chain[0].certificate())); err != 1) + if (auto err = SSL_use_certificate(ssl, const_cast(current_data->certs_chain[0].certificate())); err != 1) { LOG_ERROR(log, "Use certificate {}", Poco::Net::Utility::getLastError()); return -1; } - for (auto cert = current->certs_chain.begin() + 1; cert != current->certs_chain.end(); cert++) + for (auto cert = current_data->certs_chain.begin() + 1; cert != current_data->certs_chain.end(); cert++) { if (auto err = SSL_add1_chain_cert(ssl, const_cast(cert->certificate())); err != 1) { @@ -56,7 +60,7 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu return -1; } } - if (auto err = SSL_use_PrivateKey(ssl, const_cast(static_cast(current->key))); err != 1) + if (auto err = SSL_use_PrivateKey(ssl, const_cast(static_cast(current_data->key))); err != 1) { LOG_ERROR(log, "Use private key {}", Poco::Net::Utility::getLastError()); return -1; diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 7472d2f6baa3..28737988fddc 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -104,6 +104,9 @@ class CertificateReloader mutable std::mutex data_mutex; }; +/// A callback for OpenSSL +int setCertificateCallback(SSL * ssl, const CertificateReloader::Data * current_data, LoggerPtr log); + } #endif diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 679f72b85ff3..a0439761882a 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -57,12 +57,17 @@ #include #include +#include + #include "TCPHandler.h" #include #include +#include +#include + using namespace std::literals; using namespace DB; @@ -103,6 +108,7 @@ namespace DB::ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int UNSUPPORTED_METHOD; extern const int USER_EXPIRED; + extern const int NETWORK_ERROR; } namespace @@ -254,8 +260,8 @@ void TCPHandler::runImpl() socket().setSendTimeout(send_timeout); socket().setNoDelay(true); - in = std::make_shared(socket(), read_event); - out = std::make_shared(socket(), write_event); + in = std::make_shared(socket(), read_event); + out = std::make_shared(socket(), write_event); /// Support for PROXY protocol if (parse_proxy_protocol && !receiveProxyHeader()) @@ -280,6 +286,48 @@ void TCPHandler::runImpl() if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_ADDENDUM) receiveAddendum(); + { + /// Server side of chunked protocol negotiation. + /// Server advertises its protocol capabilities (separate for send and receive channels) by sending + /// in its 'Hello' response one of four types - chunked, notchunked, chunked_optional, notchunked_optional. + /// Not optional types are strict meaning that server only supports this type, optional means that + /// server prefer this type but capable to work in opposite. + /// Client selects which type it is going to communicate based on the settings from config or arguments, + /// and sends either "chunked" or "notchunked" protocol request in addendum section of handshake. + /// Client can detect if server's protocol capabilities are not compatible with client's settings (for example + /// server strictly requires chunked protocol but client's settings only allows notchunked protocol) - in such case + /// client should interrupt this connection. However if client continues with incompatible protocol type request, server + /// will send appropriate exception and disconnect client. + + auto is_chunked = [](const String & chunked_srv_str, const String & chunked_cl_str, const String & direction) + { + bool chunked_srv = chunked_srv_str.starts_with("chunked"); + bool optional_srv = chunked_srv_str.ends_with("_optional"); + bool chunked_cl = chunked_cl_str.starts_with("chunked"); + + if (optional_srv) + return chunked_cl; + + if (chunked_cl != chunked_srv) + throw NetException( + ErrorCodes::NETWORK_ERROR, + "Incompatible protocol: {} is {}, client requested {}", + direction, + chunked_srv ? "chunked" : "notchunked", + chunked_cl ? "chunked" : "notchunked"); + + return chunked_srv; + }; + + bool out_chunked = is_chunked(server.config().getString("proto_caps.send", "notchunked"), proto_recv_chunked_cl, "send"); + bool in_chunked = is_chunked(server.config().getString("proto_caps.recv", "notchunked"), proto_send_chunked_cl, "recv"); + + if (out_chunked) + out->enableChunked(); + if (in_chunked) + in->enableChunked(); + } + if (!is_interserver_mode) { /// If session created, then settings in session context has been updated. @@ -321,7 +369,7 @@ void TCPHandler::runImpl() { Stopwatch idle_time; UInt64 timeout_ms = std::min(poll_interval, idle_connection_timeout) * 1000000; - while (tcp_server.isOpen() && !server.isCancelled() && !static_cast(*in).poll(timeout_ms)) + while (tcp_server.isOpen() && !server.isCancelled() && !in->poll(timeout_ms)) { if (idle_time.elapsedSeconds() > idle_connection_timeout) { @@ -796,7 +844,7 @@ bool TCPHandler::readDataNext() /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down. while (true) { - if (static_cast(*in).poll(timeout_us)) + if (in->poll(timeout_us)) { /// If client disconnected. if (in->eof()) @@ -1186,6 +1234,8 @@ void TCPHandler::processTablesStatusRequest() } response.write(*out, client_tcp_protocol_version); + + out->finishChunk(); } void TCPHandler::receiveUnexpectedTablesStatusRequest() @@ -1206,6 +1256,8 @@ void TCPHandler::sendPartUUIDs() writeVarUInt(Protocol::Server::PartUUIDs, *out); writeVectorBinary(uuids, *out); + + out->finishChunk(); out->next(); } } @@ -1214,6 +1266,8 @@ void TCPHandler::sendPartUUIDs() void TCPHandler::sendReadTaskRequestAssumeLocked() { writeVarUInt(Protocol::Server::ReadTaskRequest, *out); + + out->finishChunk(); out->next(); } @@ -1221,7 +1275,9 @@ void TCPHandler::sendReadTaskRequestAssumeLocked() void TCPHandler::sendMergeTreeAllRangesAnnouncementAssumeLocked(InitialAllRangesAnnouncement announcement) { writeVarUInt(Protocol::Server::MergeTreeAllRangesAnnouncement, *out); - announcement.serialize(*out); + announcement.serialize(*out, client_parallel_replicas_protocol_version); + + out->finishChunk(); out->next(); } @@ -1229,7 +1285,9 @@ void TCPHandler::sendMergeTreeAllRangesAnnouncementAssumeLocked(InitialAllRanges void TCPHandler::sendMergeTreeReadTaskRequestAssumeLocked(ParallelReadRequest request) { writeVarUInt(Protocol::Server::MergeTreeReadTaskRequest, *out); - request.serialize(*out); + request.serialize(*out, client_parallel_replicas_protocol_version); + + out->finishChunk(); out->next(); } @@ -1238,6 +1296,8 @@ void TCPHandler::sendProfileInfo(const ProfileInfo & info) { writeVarUInt(Protocol::Server::ProfileInfo, *out); info.write(*out, client_tcp_protocol_version); + + out->finishChunk(); out->next(); } @@ -1253,6 +1313,8 @@ void TCPHandler::sendTotals(const Block & totals) state.block_out->write(totals); state.maybe_compressed_out->next(); + + out->finishChunk(); out->next(); } } @@ -1269,6 +1331,8 @@ void TCPHandler::sendExtremes(const Block & extremes) state.block_out->write(extremes); state.maybe_compressed_out->next(); + + out->finishChunk(); out->next(); } } @@ -1286,6 +1350,8 @@ void TCPHandler::sendProfileEvents() writeStringBinary("", *out); state.profile_events_block_out->write(block); + + out->finishChunk(); out->next(); auto elapsed_milliseconds = stopwatch.elapsedMilliseconds(); @@ -1323,6 +1389,8 @@ void TCPHandler::sendTimezone() LOG_DEBUG(log, "TCPHandler::sendTimezone(): {}", tz); writeVarUInt(Protocol::Server::TimezoneUpdate, *out); writeStringBinary(tz, *out); + + out->finishChunk(); out->next(); } @@ -1583,6 +1651,15 @@ void TCPHandler::receiveAddendum() if (!is_interserver_mode) session->setQuotaClientKey(quota_key); + + if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS) + { + readStringBinary(proto_send_chunked_cl, *in); + readStringBinary(proto_recv_chunked_cl, *in); + } + + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL) + readVarUInt(client_parallel_replicas_protocol_version, *in); } @@ -1610,12 +1687,19 @@ void TCPHandler::sendHello() writeVarUInt(VERSION_MAJOR, *out); writeVarUInt(VERSION_MINOR, *out); writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, *out); + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL) + writeVarUInt(DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION, *out); if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE) writeStringBinary(DateLUT::instance().getTimeZone(), *out); if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) writeStringBinary(server_display_name, *out); if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) writeVarUInt(VERSION_PATCH, *out); + if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS) + { + writeStringBinary(server.config().getString("proto_caps.send", "notchunked"), *out); + writeStringBinary(server.config().getString("proto_caps.recv", "notchunked"), *out); + } if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES) { auto rules = server.context()->getAccessControl().getPasswordComplexityRules(); @@ -1668,6 +1752,7 @@ bool TCPHandler::receivePacket() case Protocol::Client::Ping: writeVarUInt(Protocol::Server::Pong, *out); + out->finishChunk(); out->next(); return false; @@ -1798,6 +1883,13 @@ void TCPHandler::receiveQuery() Settings passed_settings; passed_settings.read(*in, settings_format); + std::string received_extra_roles; + // TODO: check if having `is_interserver_mode` doesn't break interoperability with the CH-client. + if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_INTERSERVER_EXTERNALLY_GRANTED_ROLES) + { + readStringBinary(received_extra_roles, *in); + } + /// Interserver secret. std::string received_hash; if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET) @@ -1857,6 +1949,7 @@ void TCPHandler::receiveQuery() data += state.query; data += state.query_id; data += client_info.initial_user; + data += received_extra_roles; std::string calculated_hash = encodeSHA256(data); assert(calculated_hash.size() == 32); @@ -1877,13 +1970,25 @@ void TCPHandler::receiveQuery() } else { + // In a cluster, query originator may have an access to the external auth provider (like LDAP server), + // that grants specific roles to the user. We want these roles to be granted to the user on other nodes of cluster when + // query is executed. + Strings external_roles; + if (!received_extra_roles.empty()) + { + ReadBufferFromString buffer(received_extra_roles); + + readVectorBinary(external_roles, buffer); + LOG_DEBUG(log, "Parsed extra roles [{}]", fmt::join(external_roles, ", ")); + } + LOG_DEBUG(log, "User (initial, interserver mode): {} (client: {})", client_info.initial_user, getClientAddress(client_info).toString()); /// In case of inter-server mode authorization is done with the /// initial address of the client, not the real address from which /// the query was come, since the real address is the address of /// the initiator server, while we are interested in client's /// address. - session->authenticate(AlwaysAllowCredentials{client_info.initial_user}, client_info.initial_address); + session->authenticate(AlwaysAllowCredentials{client_info.initial_user}, client_info.initial_address, external_roles); } is_interserver_authenticated = true; @@ -2197,7 +2302,7 @@ QueryState::CancellationStatus TCPHandler::getQueryCancellationStatus() after_check_cancelled.restart(); /// During request execution the only packet that can come from the client is stopping the query. - if (static_cast(*in).poll(0)) + if (in->poll(0)) { if (in->eof()) { @@ -2248,19 +2353,33 @@ void TCPHandler::sendData(const Block & block) } writeVarUInt(Protocol::Server::Data, *out); - /// Send external table name (empty name is the main table) - writeStringBinary("", *out); /// For testing hedged requests if (block.rows() > 0 && query_context->getSettingsRef().sleep_in_send_data_ms.totalMilliseconds()) { + /// This strange sequence is needed in case of chunked protocol is enabled, in order for client not to + /// hang on receiving of at least packet type - chunk will not be processed unless either chunk footer + /// or chunk continuation header is received - first 'next' is sending starting chunk containing packet type + /// and second 'next' is sending chunk continuation header. + out->next(); + /// Send external table name (empty name is the main table) + writeStringBinary("", *out); out->next(); std::chrono::milliseconds ms(query_context->getSettingsRef().sleep_in_send_data_ms.totalMilliseconds()); std::this_thread::sleep_for(ms); } + else + { + /// Send external table name (empty name is the main table) + writeStringBinary("", *out); + } state.block_out->write(block); - state.maybe_compressed_out->next(); + + if (state.maybe_compressed_out != out) + state.maybe_compressed_out->next(); + + out->finishChunk(); out->next(); } catch (...) @@ -2296,6 +2415,8 @@ void TCPHandler::sendLogData(const Block & block) writeStringBinary("", *out); state.logs_block_out->write(block); + + out->finishChunk(); out->next(); } @@ -2307,6 +2428,7 @@ void TCPHandler::sendTableColumns(const ColumnsDescription & columns) writeStringBinary("", *out); writeStringBinary(columns.toString(), *out); + out->finishChunk(); out->next(); } @@ -2316,6 +2438,8 @@ void TCPHandler::sendException(const Exception & e, bool with_stack_trace) writeVarUInt(Protocol::Server::Exception, *out); writeException(e, *out, with_stack_trace); + + out->finishChunk(); out->next(); } @@ -2326,6 +2450,8 @@ void TCPHandler::sendEndOfStream() state.io.setAllDataSent(); writeVarUInt(Protocol::Server::EndOfStream, *out); + + out->finishChunk(); out->next(); } @@ -2344,6 +2470,8 @@ void TCPHandler::sendProgress() increment.elapsed_ns = current_elapsed_ns - state.prev_elapsed_ns; state.prev_elapsed_ns = current_elapsed_ns; increment.write(*out, client_tcp_protocol_version); + + out->finishChunk(); out->next(); } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 74afb5a14a59..3b6e0059a30d 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include "Core/Types.h" #include "IServer.h" @@ -186,6 +188,9 @@ class TCPHandler : public Poco::Net::TCPServerConnection UInt64 client_version_minor = 0; UInt64 client_version_patch = 0; UInt32 client_tcp_protocol_version = 0; + UInt32 client_parallel_replicas_protocol_version = 0; + String proto_send_chunked_cl = "notchunked"; + String proto_recv_chunked_cl = "notchunked"; String quota_key; /// Connection settings, which are extracted from a context. @@ -204,8 +209,8 @@ class TCPHandler : public Poco::Net::TCPServerConnection ClientInfo::QueryKind query_kind = ClientInfo::QueryKind::NO_QUERY; /// Streams for reading/writing from/to client connection socket. - std::shared_ptr in; - std::shared_ptr out; + std::shared_ptr in; + std::shared_ptr out; ProfileEvents::Event read_event; ProfileEvents::Event write_event; diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index 1de8ccce2a68..7f368102dfd3 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -273,6 +273,8 @@ ConnectionPoolWithFailoverPtr DistributedAsyncInsertDirectoryQueue::createPool(c address.default_database, address.user, address.password, + address.proto_send_chunked, + address.proto_recv_chunked, address.quota_key, address.cluster, address.cluster_secret, diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index b5e32fc9593b..1a57f780f65d 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -271,6 +271,7 @@ class IStorage : public std::enable_shared_from_this, public TypePromo /// Return true if the trivial count query could be optimized without reading the data at all /// in totalRows() or totalRowsByPartitionPredicate() methods or with optimized reading in read() method. + /// 'storage_snapshot' may be nullptr. virtual bool supportsTrivialCountOptimization(const StorageSnapshotPtr & /*storage_snapshot*/, ContextPtr /*query_context*/) const { return false; diff --git a/src/Storages/MergeTree/AlterConversions.cpp b/src/Storages/MergeTree/AlterConversions.cpp index 31f8f17e2c14..a36611e3d87f 100644 --- a/src/Storages/MergeTree/AlterConversions.cpp +++ b/src/Storages/MergeTree/AlterConversions.cpp @@ -9,9 +9,15 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -bool AlterConversions::supportsMutationCommandType(MutationCommand::Type t) +bool AlterConversions::isSupportedDataMutation(MutationCommand::Type) { - return t == MutationCommand::Type::RENAME_COLUMN; + /// Currently there is no such mutations. See setting 'apply_mutations_on_fly'. + return false; +} + +bool AlterConversions::isSupportedMetadataMutation(MutationCommand::Type type) +{ + return type == MutationCommand::Type::RENAME_COLUMN; } void AlterConversions::addMutationCommand(const MutationCommand & command) diff --git a/src/Storages/MergeTree/AlterConversions.h b/src/Storages/MergeTree/AlterConversions.h index 7bcd4af8b36d..046cc1d2491f 100644 --- a/src/Storages/MergeTree/AlterConversions.h +++ b/src/Storages/MergeTree/AlterConversions.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include namespace DB @@ -11,11 +11,17 @@ namespace DB /// Alter conversions which should be applied on-fly for part. /// Built from of the most recent mutation commands for part. /// Now only ALTER RENAME COLUMN is applied. -class AlterConversions : private boost::noncopyable +class AlterConversions : private WithContext, boost::noncopyable { public: AlterConversions() = default; + AlterConversions(StorageMetadataPtr metadata_snapshot_, ContextPtr context_) + : WithContext(context_) + , metadata_snapshot(std::move(metadata_snapshot_)) + { + } + struct RenamePair { std::string rename_to; @@ -34,11 +40,13 @@ class AlterConversions : private boost::noncopyable /// Get column old name before rename (lookup by key in rename_map) std::string getColumnOldName(const std::string & new_name) const; - static bool supportsMutationCommandType(MutationCommand::Type); + static bool isSupportedDataMutation(MutationCommand::Type type); + static bool isSupportedMetadataMutation(MutationCommand::Type type); private: /// Rename map new_name -> old_name. std::vector rename_map; + StorageMetadataPtr metadata_snapshot; }; using AlterConversionsPtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index caaaf53b9d46..e54d5237ae75 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -800,7 +800,7 @@ void KeyCondition::getAllSpaceFillingCurves() KeyCondition::KeyCondition( const ActionsDAG * filter_dag, ContextPtr context, - const Names & key_column_names, + const Names & key_column_names_, const ExpressionActionsPtr & key_expr_, bool single_point_) : key_expr(key_expr_) @@ -808,7 +808,7 @@ KeyCondition::KeyCondition( , single_point(single_point_) { size_t key_index = 0; - for (const auto & name : key_column_names) + for (const auto & name : key_column_names_) { if (!key_columns.contains(name)) { diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 7d3c2819d8dc..0127a3b7ae63 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -274,6 +274,14 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() if (enabledBlockOffsetColumn(global_ctx)) addGatheringColumn(global_ctx, BlockOffsetColumn::name, BlockOffsetColumn::type); + MergeTreeData::IMutationsSnapshot::Params params + { + .metadata_version = global_ctx->metadata_snapshot->getMetadataVersion(), + .min_part_metadata_version = MergeTreeData::getMinMetadataVersion(global_ctx->future_part->parts), + }; + + auto mutations_snapshot = global_ctx->data->getMutationsSnapshot(params); + SerializationInfo::Settings info_settings = { .ratio_of_defaults_for_sparse = global_ctx->data->getSettings()->ratio_of_defaults_for_sparse_serialization, @@ -281,10 +289,12 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() }; SerializationInfoByName infos(global_ctx->storage_columns, info_settings); + global_ctx->alter_conversions.reserve(global_ctx->future_part->parts.size()); for (const auto & part : global_ctx->future_part->parts) { global_ctx->new_data_part->ttl_infos.update(part->ttl_infos); + if (global_ctx->metadata_snapshot->hasAnyTTL() && !part->checkAllTTLCalculated(global_ctx->metadata_snapshot)) { LOG_INFO(ctx->log, "Some TTL values were not calculated for part {}. Will calculate them forcefully during merge.", part->name); @@ -305,6 +315,8 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() infos.add(part_infos); } + + global_ctx->alter_conversions.push_back(MergeTreeData::getAlterConversionsForPart(part, mutations_snapshot, global_ctx->metadata_snapshot, global_ctx->context)); } const auto & local_part_min_ttl = global_ctx->new_data_part->ttl_infos.part_min_ttl; @@ -657,6 +669,7 @@ Pipe MergeTask::VerticalMergeStage::createPipeForReadingOneColumn(const String & *global_ctx->data, global_ctx->storage_snapshot, global_ctx->future_part->parts[part_num], + global_ctx->alter_conversions[part_num], Names{column_name}, /*mark_ranges=*/ {}, global_ctx->input_rows_filtered, @@ -1107,13 +1120,14 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() global_ctx->horizontal_stage_progress = std::make_unique( ctx->column_sizes ? ctx->column_sizes->keyColumnsWeight() : 1.0); - for (const auto & part : global_ctx->future_part->parts) + for (size_t i = 0; i < global_ctx->future_part->parts.size(); ++i) { Pipe pipe = createMergeTreeSequentialSource( MergeTreeSequentialSourceType::Merge, *global_ctx->data, global_ctx->storage_snapshot, - part, + global_ctx->future_part->parts[i], + global_ctx->alter_conversions[i], global_ctx->merging_columns.getNames(), /*mark_ranges=*/ {}, global_ctx->input_rows_filtered, diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index 2871e8c0f4a7..7e8969145d5d 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -162,6 +162,7 @@ class MergeTask StorageSnapshotPtr storage_snapshot{nullptr}; StorageMetadataPtr metadata_snapshot{nullptr}; FutureMergedMutatedPartPtr future_part{nullptr}; + std::vector alter_conversions; /// This will be either nullptr or new_data_part, so raw pointer is ok. IMergeTreeDataPart * parent_part{nullptr}; ContextPtr context{nullptr}; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 42214891db8c..76215c0f8673 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7168,11 +7168,16 @@ UInt64 MergeTreeData::estimateNumberOfRowsToRead( ContextPtr query_context, const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info) const { const auto & snapshot_data = assert_cast(*storage_snapshot->data); - const auto & parts = snapshot_data.parts; MergeTreeDataSelectExecutor reader(*this); auto result_ptr = reader.estimateNumMarksToRead( - parts, {}, storage_snapshot->metadata, query_info, query_context, query_context->getSettingsRef().max_threads); + snapshot_data.parts, + snapshot_data.mutations_snapshot, + storage_snapshot->metadata->getColumns().getAll().getNames(), + storage_snapshot->metadata, + query_info, + query_context, + query_context->getSettingsRef().max_threads); UInt64 total_rows = result_ptr->selected_rows; if (query_info.trivial_limit > 0 && query_info.trivial_limit < total_rows) @@ -8202,11 +8207,15 @@ bool MergeTreeData::canUsePolymorphicParts(const MergeTreeSettings & settings, S return true; } -AlterConversionsPtr MergeTreeData::getAlterConversionsForPart(MergeTreeDataPartPtr part) const +AlterConversionsPtr MergeTreeData::getAlterConversionsForPart( + const MergeTreeDataPartPtr & part, + const MutationsSnapshotPtr & mutations, + const StorageMetadataPtr & metadata, + const ContextPtr & query_context) { - auto commands = getAlterMutationCommandsForPart(part); + auto commands = mutations->getAlterMutationCommandsForPart(part); + auto result = std::make_shared(metadata, query_context); - auto result = std::make_shared(); for (const auto & command : commands | std::views::reverse) result->addMutationCommand(command); @@ -8498,9 +8507,28 @@ void MergeTreeData::updateObjectColumns(const DataPartPtr & part, const DataPart DB::updateObjectColumns(object_columns, columns, part->getColumns()); } -bool MergeTreeData::supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const +bool MergeTreeData::supportsTrivialCountOptimization(const StorageSnapshotPtr & storage_snapshot, ContextPtr query_context) const { - return !hasLightweightDeletedMask(); + if (hasLightweightDeletedMask()) + return false; + + if (!storage_snapshot) + return !query_context->getSettingsRef().apply_mutations_on_fly; + + const auto & snapshot_data = assert_cast(*storage_snapshot->data); + return !snapshot_data.mutations_snapshot->hasDataMutations(); +} + +Int64 MergeTreeData::getMinMetadataVersion(const DataPartsVector & parts) +{ + Int64 version = -1; + for (const auto & part : parts) + { + Int64 part_version = part->getMetadataVersion(); + if (version == -1 || part_version < version) + version = part_version; + } + return version; } StorageSnapshotPtr MergeTreeData::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const @@ -8514,10 +8542,14 @@ StorageSnapshotPtr MergeTreeData::getStorageSnapshot(const StorageMetadataPtr & object_columns_copy = object_columns; } - snapshot_data->alter_conversions.reserve(snapshot_data->parts.size()); - for (const auto & part : snapshot_data->parts) - snapshot_data->alter_conversions.push_back(getAlterConversionsForPart(part)); + IMutationsSnapshot::Params params + { + .metadata_version = metadata_snapshot->getMetadataVersion(), + .min_part_metadata_version = getMinMetadataVersion(snapshot_data->parts), + .need_data_mutations = query_context->getSettingsRef().apply_mutations_on_fly, + }; + snapshot_data->mutations_snapshot = getMutationsSnapshot(params); return std::make_shared(*this, metadata_snapshot, std::move(object_columns_copy), std::move(snapshot_data)); } @@ -8735,28 +8767,57 @@ void MergeTreeData::verifySortingKey(const KeyDescription & sorting_key) } } -bool updateAlterConversionsMutations(const MutationCommands & commands, std::atomic & alter_conversions_mutations, bool remove) +static void updateMutationsCounters( + Int64 & num_data_mutations_to_apply, + Int64 & num_metadata_mutations_to_apply, + const MutationCommands & commands, + Int64 increment) { + if (num_data_mutations_to_apply < 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "On-fly data mutations counter is negative ({})", num_data_mutations_to_apply); + + if (num_metadata_mutations_to_apply < 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "On-fly metadata mutations counter is negative ({})", num_metadata_mutations_to_apply); + + bool has_data_mutation = false; + bool has_metadata_mutation = false; + for (const auto & command : commands) { - if (AlterConversions::supportsMutationCommandType(command.type)) + if (!has_data_mutation && AlterConversions::isSupportedDataMutation(command.type)) { - if (remove) - { - --alter_conversions_mutations; - if (alter_conversions_mutations < 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "On-fly mutations counter is negative ({})", alter_conversions_mutations); - } - else - { - if (alter_conversions_mutations < 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "On-fly mutations counter is negative ({})", alter_conversions_mutations); - ++alter_conversions_mutations; - } - return true; + num_data_mutations_to_apply += increment; + has_data_mutation = true; + + if (num_data_mutations_to_apply < 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "On-fly data mutations counter is negative ({})", num_data_mutations_to_apply); + } + + if (!has_metadata_mutation && AlterConversions::isSupportedMetadataMutation(command.type)) + { + num_metadata_mutations_to_apply += increment; + has_metadata_mutation = true; + + if (num_metadata_mutations_to_apply < 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "On-fly metadata mutations counter is negative ({})", num_metadata_mutations_to_apply); } } - return false; +} + +void incrementMutationsCounters( + Int64 & num_data_mutations_to_apply, + Int64 & num_metadata_mutations_to_apply, + const MutationCommands & commands) +{ + updateMutationsCounters(num_data_mutations_to_apply, num_metadata_mutations_to_apply, commands, 1); +} + +void decrementMutationsCounters( + Int64 & num_data_mutations_to_apply, + Int64 & num_metadata_mutations_to_apply, + const MutationCommands & commands) +{ + updateMutationsCounters(num_data_mutations_to_apply, num_metadata_mutations_to_apply, commands, -1); } } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 6662df3db84b..dc37d5e7dadf 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -443,14 +443,53 @@ class MergeTreeData : public IStorage, public WithMutableContext bool areAsynchronousInsertsEnabled() const override; - bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override; + bool supportsTrivialCountOptimization(const StorageSnapshotPtr & storage_snapshot, ContextPtr query_context) const override; + + /// A snapshot of pending mutations that weren't applied to some of the parts yet + /// and should be applied on the fly (i.e. when reading from the part). + /// Mutations not supported by AlterConversions (supportsMutationCommandType()) can be omitted. + struct IMutationsSnapshot + { + /// Contains info that doesn't depend on state of mutations. + struct Params + { + Int64 metadata_version = -1; + Int64 min_part_metadata_version = -1; + bool need_data_mutations = false; + }; + + /// Contains info that depends on state of mutations. + struct Info + { + Int64 num_data_mutations = 0; + Int64 num_metadata_mutations = 0; + }; + + Params params; + Info info; + + IMutationsSnapshot() = default; + IMutationsSnapshot(Params params_, Info info_): params(std::move(params_)), info(std::move(info_)) {} + + /// Returns mutation commands that are required to be applied to the `part`. + /// @return list of mutation commands, in *reverse* order (newest to oldest) + virtual MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const = 0; + virtual std::shared_ptr cloneEmpty() const = 0; + virtual NameSet getAllUpdatedColumns() const = 0; + + bool hasDataMutations() const { return params.need_data_mutations && info.num_data_mutations > 0; } + + virtual ~IMutationsSnapshot() = default; + }; + + using MutationsSnapshotPtr = std::shared_ptr; /// Snapshot for MergeTree contains the current set of data parts - /// at the moment of the start of query. + /// and mutations required to be applied at the moment of the start of query. struct SnapshotData : public StorageSnapshot::Data { DataPartsVector parts; - std::vector alter_conversions; + MutationsSnapshotPtr mutations_snapshot; }; StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const override; @@ -929,8 +968,18 @@ class MergeTreeData : public IStorage, public WithMutableContext Disks getDisks() const { return getStoragePolicy()->getDisks(); } + /// Returns a snapshot of mutations that probably will be applied on the fly to parts during reading. + virtual MutationsSnapshotPtr getMutationsSnapshot(const IMutationsSnapshot::Params & params) const = 0; + + /// Returns the minimum version of metadata among parts. + static Int64 getMinMetadataVersion(const DataPartsVector & parts); + /// Return alter conversions for part which must be applied on fly. - AlterConversionsPtr getAlterConversionsForPart(MergeTreeDataPartPtr part) const; + static AlterConversionsPtr getAlterConversionsForPart( + const MergeTreeDataPartPtr & part, + const MutationsSnapshotPtr & mutations, + const StorageMetadataPtr & metadata, + const ContextPtr & query_context); /// Returns destination disk or volume for the TTL rule according to current storage policy. SpacePtr getDestinationForMoveTTL(const TTLDescription & move_ttl) const; @@ -1450,13 +1499,6 @@ class MergeTreeData : public IStorage, public WithMutableContext /// mechanisms for parts locking virtual bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const = 0; - /// Return pending mutations that weren't applied to `part` yet and should be applied on the fly - /// (i.e. when reading from the part). Mutations not supported by AlterConversions - /// (supportsMutationCommandType()) can be omitted. - /// - /// @return list of mutations, in *reverse* order (newest to oldest) - virtual MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const = 0; - struct PartBackupEntries { String part_name; @@ -1731,7 +1773,14 @@ struct CurrentlySubmergingEmergingTagger }; /// Look at MutationCommands if it contains mutations for AlterConversions, update the counter. -/// Return true if the counter had been updated -bool updateAlterConversionsMutations(const MutationCommands & commands, std::atomic & alter_conversions_mutations, bool remove); +void incrementMutationsCounters( + Int64 & num_data_mutations_to_apply, + Int64 & num_metadata_mutations_to_apply, + const MutationCommands & commands); + +void decrementMutationsCounters( + Int64 & num_data_mutations_to_apply, + Int64 & num_metadata_mutations_to_apply, + const MutationCommands & commands); } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index bf12b3966d33..f545e2e054c6 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -133,12 +133,10 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( bool enable_parallel_reading) const { const auto & snapshot_data = assert_cast(*storage_snapshot->data); - const auto & parts = snapshot_data.parts; - const auto & alter_conversions = snapshot_data.alter_conversions; auto step = readFromParts( - parts, - alter_conversions, + snapshot_data.parts, + snapshot_data.mutations_snapshot, column_names_to_return, storage_snapshot, query_info, @@ -271,10 +269,14 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling( * It is also important that the entire universe can be covered using SAMPLE 0.1 OFFSET 0, ... OFFSET 0.9 and similar decimals. */ - auto parallel_replicas_mode = context->getParallelReplicasMode(); + const bool can_use_sampling_key_parallel_replicas = + settings.allow_experimental_parallel_reading_from_replicas > 0 + && settings.max_parallel_replicas > 1 + && settings.parallel_replicas_mode == ParallelReplicasMode::SAMPLING_KEY; + /// Parallel replicas has been requested but there is no way to sample data. /// Select all data from first replica and no data from other replicas. - if (settings.parallel_replicas_count > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY + if (can_use_sampling_key_parallel_replicas && settings.parallel_replicas_count > 1 && !data.supportsSampling() && settings.parallel_replica_offset > 0) { LOG_DEBUG( @@ -285,9 +287,7 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling( return sampling; } - sampling.use_sampling = relative_sample_size > 0 - || (settings.parallel_replicas_count > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY - && data.supportsSampling()); + sampling.use_sampling = relative_sample_size > 0 || (can_use_sampling_key_parallel_replicas && settings.parallel_replicas_count > 1 && data.supportsSampling()); bool no_data = false; /// There is nothing left after sampling. if (sampling.use_sampling) @@ -494,10 +494,9 @@ std::optional> MergeTreeDataSelectExecutor::filterPar } void MergeTreeDataSelectExecutor::filterPartsByPartition( + MergeTreeData::DataPartsVector & parts, const std::optional & partition_pruner, const std::optional & minmax_idx_condition, - MergeTreeData::DataPartsVector & parts, - std::vector & alter_conversions, const std::optional> & part_values, const StorageMetadataPtr & metadata_snapshot, const MergeTreeData & data, @@ -506,8 +505,6 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( LoggerPtr log, ReadFromMergeTree::IndexStats & index_stats) { - chassert(alter_conversions.empty() || parts.size() == alter_conversions.size()); - const Settings & settings = context->getSettingsRef(); DataTypes minmax_columns_types; @@ -531,7 +528,6 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( if (query_context->getSettingsRef().allow_experimental_query_deduplication) selectPartsToReadWithUUIDFilter( parts, - alter_conversions, part_values, data.getPinnedPartUUIDs(), minmax_idx_condition, @@ -544,7 +540,6 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( else selectPartsToRead( parts, - alter_conversions, part_values, minmax_idx_condition, minmax_columns_types, @@ -583,7 +578,6 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( MergeTreeData::DataPartsVector && parts, - std::vector && alter_conversions, StorageMetadataPtr metadata_snapshot, const ContextPtr & context, const KeyCondition & key_condition, @@ -596,8 +590,6 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd bool use_skip_indexes, bool find_exact_ranges) { - chassert(alter_conversions.empty() || parts.size() == alter_conversions.size()); - RangesInDataParts parts_with_ranges; parts_with_ranges.resize(parts.size()); const Settings & settings = context->getSettingsRef(); @@ -656,11 +648,8 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd auto process_part = [&](size_t part_index) { auto & part = parts[part_index]; - auto alter_conversions_for_part = !alter_conversions.empty() - ? alter_conversions[part_index] - : std::make_shared(); - RangesInDataPart ranges(part, alter_conversions_for_part, part_index); + RangesInDataPart ranges(part, part_index); size_t total_marks_count = part->index_granularity.getMarksCountWithoutFinal(); if (metadata_snapshot->hasPrimaryKey() || part_offset_condition) @@ -898,6 +887,7 @@ std::shared_ptr MergeTreeDataSelectExecutor::checkLimits( ReadFromMergeTree::AnalysisResultPtr MergeTreeDataSelectExecutor::estimateNumMarksToRead( MergeTreeData::DataPartsVector parts, + MergeTreeData::MutationsSnapshotPtr mutations_snapshot, const Names & column_names_to_return, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, @@ -910,11 +900,9 @@ ReadFromMergeTree::AnalysisResultPtr MergeTreeDataSelectExecutor::estimateNumMar return std::make_shared(); std::optional indexes; - /// NOTE: We don't need alter_conversions because the returned analysis_result is only used for: - /// 1. estimate the number of rows to read; 2. projection reading, which doesn't have alter_conversions. return ReadFromMergeTree::selectRangesToRead( std::move(parts), - /*alter_conversions=*/{}, + mutations_snapshot, metadata_snapshot, query_info, context, @@ -929,7 +917,7 @@ ReadFromMergeTree::AnalysisResultPtr MergeTreeDataSelectExecutor::estimateNumMar QueryPlanStepPtr MergeTreeDataSelectExecutor::readFromParts( MergeTreeData::DataPartsVector parts, - std::vector alter_conversions, + MergeTreeData::MutationsSnapshotPtr mutations_snapshot, const Names & column_names_to_return, const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, @@ -951,7 +939,7 @@ QueryPlanStepPtr MergeTreeDataSelectExecutor::readFromParts( return std::make_unique( std::move(parts), - std::move(alter_conversions), + std::move(mutations_snapshot), column_names_to_return, data, query_info, @@ -1548,7 +1536,6 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingMergedIndex( void MergeTreeDataSelectExecutor::selectPartsToRead( MergeTreeData::DataPartsVector & parts, - std::vector & alter_conversions, const std::optional> & part_values, const std::optional & minmax_idx_condition, const DataTypes & minmax_columns_types, @@ -1557,14 +1544,11 @@ void MergeTreeDataSelectExecutor::selectPartsToRead( PartFilterCounters & counters) { MergeTreeData::DataPartsVector prev_parts; - std::vector prev_conversions; - std::swap(prev_parts, parts); - std::swap(prev_conversions, alter_conversions); - for (size_t i = 0; i < prev_parts.size(); ++i) + for (const auto & part_or_projection : prev_parts) { - const auto * part = prev_parts[i]->isProjectionPart() ? prev_parts[i]->getParentPart() : prev_parts[i].get(); + const auto * part = part_or_projection->isProjectionPart() ? part_or_projection->getParentPart() : part_or_projection.get(); if (part_values && part_values->find(part->name) == part_values->end()) continue; @@ -1601,15 +1585,12 @@ void MergeTreeDataSelectExecutor::selectPartsToRead( counters.num_parts_after_partition_pruner += 1; counters.num_granules_after_partition_pruner += num_granules; - parts.push_back(prev_parts[i]); - if (!prev_conversions.empty()) - alter_conversions.push_back(prev_conversions[i]); + parts.push_back(part_or_projection); } } void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( MergeTreeData::DataPartsVector & parts, - std::vector & alter_conversions, const std::optional> & part_values, MergeTreeData::PinnedPartUUIDsPtr pinned_part_uuids, const std::optional & minmax_idx_condition, @@ -1622,22 +1603,17 @@ void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( { /// process_parts prepare parts that have to be read for the query, /// returns false if duplicated parts' UUID have been met - auto select_parts = [&] ( - MergeTreeData::DataPartsVector & selected_parts, - std::vector & selected_conversions) -> bool + auto select_parts = [&](MergeTreeData::DataPartsVector & selected_parts) -> bool { auto ignored_part_uuids = query_context->getIgnoredPartUUIDs(); std::unordered_set temp_part_uuids; MergeTreeData::DataPartsVector prev_parts; - std::vector prev_conversions; - std::swap(prev_parts, selected_parts); - std::swap(prev_conversions, selected_conversions); - for (size_t i = 0; i < prev_parts.size(); ++i) + for (const auto & part_or_projection : prev_parts) { - const auto * part = prev_parts[i]->isProjectionPart() ? prev_parts[i]->getParentPart() : prev_parts[i].get(); + const auto * part = part_or_projection->isProjectionPart() ? part_or_projection->getParentPart() : part_or_projection.get(); if (part_values && part_values->find(part->name) == part_values->end()) continue; @@ -1687,9 +1663,7 @@ void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( throw Exception(ErrorCodes::LOGICAL_ERROR, "Found a part with the same UUID on the same replica."); } - selected_parts.push_back(prev_parts[i]); - if (!prev_conversions.empty()) - selected_conversions.push_back(prev_conversions[i]); + selected_parts.push_back(part_or_projection); } if (!temp_part_uuids.empty()) @@ -1708,7 +1682,7 @@ void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( }; /// Process parts that have to be read for a query. - auto needs_retry = !select_parts(parts, alter_conversions); + auto needs_retry = !select_parts(parts); /// If any duplicated part UUIDs met during the first step, try to ignore them in second pass. /// This may happen when `prefer_localhost_replica` is set and "distributed" stage runs in the same process with "remote" stage. @@ -1719,7 +1693,7 @@ void MergeTreeDataSelectExecutor::selectPartsToReadWithUUIDFilter( counters = PartFilterCounters(); /// Second attempt didn't help, throw an exception - if (!select_parts(parts, alter_conversions)) + if (!select_parts(parts)) throw Exception(ErrorCodes::DUPLICATED_PART_UUIDS, "Found duplicate UUIDs while processing query."); } } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 39bff5eacd6a..3668eb0ad90a 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -40,7 +40,7 @@ class MergeTreeDataSelectExecutor /// The same as read, but with specified set of parts. QueryPlanStepPtr readFromParts( MergeTreeData::DataPartsVector parts, - std::vector alter_conversions, + MergeTreeData::MutationsSnapshotPtr mutations_snapshot, const Names & column_names, const StorageSnapshotPtr & storage_snapshot, const SelectQueryInfo & query_info, @@ -56,6 +56,7 @@ class MergeTreeDataSelectExecutor /// This method is used to select best projection for table. ReadFromMergeTree::AnalysisResultPtr estimateNumMarksToRead( MergeTreeData::DataPartsVector parts, + MergeTreeData::MutationsSnapshotPtr mutations_snapshot, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, @@ -120,7 +121,6 @@ class MergeTreeDataSelectExecutor /// as well as `max_block_number_to_read`. static void selectPartsToRead( MergeTreeData::DataPartsVector & parts, - std::vector & alter_conversions, const std::optional> & part_values, const std::optional & minmax_idx_condition, const DataTypes & minmax_columns_types, @@ -131,7 +131,6 @@ class MergeTreeDataSelectExecutor /// Same as previous but also skip parts uuids if any to the query context, or skip parts which uuids marked as excluded. static void selectPartsToReadWithUUIDFilter( MergeTreeData::DataPartsVector & parts, - std::vector & alter_conversions, const std::optional> & part_values, MergeTreeData::PinnedPartUUIDsPtr pinned_part_uuids, const std::optional & minmax_idx_condition, @@ -175,10 +174,9 @@ class MergeTreeDataSelectExecutor /// Filter parts using minmax index and partition key. static void filterPartsByPartition( + MergeTreeData::DataPartsVector & parts, const std::optional & partition_pruner, const std::optional & minmax_idx_condition, - MergeTreeData::DataPartsVector & parts, - std::vector & alter_conversions, const std::optional> & part_values, const StorageMetadataPtr & metadata_snapshot, const MergeTreeData & data, @@ -192,7 +190,6 @@ class MergeTreeDataSelectExecutor /// If 'check_limits = true' it will throw exception if the amount of data exceed the limits from settings. static RangesInDataParts filterPartsByPrimaryKeyAndSkipIndexes( MergeTreeData::DataPartsVector && parts, - std::vector && alter_conversions, StorageMetadataPtr metadata_snapshot, const ContextPtr & context, const KeyCondition & key_condition, diff --git a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp index 4dbccb916202..5970aed497e0 100644 --- a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp +++ b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp @@ -50,7 +50,7 @@ UInt64 MergeTreeMutationEntry::parseFileName(const String & file_name_) MergeTreeMutationEntry::MergeTreeMutationEntry(MutationCommands commands_, DiskPtr disk_, const String & path_prefix_, UInt64 tmp_number, const TransactionID & tid_, const WriteSettings & settings) : create_time(time(nullptr)) - , commands(std::move(commands_)) + , commands(std::make_shared(std::move(commands_))) , disk(std::move(disk_)) , path_prefix(path_prefix_) , file_name("tmp_mutation_" + toString(tmp_number) + ".txt") @@ -63,7 +63,7 @@ MergeTreeMutationEntry::MergeTreeMutationEntry(MutationCommands commands_, DiskP *out << "format version: 1\n" << "create time: " << LocalDateTime(create_time, DateLUT::serverTimezoneInstance()) << "\n"; *out << "commands: "; - commands.writeText(*out, /* with_pure_metadata_commands = */ false); + commands->writeText(*out, /* with_pure_metadata_commands = */ false); *out << "\n"; if (tid.isPrehistoric()) { @@ -116,7 +116,8 @@ void MergeTreeMutationEntry::writeCSN(CSN csn_) } MergeTreeMutationEntry::MergeTreeMutationEntry(DiskPtr disk_, const String & path_prefix_, const String & file_name_) - : disk(std::move(disk_)) + : commands(std::make_shared()) + , disk(std::move(disk_)) , path_prefix(path_prefix_) , file_name(file_name_) , is_temp(false) @@ -133,7 +134,7 @@ MergeTreeMutationEntry::MergeTreeMutationEntry(DiskPtr disk_, const String & pat create_time_dt.hour(), create_time_dt.minute(), create_time_dt.second()); *buf >> "commands: "; - commands.readText(*buf); + commands->readText(*buf); *buf >> "\n"; if (buf->eof()) @@ -177,7 +178,7 @@ std::shared_ptr MergeTreeMutationEntry::backup() const out << "block number: " << block_number << "\n"; out << "commands: "; - commands.writeText(out, /* with_pure_metadata_commands = */ false); + commands->writeText(out, /* with_pure_metadata_commands = */ false); out << "\n"; return std::make_shared(out.str()); diff --git a/src/Storages/MergeTree/MergeTreeMutationEntry.h b/src/Storages/MergeTree/MergeTreeMutationEntry.h index 04297f2852a1..f41ad2a17f85 100644 --- a/src/Storages/MergeTree/MergeTreeMutationEntry.h +++ b/src/Storages/MergeTree/MergeTreeMutationEntry.h @@ -16,7 +16,7 @@ class IBackupEntry; struct MergeTreeMutationEntry { time_t create_time = 0; - MutationCommands commands; + std::shared_ptr commands; DiskPtr disk; String path_prefix; diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp index a9b77fb6c034..09bbf33ba9b3 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp @@ -85,6 +85,7 @@ MergeTreeReadTask::Readers MergeTreePrefetchedReadPool::PrefetchedReaders::get() MergeTreePrefetchedReadPool::MergeTreePrefetchedReadPool( RangesInDataParts && parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, @@ -95,6 +96,7 @@ MergeTreePrefetchedReadPool::MergeTreePrefetchedReadPool( const ContextPtr & context_) : MergeTreeReadPoolBase( std::move(parts_), + std::move(mutations_snapshot_), std::move(shared_virtual_fields_), storage_snapshot_, prewhere_info_, @@ -103,7 +105,6 @@ MergeTreePrefetchedReadPool::MergeTreePrefetchedReadPool( column_names_, settings_, context_) - , WithContext(context_) , prefetch_threadpool(getContext()->getPrefetchThreadpool()) , log(getLogger("MergeTreePrefetchedReadPool(" + (parts_ranges.empty() ? "" : parts_ranges.front().data_part->storage.getStorageID().getNameForLogs()) + ")")) { diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h index a3a572276301..1a7092509378 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h @@ -14,11 +14,12 @@ using MergeTreeReaderPtr = std::unique_ptr; /// A class which is responsible for creating read tasks /// which are later taken by readers via getTask method. /// Does prefetching for the read tasks it creates. -class MergeTreePrefetchedReadPool : public MergeTreeReadPoolBase, private WithContext +class MergeTreePrefetchedReadPool : public MergeTreeReadPoolBase { public: MergeTreePrefetchedReadPool( RangesInDataParts && parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index cc321cd5a4da..97acd917d8ba 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -28,6 +28,7 @@ extern const int LOGICAL_ERROR; MergeTreeReadPool::MergeTreeReadPool( RangesInDataParts && parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, @@ -38,6 +39,7 @@ MergeTreeReadPool::MergeTreeReadPool( const ContextPtr & context_) : MergeTreeReadPoolBase( std::move(parts_), + std::move(mutations_snapshot_), std::move(shared_virtual_fields_), storage_snapshot_, prewhere_info_, diff --git a/src/Storages/MergeTree/MergeTreeReadPool.h b/src/Storages/MergeTree/MergeTreeReadPool.h index 7f0de21e1a46..c51dca315f99 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/src/Storages/MergeTree/MergeTreeReadPool.h @@ -26,6 +26,7 @@ class MergeTreeReadPool : public MergeTreeReadPoolBase MergeTreeReadPool( RangesInDataParts && parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp index 6d2560bc9c79..021b340d7467 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp @@ -4,9 +4,6 @@ #include #include -#include - - namespace DB { @@ -17,6 +14,7 @@ namespace ErrorCodes MergeTreeReadPoolBase::MergeTreeReadPoolBase( RangesInDataParts && parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, @@ -25,7 +23,9 @@ MergeTreeReadPoolBase::MergeTreeReadPoolBase( const Names & column_names_, const PoolSettings & pool_settings_, const ContextPtr & context_) - : parts_ranges(std::move(parts_)) + : WithContext(context_) + , parts_ranges(std::move(parts_)) + , mutations_snapshot(std::move(mutations_snapshot_)) , shared_virtual_fields(std::move(shared_virtual_fields_)) , storage_snapshot(storage_snapshot_) , prewhere_info(prewhere_info_) @@ -119,9 +119,9 @@ void MergeTreeReadPoolBase::fillPerPartInfos(const Settings & settings) } read_task_info.part_index_in_query = part_with_ranges.part_index_in_query; - read_task_info.alter_conversions = part_with_ranges.alter_conversions; + read_task_info.alter_conversions = MergeTreeData::getAlterConversionsForPart(part_with_ranges.data_part, mutations_snapshot, storage_snapshot->metadata, getContext()); - LoadedMergeTreeDataPartInfoForReader part_info(part_with_ranges.data_part, part_with_ranges.alter_conversions); + LoadedMergeTreeDataPartInfoForReader part_info(part_with_ranges.data_part, read_task_info.alter_conversions); read_task_info.task_columns = getReadTaskColumns( part_info, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.h b/src/Storages/MergeTree/MergeTreeReadPoolBase.h index 123f7538ba8d..7f9106d476ef 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolBase.h +++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.h @@ -6,9 +6,11 @@ namespace DB { -class MergeTreeReadPoolBase : public IMergeTreeReadPool +class MergeTreeReadPoolBase : public IMergeTreeReadPool, protected WithContext { public: + using MutationsSnapshotPtr = MergeTreeData::MutationsSnapshotPtr; + struct PoolSettings { size_t threads = 0; @@ -23,6 +25,7 @@ class MergeTreeReadPoolBase : public IMergeTreeReadPool MergeTreeReadPoolBase( RangesInDataParts && parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, @@ -37,6 +40,7 @@ class MergeTreeReadPoolBase : public IMergeTreeReadPool protected: /// Initialized in constructor const RangesInDataParts parts_ranges; + const MutationsSnapshotPtr mutations_snapshot; const VirtualFields shared_virtual_fields; const StorageSnapshotPtr storage_snapshot; const PrewhereInfoPtr prewhere_info; diff --git a/src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp b/src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp index 4c0391ffa571..60f127acdaed 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp @@ -12,6 +12,7 @@ MergeTreeReadPoolInOrder::MergeTreeReadPoolInOrder( bool has_limit_below_one_block_, MergeTreeReadType read_type_, RangesInDataParts parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, @@ -22,6 +23,7 @@ MergeTreeReadPoolInOrder::MergeTreeReadPoolInOrder( const ContextPtr & context_) : MergeTreeReadPoolBase( std::move(parts_), + std::move(mutations_snapshot_), std::move(shared_virtual_fields_), storage_snapshot_, prewhere_info_, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolInOrder.h b/src/Storages/MergeTree/MergeTreeReadPoolInOrder.h index 9fedf396a6bd..a3668acb1707 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolInOrder.h +++ b/src/Storages/MergeTree/MergeTreeReadPoolInOrder.h @@ -11,6 +11,7 @@ class MergeTreeReadPoolInOrder : public MergeTreeReadPoolBase bool has_limit_below_one_block_, MergeTreeReadType read_type_, RangesInDataParts parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp index 33eaf5a49bd9..d138bed00c79 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp @@ -1,18 +1,106 @@ -#include #include +#include +#include + +#include +#include +#include + + +namespace +{ + +size_t chooseSegmentSize( + LoggerPtr log, size_t mark_segment_size, size_t min_marks_per_task, size_t threads, size_t sum_marks, size_t number_of_replicas) +{ + /// Mark segment size determines the granularity of work distribution between replicas. + /// Namely, coordinator will take mark segments of size `mark_segment_size` granules, calculate hash of this segment and assign it to corresponding replica. + /// Small segments are good when we read a small random subset of a table, big - when we do full-scan over a large table. + /// With small segments there is a problem: consider a query like `select max(time) from wikistat`. Average size of `time` per granule is ~5KB. So when we + /// read 128 granules we still read only ~0.5MB of data. With default fs cache segment size of 4MB it means a lot of data will be downloaded and written + /// in cache for no reason. General case will look like this: + /// + /// +---------- useful data + /// v + /// +------+--+------+ + /// |------|++| | + /// |------|++| | + /// +------+--+------+ + /// ^ + /// predownloaded data -----------+ + /// + /// Having large segments solves all the problems in this case. Also bigger segments mean less requests (especially for big tables and full-scans). + /// These three values below chosen mostly intuitively. 128 granules is 1M rows - just a good starting point, 16384 seems to still make sense when reading + /// billions of rows and 1024 - is a reasonable point in between. We limit our choice to only these three options because when we change segment size + /// we essentially change distribution of data between replicas and of course we don't want to use simultaneously tens of different distributions, because + /// it would be a huge waste of cache space. + constexpr std::array borders{128, 1024, 16384}; + + LOG_DEBUG( + log, + "mark_segment_size={}, min_marks_per_task*threads={}, sum_marks/number_of_replicas^2={}", + mark_segment_size, + min_marks_per_task * threads, + sum_marks / number_of_replicas / number_of_replicas); + + /// Here we take max of two numbers: + /// * (min_marks_per_task * threads) = the number of marks we request from the coordinator each time - there is no point to have segments smaller than one unit of work for a replica + /// * (sum_marks / number_of_replicas^2) - we use consistent hashing for work distribution (including work stealing). If we have a really slow replica + /// everything except (1/number_of_replicas) portion of its work will be stolen by other replicas. And it owns (1/number_of_replicas) share of total number of marks. + /// Also important to note here that sum_marks is calculated after PK analysis, it means in particular that different segment sizes might be used for the + /// same table for different queries (it is intentional). + /// + /// Positive `mark_segment_size` means it is a user provided value, we have to preserve it. + if (mark_segment_size == 0) + mark_segment_size = std::max(min_marks_per_task * threads, sum_marks / number_of_replicas / number_of_replicas); + + /// Squeeze the value to the borders. + mark_segment_size = std::clamp(mark_segment_size, borders.front(), borders.back()); + /// After we calculated a hopefully good value for segment_size let's just find the maximal border that is not bigger than the chosen value. + for (auto border : borders | std::views::reverse) + { + if (mark_segment_size >= border) + { + LOG_DEBUG(log, "Chosen segment size: {}", border); + return border; + } + } + + UNREACHABLE(); +} + +size_t getMinMarksPerTask(size_t min_marks_per_task, const std::vector & per_part_infos) +{ + for (const auto & info : per_part_infos) + min_marks_per_task = std::max(min_marks_per_task, info->min_marks_per_task); + + if (min_marks_per_task == 0) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, "Chosen number of marks to read is zero (likely because of weird interference of settings)"); + + return min_marks_per_task; +} +} + +namespace ProfileEvents +{ +extern const Event ParallelReplicasReadMarks; +} namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; +extern const int LOGICAL_ERROR; +extern const int BAD_ARGUMENTS; } MergeTreeReadPoolParallelReplicas::MergeTreeReadPoolParallelReplicas( ParallelReadingExtension extension_, RangesInDataParts && parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, @@ -23,6 +111,7 @@ MergeTreeReadPoolParallelReplicas::MergeTreeReadPoolParallelReplicas( const ContextPtr & context_) : MergeTreeReadPoolBase( std::move(parts_), + std::move(mutations_snapshot_), std::move(shared_virtual_fields_), storage_snapshot_, prewhere_info_, @@ -33,13 +122,16 @@ MergeTreeReadPoolParallelReplicas::MergeTreeReadPoolParallelReplicas( context_) , extension(std::move(extension_)) , coordination_mode(CoordinationMode::Default) - , min_marks_per_task(pool_settings.min_marks_for_concurrent_read) + , min_marks_per_task(getMinMarksPerTask(pool_settings.min_marks_for_concurrent_read, per_part_infos)) + , mark_segment_size(chooseSegmentSize( + log, + context_->getSettingsRef().parallel_replicas_mark_segment_size, + min_marks_per_task, + pool_settings.threads, + pool_settings.sum_marks, + extension.getTotalNodesCount())) { - for (const auto & info : per_part_infos) - min_marks_per_task = std::max(min_marks_per_task, info->min_marks_per_task); - - extension.all_callback( - InitialAllRangesAnnouncement(coordination_mode, parts_ranges.getDescriptions(), extension.number_of_current_replica)); + extension.sendInitialRequest(coordination_mode, parts_ranges, mark_segment_size); } MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicas::getTask(size_t /*task_idx*/, MergeTreeReadTask * previous_task) @@ -51,12 +143,11 @@ MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicas::getTask(size_t /*task_id if (buffered_ranges.empty()) { - auto result = extension.callback(ParallelReadRequest( + auto result = extension.sendReadRequest( coordination_mode, - extension.number_of_current_replica, min_marks_per_task * pool_settings.threads, /// For Default coordination mode we don't need to pass part names. - RangesInDataPartsDescription{})); + RangesInDataPartsDescription{}); if (!result || result->finish) { @@ -104,6 +195,7 @@ MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicas::getTask(size_t /*task_id if (current_task.ranges.empty()) buffered_ranges.pop_front(); + ProfileEvents::increment(ProfileEvents::ParallelReplicasReadMarks, current_sum_marks); return createTask(per_part_infos[part_idx], std::move(ranges_to_read), previous_task); } diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h index 6ba63cc2c9a0..b9f2e133c4a2 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h @@ -11,6 +11,7 @@ class MergeTreeReadPoolParallelReplicas : public MergeTreeReadPoolBase MergeTreeReadPoolParallelReplicas( ParallelReadingExtension extension_, RangesInDataParts && parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, @@ -30,12 +31,13 @@ class MergeTreeReadPoolParallelReplicas : public MergeTreeReadPoolBase private: mutable std::mutex mutex; + LoggerPtr log = getLogger("MergeTreeReadPoolParallelReplicas"); const ParallelReadingExtension extension; const CoordinationMode coordination_mode; size_t min_marks_per_task{0}; + size_t mark_segment_size{0}; RangesInDataPartsDescription buffered_ranges; bool no_more_tasks_available{false}; - LoggerPtr log = getLogger("MergeTreeReadPoolParallelReplicas"); }; } diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp index 6b5cf9784232..2fdf65dcf36d 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp @@ -1,5 +1,10 @@ #include +namespace ProfileEvents +{ +extern const Event ParallelReplicasReadMarks; +} + namespace DB { @@ -12,6 +17,7 @@ MergeTreeReadPoolParallelReplicasInOrder::MergeTreeReadPoolParallelReplicasInOrd ParallelReadingExtension extension_, CoordinationMode mode_, RangesInDataParts parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, @@ -22,6 +28,7 @@ MergeTreeReadPoolParallelReplicasInOrder::MergeTreeReadPoolParallelReplicasInOrd const ContextPtr & context_) : MergeTreeReadPoolBase( std::move(parts_), + std::move(mutations_snapshot_), std::move(shared_virtual_fields_), storage_snapshot_, prewhere_info_, @@ -43,11 +50,7 @@ MergeTreeReadPoolParallelReplicasInOrder::MergeTreeReadPoolParallelReplicasInOrd for (const auto & part : parts_ranges) buffered_tasks.push_back({part.data_part->info, MarkRanges{}}); - extension.all_callback(InitialAllRangesAnnouncement( - mode, - parts_ranges.getDescriptions(), - extension.number_of_current_replica - )); + extension.sendInitialRequest(mode, parts_ranges, /*mark_segment_size_=*/0); } MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicasInOrder::getTask(size_t task_idx, MergeTreeReadTask * previous_task) @@ -68,20 +71,20 @@ MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicasInOrder::getTask(size_t ta { auto result = std::move(desc.ranges); desc.ranges = MarkRanges{}; + ProfileEvents::increment(ProfileEvents::ParallelReplicasReadMarks, desc.ranges.getNumberOfMarks()); return result; } } return std::nullopt; }; - if (auto result = get_from_buffer(); result) + if (auto result = get_from_buffer()) return createTask(per_part_infos[task_idx], std::move(*result), previous_task); if (no_more_tasks) return nullptr; - auto response - = extension.callback(ParallelReadRequest(mode, extension.number_of_current_replica, min_marks_per_task * request.size(), request)); + auto response = extension.sendReadRequest(mode, min_marks_per_task * request.size(), request); if (!response || response->description.empty() || response->finish) { @@ -97,7 +100,7 @@ MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicasInOrder::getTask(size_t ta std::move(new_ranges.begin(), new_ranges.end(), std::back_inserter(old_ranges)); } - if (auto result = get_from_buffer(); result) + if (auto result = get_from_buffer()) return createTask(per_part_infos[task_idx], std::move(*result), previous_task); return nullptr; diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h index 22841bea212d..98a4d95768ad 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h @@ -12,6 +12,7 @@ class MergeTreeReadPoolParallelReplicasInOrder : public MergeTreeReadPoolBase ParallelReadingExtension extension_, CoordinationMode mode_, RangesInDataParts parts_, + MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, const PrewhereInfoPtr & prewhere_info_, diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 1a0709faf1cd..a7ed91c8fc1e 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -15,6 +15,39 @@ #include #include +namespace +{ + +template +struct TelemetryWrapper +{ + TelemetryWrapper(Func callback_, ProfileEvents::Event event_, std::string span_name_) + : callback(std::move(callback_)), event(event_), span_name(std::move(span_name_)) + { + } + + template + auto operator()(Args &&... args) + { + DB::OpenTelemetry::SpanHolder span(span_name); + DB::ProfileEventTimeIncrement increment(event); + return callback(std::forward(args)...); + } + +private: + Func callback; + ProfileEvents::Event event; + std::string span_name; +}; + +} + +namespace ProfileEvents +{ +extern const Event ParallelReplicasAnnouncementMicroseconds; +extern const Event ParallelReplicasReadRequestMicroseconds; +} + namespace DB { @@ -23,6 +56,31 @@ namespace ErrorCodes extern const int QUERY_WAS_CANCELLED; } +ParallelReadingExtension::ParallelReadingExtension( + MergeTreeAllRangesCallback all_callback_, + MergeTreeReadTaskCallback callback_, + size_t number_of_current_replica_, + size_t total_nodes_count_) + : number_of_current_replica(number_of_current_replica_), total_nodes_count(total_nodes_count_) +{ + all_callback = TelemetryWrapper{ + std::move(all_callback_), ProfileEvents::ParallelReplicasAnnouncementMicroseconds, "ParallelReplicasAnnouncement"}; + + callback = TelemetryWrapper{ + std::move(callback_), ProfileEvents::ParallelReplicasReadRequestMicroseconds, "ParallelReplicasReadRequest"}; +} + +void ParallelReadingExtension::sendInitialRequest(CoordinationMode mode, const RangesInDataParts & ranges, size_t mark_segment_size) const +{ + all_callback(InitialAllRangesAnnouncement{mode, ranges.getDescriptions(), number_of_current_replica, mark_segment_size}); +} + +std::optional ParallelReadingExtension::sendReadRequest( + CoordinationMode mode, size_t min_number_of_marks, const RangesInDataPartsDescription & description) const +{ + return callback(ParallelReadRequest{mode, number_of_current_replica, min_number_of_marks, description}); +} + MergeTreeSelectProcessor::MergeTreeSelectProcessor( MergeTreeReadPoolPtr pool_, MergeTreeSelectAlgorithmPtr algorithm_, diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 7a9cebbcb2ed..8a9e3580a9f7 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -1,9 +1,12 @@ #pragma once + +#include #include -#include #include -#include #include +#include +#include + #include @@ -22,11 +25,27 @@ struct ChunkAndProgress bool is_finished = false; }; -struct ParallelReadingExtension +class ParallelReadingExtension { +public: + ParallelReadingExtension( + MergeTreeAllRangesCallback all_callback_, + MergeTreeReadTaskCallback callback_, + size_t number_of_current_replica_, + size_t total_nodes_count_); + + void sendInitialRequest(CoordinationMode mode, const RangesInDataParts & ranges, size_t mark_segment_size) const; + + std::optional + sendReadRequest(CoordinationMode mode, size_t min_number_of_marks, const RangesInDataPartsDescription & description) const; + + size_t getTotalNodesCount() const { return total_nodes_count; } + +private: MergeTreeAllRangesCallback all_callback; MergeTreeReadTaskCallback callback; - size_t number_of_current_replica{0}; + const size_t number_of_current_replica; + const size_t total_nodes_count; }; /// Base class for MergeTreeThreadSelectAlgorithm and MergeTreeSelectAlgorithm diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index f06dbabaeb5d..3b33bfce2676 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -34,6 +34,7 @@ class MergeTreeSequentialSource : public ISource const MergeTreeData & storage_, const StorageSnapshotPtr & storage_snapshot_, MergeTreeData::DataPartPtr data_part_, + AlterConversionsPtr alter_conversions_, Names columns_to_read_, std::optional mark_ranges_, bool apply_deleted_mask, @@ -58,6 +59,9 @@ class MergeTreeSequentialSource : public ISource /// Data part will not be removed if the pointer owns it MergeTreeData::DataPartPtr data_part; + /// Alter and mutation commands that are required to be applied to the part on-fly. + AlterConversionsPtr alter_conversions; + /// Columns we have to read (each Block from read will contain them) Names columns_to_read; @@ -87,6 +91,7 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( const MergeTreeData & storage_, const StorageSnapshotPtr & storage_snapshot_, MergeTreeData::DataPartPtr data_part_, + AlterConversionsPtr alter_conversions_, Names columns_to_read_, std::optional mark_ranges_, bool apply_deleted_mask, @@ -96,6 +101,7 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( , storage(storage_) , storage_snapshot(storage_snapshot_) , data_part(std::move(data_part_)) + , alter_conversions(std::move(alter_conversions_)) , columns_to_read(std::move(columns_to_read_)) , read_with_direct_io(read_with_direct_io_) , mark_ranges(std::move(mark_ranges_)) @@ -109,8 +115,6 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( LOG_DEBUG(log, "Reading {} marks from part {}, total {} rows starting from the beginning of the part", data_part->getMarksCount(), data_part->name, data_part->rows_count); - auto alter_conversions = storage.getAlterConversionsForPart(data_part); - /// Note, that we don't check setting collaborate_with_coordinator presence, because this source /// is only used in background merges. addTotalRowsApprox(data_part->rows_count); @@ -299,6 +303,7 @@ Pipe createMergeTreeSequentialSource( const MergeTreeData & storage, const StorageSnapshotPtr & storage_snapshot, MergeTreeData::DataPartPtr data_part, + AlterConversionsPtr alter_conversions, Names columns_to_read, std::optional mark_ranges, std::shared_ptr> filtered_rows_count, @@ -315,7 +320,8 @@ Pipe createMergeTreeSequentialSource( columns_to_read.emplace_back(RowExistsColumn::name); auto column_part_source = std::make_shared(type, - storage, storage_snapshot, data_part, columns_to_read, std::move(mark_ranges), + storage, storage_snapshot, data_part, alter_conversions, + columns_to_read, std::move(mark_ranges), /*apply_deleted_mask=*/ false, read_with_direct_io, prefetch); Pipe pipe(std::move(column_part_source)); @@ -346,6 +352,7 @@ class ReadFromPart final : public ISourceStep const MergeTreeData & storage_, const StorageSnapshotPtr & storage_snapshot_, MergeTreeData::DataPartPtr data_part_, + AlterConversionsPtr alter_conversions_, Names columns_to_read_, bool apply_deleted_mask_, std::optional filter_, @@ -356,6 +363,7 @@ class ReadFromPart final : public ISourceStep , storage(storage_) , storage_snapshot(storage_snapshot_) , data_part(std::move(data_part_)) + , alter_conversions(std::move(alter_conversions_)) , columns_to_read(std::move(columns_to_read_)) , apply_deleted_mask(apply_deleted_mask_) , filter(std::move(filter_)) @@ -399,6 +407,7 @@ class ReadFromPart final : public ISourceStep storage, storage_snapshot, data_part, + alter_conversions, columns_to_read, std::move(mark_ranges), /*filtered_rows_count=*/ nullptr, @@ -414,6 +423,7 @@ class ReadFromPart final : public ISourceStep const MergeTreeData & storage; StorageSnapshotPtr storage_snapshot; MergeTreeData::DataPartPtr data_part; + AlterConversionsPtr alter_conversions; Names columns_to_read; bool apply_deleted_mask; std::optional filter; @@ -427,6 +437,7 @@ void createReadFromPartStep( const MergeTreeData & storage, const StorageSnapshotPtr & storage_snapshot, MergeTreeData::DataPartPtr data_part, + AlterConversionsPtr alter_conversions, Names columns_to_read, bool apply_deleted_mask, std::optional filter, @@ -434,7 +445,8 @@ void createReadFromPartStep( LoggerPtr log) { auto reading = std::make_unique(type, - storage, storage_snapshot, std::move(data_part), + storage, storage_snapshot, + std::move(data_part), std::move(alter_conversions), std::move(columns_to_read), apply_deleted_mask, std::move(filter), std::move(context), log); diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.h b/src/Storages/MergeTree/MergeTreeSequentialSource.h index 1b05512b9a30..5b7c80385f67 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.h +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.h @@ -21,6 +21,7 @@ Pipe createMergeTreeSequentialSource( const MergeTreeData & storage, const StorageSnapshotPtr & storage_snapshot, MergeTreeData::DataPartPtr data_part, + AlterConversionsPtr alter_conversions, Names columns_to_read, std::optional mark_ranges, std::shared_ptr> filtered_rows_count, @@ -36,6 +37,7 @@ void createReadFromPartStep( const MergeTreeData & storage, const StorageSnapshotPtr & storage_snapshot, MergeTreeData::DataPartPtr data_part, + AlterConversionsPtr alter_conversions, Names columns_to_read, bool apply_deleted_mask, std::optional filter, diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index e1ca6347d3b2..1e877362c152 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -114,6 +114,7 @@ static UInt64 getExistingRowsCount(const Block & block) static void splitAndModifyMutationCommands( MergeTreeData::DataPartPtr part, StorageMetadataPtr metadata_snapshot, + AlterConversionsPtr alter_conversions, const MutationCommands & commands, MutationCommands & for_interpreter, MutationCommands & for_file_renames, @@ -179,8 +180,6 @@ static void splitAndModifyMutationCommands( } - auto alter_conversions = part->storage.getAlterConversionsForPart(part); - /// We don't add renames from commands, instead we take them from rename_map. /// It's important because required renames depend not only on part's data version (i.e. mutation version) /// but also on part's metadata version. Why we have such logic only for renames? Because all other types of alter @@ -296,7 +295,6 @@ static void splitAndModifyMutationCommands( } } - auto alter_conversions = part->storage.getAlterConversionsForPart(part); /// We don't add renames from commands, instead we take them from rename_map. /// It's important because required renames depend not only on part's data version (i.e. mutation version) /// but also on part's metadata version. Why we have such logic only for renames? Because all other types of alter @@ -2160,6 +2158,15 @@ bool MutateTask::prepare() ctx->num_mutations = std::make_unique(CurrentMetrics::PartMutation); + MergeTreeData::IMutationsSnapshot::Params params + { + .metadata_version = ctx->metadata_snapshot->getMetadataVersion(), + .min_part_metadata_version = ctx->source_part->getMetadataVersion(), + }; + + auto mutations_snapshot = ctx->data->getMutationsSnapshot(params); + auto alter_conversions = MergeTreeData::getAlterConversionsForPart(ctx->source_part, mutations_snapshot, ctx->metadata_snapshot, ctx->context); + auto context_for_reading = Context::createCopy(ctx->context); /// Allow mutations to work when force_index_by_date or force_primary_key is on. @@ -2174,7 +2181,7 @@ bool MutateTask::prepare() ctx->commands_for_part.emplace_back(command); if (ctx->source_part->isStoredOnDisk() && !isStorageTouchedByMutations( - ctx->source_part, ctx->metadata_snapshot, ctx->commands_for_part, context_for_reading)) + ctx->source_part, mutations_snapshot, ctx->metadata_snapshot, ctx->commands_for_part, context_for_reading)) { NameSet files_to_copy_instead_of_hardlinks; auto settings_ptr = ctx->data->getSettings(); @@ -2234,8 +2241,13 @@ bool MutateTask::prepare() context_for_reading->setSetting("read_from_filesystem_cache_if_exists_otherwise_bypass_cache", 1); MutationHelpers::splitAndModifyMutationCommands( - ctx->source_part, ctx->metadata_snapshot, - ctx->commands_for_part, ctx->for_interpreter, ctx->for_file_renames, ctx->log); + ctx->source_part, + ctx->metadata_snapshot, + alter_conversions, + ctx->commands_for_part, + ctx->for_interpreter, + ctx->for_file_renames, + ctx->log); ctx->stage_progress = std::make_unique(1.0); @@ -2247,7 +2259,8 @@ bool MutateTask::prepare() settings.apply_deleted_mask = false; ctx->interpreter = std::make_unique( - *ctx->data, ctx->source_part, ctx->metadata_snapshot, ctx->for_interpreter, + *ctx->data, ctx->source_part, alter_conversions, + ctx->metadata_snapshot, ctx->for_interpreter, ctx->metadata_snapshot->getColumns().getNamesOfPhysical(), context_for_reading, settings); ctx->materialized_indices = ctx->interpreter->grabMaterializedIndices(); diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index f46b4de10b7d..4067480cd7fd 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -185,7 +185,8 @@ class ParallelReplicasReadingCoordinator::ImplInterface void handleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement announcement) { if (++sent_initial_requests > replicas_count) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Initiator received more initial requests than there are replicas"); + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Initiator received more initial requests than there are replicas: replica_num={}", announcement.replica_num); doHandleInitialAllRangesAnnouncement(std::move(announcement)); } @@ -211,14 +212,11 @@ using PartRefs = std::deque; class DefaultCoordinator : public ParallelReplicasReadingCoordinator::ImplInterface { public: - explicit DefaultCoordinator(size_t replicas_count_, size_t mark_segment_size_) + explicit DefaultCoordinator(size_t replicas_count_) : ParallelReplicasReadingCoordinator::ImplInterface(replicas_count_) - , mark_segment_size(mark_segment_size_) , replica_status(replicas_count_) , distribution_by_hash_queue(replicas_count_) { - if (mark_segment_size == 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Zero value provided for `mark_segment_size`"); } ~DefaultCoordinator() override; @@ -231,7 +229,7 @@ class DefaultCoordinator : public ParallelReplicasReadingCoordinator::ImplInterf private: /// This many granules will represent a single segment of marks that will be assigned to a replica - const size_t mark_segment_size{0}; + size_t mark_segment_size{0}; bool state_initialized{false}; size_t finished_replicas{0}; @@ -393,7 +391,11 @@ void DefaultCoordinator::initializeReadingState(InitialAllRangesAnnouncement ann state_initialized = true; source_replica_for_parts_snapshot = announcement.replica_num; - LOG_DEBUG(log, "Reading state is fully initialized: {}", fmt::join(all_parts_to_read, "; ")); + mark_segment_size = announcement.mark_segment_size; + if (mark_segment_size == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Zero value provided for `mark_segment_size`"); + + LOG_DEBUG(log, "Reading state is fully initialized: {}, mark_segment_size: {}", fmt::join(all_parts_to_read, "; "), mark_segment_size); } void DefaultCoordinator::markReplicaAsUnavailable(size_t replica_number) @@ -433,9 +435,9 @@ void DefaultCoordinator::setProgressCallback() void DefaultCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement announcement) { - const auto replica_num = announcement.replica_num; + LOG_DEBUG(log, "Initial request: {}", announcement.describe()); - LOG_DEBUG(log, "Initial request from replica {}: {}", announcement.replica_num, announcement.describe()); + const auto replica_num = announcement.replica_num; initializeReadingState(std::move(announcement)); @@ -839,6 +841,7 @@ class InOrderCoordinator : public ParallelReplicasReadingCoordinator::ImplInterf Parts all_parts_to_read; size_t total_rows_to_read = 0; + bool state_initialized{false}; LoggerPtr log = getLogger(fmt::format("{}{}", magic_enum::enum_name(mode), "Coordinator")); }; @@ -858,7 +861,9 @@ void InOrderCoordinator::markReplicaAsUnavailable(size_t replica_number) template void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement announcement) { - LOG_TRACE(log, "Received an announcement {}", announcement.describe()); + LOG_TRACE(log, "Received an announcement : {}", announcement.describe()); + + ++stats[announcement.replica_num].number_of_requests; size_t new_rows_to_read = 0; @@ -868,13 +873,16 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa auto the_same_it = std::find_if(all_parts_to_read.begin(), all_parts_to_read.end(), [&part] (const Part & other) { return other.description.info == part.info; }); - /// We have the same part - add the info about presence on current replica to it + /// We have the same part - add the info about presence on the corresponding replica to it if (the_same_it != all_parts_to_read.end()) { the_same_it->replicas.insert(announcement.replica_num); continue; } + if (state_initialized) + continue; + auto covering_or_the_same_it = std::find_if(all_parts_to_read.begin(), all_parts_to_read.end(), [&part] (const Part & other) { return other.description.info.contains(part.info) || part.info.contains(other.description.info); }); @@ -889,9 +897,10 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa std::sort(ranges.begin(), ranges.end()); } - ++stats[announcement.replica_num].number_of_requests; + state_initialized = true; - if (new_rows_to_read > 0) + // progress_callback is not set when local plan is used for initiator + if (progress_callback && new_rows_to_read > 0) { Progress progress; progress.total_rows_to_read = new_rows_to_read; @@ -911,7 +920,7 @@ ParallelReadResponse InOrderCoordinator::handleRequest(ParallelReadRequest "Replica {} decided to read in {} mode, not in {}. This is a bug", request.replica_num, magic_enum::enum_name(request.mode), magic_enum::enum_name(mode)); - LOG_TRACE(log, "Got request from replica {}, data {}", request.replica_num, request.describe()); + LOG_TRACE(log, "Got read request: {}", request.describe()); ParallelReadResponse response; response.description = request.description; @@ -925,8 +934,15 @@ ParallelReadResponse InOrderCoordinator::handleRequest(ParallelReadRequest if (global_part_it == all_parts_to_read.end()) continue; + if (global_part_it->replicas.empty()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Part {} requested by replica {} is not registered in working set", + part.info.getPartNameV1(), + request.replica_num); + if (!global_part_it->replicas.contains(request.replica_num)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} doesn't exist on replica {} according to the global state", part.info.getPartNameV1(), request.replica_num); + continue; size_t current_mark_size = 0; @@ -1043,7 +1059,7 @@ void ParallelReplicasReadingCoordinator::initialize(CoordinationMode mode) switch (mode) { case CoordinationMode::Default: - pimpl = std::make_unique(replicas_count, mark_segment_size); + pimpl = std::make_unique(replicas_count); break; case CoordinationMode::WithOrder: pimpl = std::make_unique>(replicas_count); @@ -1053,6 +1069,7 @@ void ParallelReplicasReadingCoordinator::initialize(CoordinationMode mode) break; } + // progress_callback is not set when local plan is used for initiator if (progress_callback) pimpl->setProgressCallback(std::move(progress_callback)); @@ -1060,8 +1077,7 @@ void ParallelReplicasReadingCoordinator::initialize(CoordinationMode mode) pimpl->markReplicaAsUnavailable(replica); } -ParallelReplicasReadingCoordinator::ParallelReplicasReadingCoordinator(size_t replicas_count_, size_t mark_segment_size_) - : replicas_count(replicas_count_), mark_segment_size(mark_segment_size_) +ParallelReplicasReadingCoordinator::ParallelReplicasReadingCoordinator(size_t replicas_count_) : replicas_count(replicas_count_) { } diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h index 8b463fda395b..ad51d20f5531 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h @@ -15,7 +15,7 @@ class ParallelReplicasReadingCoordinator public: class ImplInterface; - explicit ParallelReplicasReadingCoordinator(size_t replicas_count_, size_t mark_segment_size_ = 0); + explicit ParallelReplicasReadingCoordinator(size_t replicas_count_); ~ParallelReplicasReadingCoordinator(); void handleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement); @@ -35,7 +35,6 @@ class ParallelReplicasReadingCoordinator std::mutex mutex; const size_t replicas_count{0}; - size_t mark_segment_size{0}; std::unique_ptr pimpl; ProgressCallback progress_callback; // store the callback only to bypass it to coordinator implementation std::set replicas_used; diff --git a/src/Storages/MergeTree/RangesInDataPart.h b/src/Storages/MergeTree/RangesInDataPart.h index bf9e4c7dfb2e..966637d08124 100644 --- a/src/Storages/MergeTree/RangesInDataPart.h +++ b/src/Storages/MergeTree/RangesInDataPart.h @@ -42,7 +42,6 @@ struct RangesInDataPartsDescription: public std::dequeznode_name, MutationStatus(entry, format_version)) - .first->second; + auto & mutation = mutations_by_znode.emplace(entry->znode_name, MutationStatus(entry, format_version)).first->second; + incrementMutationsCounters(num_data_mutations_to_apply, num_metadata_mutations_to_apply, entry->commands); - updateAlterConversionsMutations(entry->commands, alter_conversions_mutations, /* remove= */ false); NOEXCEPT_SCOPE({ for (const auto & pair : entry->block_numbers) { @@ -1076,7 +1075,7 @@ ReplicatedMergeTreeMutationEntryPtr ReplicatedMergeTreeQueue::removeMutation( } mutations_by_znode.erase(it); - /// updateAlterConversionsMutations() will be called in updateMutations() + /// decrementMutationsCounters() will be called in updateMutations() LOG_DEBUG(log, "Removed mutation {} from local state.", entry->znode_name); } @@ -1901,50 +1900,47 @@ ReplicatedMergeTreeMergePredicate ReplicatedMergeTreeQueue::getMergePredicate(zk } -MutationCommands ReplicatedMergeTreeQueue::getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const +MutationCommands ReplicatedMergeTreeQueue::MutationsSnapshot::getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const { - int32_t part_metadata_version = part->getMetadataVersion(); - int32_t metadata_version = storage.getInMemoryMetadataPtr()->getMetadataVersion(); - - chassert(alter_conversions_mutations >= 0); - /// NOTE: that just checking part_metadata_version is not enough, since we - /// need to check for non-metadata mutations as well. - if (alter_conversions_mutations == 0 && metadata_version == part_metadata_version) - return {}; - - std::unique_lock lock(state_mutex); - auto in_partition = mutations_by_partition.find(part->info.partition_id); if (in_partition == mutations_by_partition.end()) return {}; Int64 part_data_version = part->info.getDataVersion(); + Int64 part_metadata_version = part->getMetadataVersion(); + MutationCommands result; - bool seen_all_data_mutations = false; - bool seen_all_metadata_mutations = false; + bool seen_all_data_mutations = !hasDataMutations(); + bool seen_all_metadata_mutations = part_metadata_version >= params.metadata_version; + + if (seen_all_data_mutations && seen_all_metadata_mutations) + return {}; auto add_to_result = [&](const ReplicatedMergeTreeMutationEntryPtr & entry) { for (const auto & command : entry->commands | std::views::reverse) - if (AlterConversions::supportsMutationCommandType(command.type)) - result.emplace_back(command); + { + if (AlterConversions::isSupportedMetadataMutation(command.type)) + result.push_back(command); + else if (params.need_data_mutations && AlterConversions::isSupportedDataMutation(command.type)) + result.push_back(command); + } }; /// Here we return mutation commands for part which has bigger alter version than part metadata version. /// Please note, we don't use getDataVersion(). It's because these alter commands are used for in-fly conversions /// of part's metadata. - for (const auto & [mutation_version, mutation_status] : in_partition->second | std::views::reverse) + for (const auto & [mutation_version, entry] : in_partition->second | std::views::reverse) { if (seen_all_data_mutations && seen_all_metadata_mutations) break; - auto & entry = mutation_status->entry; - auto alter_version = entry->alter_version; + if (alter_version != -1) { - if (alter_version > metadata_version) + if (seen_all_metadata_mutations || alter_version > params.metadata_version) continue; /// We take commands with bigger metadata version @@ -1953,7 +1949,7 @@ MutationCommands ReplicatedMergeTreeQueue::getAlterMutationCommandsForPart(const else seen_all_metadata_mutations = true; } - else + else if (!seen_all_data_mutations) { if (mutation_version > part_data_version) add_to_result(entry); @@ -1965,6 +1961,104 @@ MutationCommands ReplicatedMergeTreeQueue::getAlterMutationCommandsForPart(const return result; } +NameSet ReplicatedMergeTreeQueue::MutationsSnapshot::getAllUpdatedColumns() const +{ + if (!hasDataMutations()) + return {}; + + NameSet res; + for (const auto & [partition_id, mutations] : mutations_by_partition) + { + for (const auto & [version, entry] : mutations) + { + auto names = entry->commands.getAllUpdatedColumns(); + std::move(names.begin(), names.end(), std::inserter(res, res.end())); + } + } + return res; +} + +MergeTreeData::MutationsSnapshotPtr ReplicatedMergeTreeQueue::getMutationsSnapshot(const MutationsSnapshot::Params & params) const +{ + std::lock_guard lock(state_mutex); + + MutationsSnapshot::Info info + { + .num_data_mutations = num_data_mutations_to_apply, + .num_metadata_mutations = num_metadata_mutations_to_apply, + }; + + auto res = std::make_shared(params, std::move(info)); + + bool need_data_mutations = res->hasDataMutations(); + bool need_metatadata_mutations = params.min_part_metadata_version < params.metadata_version; + + if (!need_data_mutations && !need_metatadata_mutations) + return res; + + auto is_supported_command = [&](const auto & command) + { + if (need_data_mutations && AlterConversions::isSupportedDataMutation(command.type)) + return true; + + if (need_metatadata_mutations && AlterConversions::isSupportedMetadataMutation(command.type)) + return true; + + return false; + }; + + for (const auto & [partition_id, mutations] : mutations_by_partition) + { + auto & in_partition = res->mutations_by_partition[partition_id]; + + bool seen_all_data_mutations = !need_data_mutations; + bool seen_all_metadata_mutations = !need_metatadata_mutations; + + for (const auto & [mutation_version, status] : mutations | std::views::reverse) + { + if (seen_all_data_mutations && seen_all_metadata_mutations) + break; + + auto alter_version = status->entry->alter_version; + + if (alter_version != -1) + { + if (seen_all_metadata_mutations || alter_version > params.metadata_version) + continue; + + /// We take commands with bigger metadata version + if (alter_version > params.min_part_metadata_version) + { + /// Copy a pointer to the whole entry to avoid extracting and copying commands. + /// Required commands will be copied later only for specific parts. + if (std::ranges::any_of(status->entry->commands, is_supported_command)) + in_partition.emplace(mutation_version, status->entry); + } + else + { + seen_all_metadata_mutations = true; + } + } + else if (!seen_all_data_mutations) + { + if (!status->is_done) + { + /// Copy a pointer to the whole entry to avoid extracting and copying commands. + /// Required commands will be copied later only for specific parts. + if (std::ranges::any_of(status->entry->commands, is_supported_command)) + in_partition.emplace(mutation_version, status->entry); + } + else + { + seen_all_data_mutations = true; + } + } + } + } + + return res; +} + MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( const MergeTreeData::DataPartPtr & part, Int64 desired_mutation_version, Strings & mutation_ids) const { @@ -2045,7 +2139,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep mutation.parts_to_do.clear(); } - updateAlterConversionsMutations(mutation.entry->commands, alter_conversions_mutations, /* remove= */ true); + decrementMutationsCounters(num_data_mutations_to_apply, num_metadata_mutations_to_apply, mutation.entry->commands); } else if (mutation.parts_to_do.size() == 0) { @@ -2102,7 +2196,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep LOG_TRACE(log, "Finishing data alter with version {} for entry {}", entry->alter_version, entry->znode_name); alter_sequence.finishDataAlter(entry->alter_version, lock); } - updateAlterConversionsMutations(entry->commands, alter_conversions_mutations, /* remove= */ true); + decrementMutationsCounters(num_data_mutations_to_apply, num_metadata_mutations_to_apply, entry->commands); } } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index e300078a35af..6ec8818b0c65 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -151,8 +152,11 @@ class ReplicatedMergeTreeQueue /// Mapping from znode path to Mutations Status std::map mutations_by_znode; - /// Unfinished mutations that is required AlterConversions (see getAlterMutationCommandsForPart()) - std::atomic alter_conversions_mutations = 0; + + /// Unfinished mutations that are required for AlterConversions. + Int64 num_data_mutations_to_apply = 0; + Int64 num_metadata_mutations_to_apply = 0; + /// Partition -> (block_number -> MutationStatus) std::unordered_map> mutations_by_partition; /// Znode ID of the latest mutation that is done. @@ -410,10 +414,26 @@ class ReplicatedMergeTreeQueue MutationCommands getMutationCommands(const MergeTreeData::DataPartPtr & part, Int64 desired_mutation_version, Strings & mutation_ids) const; + struct MutationsSnapshot : public MergeTreeData::IMutationsSnapshot + { + public: + MutationsSnapshot() = default; + MutationsSnapshot(Params params_, Info info_) : IMutationsSnapshot(std::move(params_), std::move(info_)) {} + + using Params = MergeTreeData::IMutationsSnapshot::Params; + using MutationsByPartititon = std::unordered_map>; + + MutationsByPartititon mutations_by_partition; + + MutationCommands getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const override; + std::shared_ptr cloneEmpty() const override { return std::make_shared(); } + NameSet getAllUpdatedColumns() const override; + }; + /// Return mutation commands for part which could be not applied to /// it according to part mutation version. Used when we apply alter commands on fly, /// without actual data modification on disk. - MutationCommands getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const; + MergeTreeData::MutationsSnapshotPtr getMutationsSnapshot(const MutationsSnapshot::Params & params) const; /// Mark finished mutations as done. If the function needs to be called again at some later time /// (because some mutations are probably done but we are not sure yet), returns true. diff --git a/src/Storages/MergeTree/RequestResponse.cpp b/src/Storages/MergeTree/RequestResponse.cpp index 2ce0e20dcd20..f6859b269082 100644 --- a/src/Storages/MergeTree/RequestResponse.cpp +++ b/src/Storages/MergeTree/RequestResponse.cpp @@ -2,10 +2,10 @@ #include #include -#include +#include #include #include -#include +#include #include @@ -14,25 +14,29 @@ namespace DB namespace ErrorCodes { - extern const int UNKNOWN_PROTOCOL; - extern const int UNKNOWN_ELEMENT_OF_ENUM; +extern const int UNKNOWN_PROTOCOL; +extern const int UNKNOWN_ELEMENT_OF_ENUM; } namespace { - CoordinationMode validateAndGet(uint8_t candidate) - { - if (candidate <= static_cast(CoordinationMode::MAX)) - return static_cast(candidate); +CoordinationMode validateAndGet(uint8_t candidate) +{ + if (candidate <= static_cast(CoordinationMode::MAX)) + return static_cast(candidate); - throw Exception(ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM, "Unknown reading mode: {}", candidate); - } + throw Exception(ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM, "Unknown reading mode: {}", candidate); +} } -void ParallelReadRequest::serialize(WriteBuffer & out) const +void ParallelReadRequest::serialize(WriteBuffer & out, UInt64 initiator_protocol_version) const { - UInt64 version = DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION; - /// Must be the first + /// Previously we didn't maintain backward compatibility and every change was breaking. + /// Particularly, we had an equality check for the version. To work around that code + /// in previous server versions we now have to lie to them about the version. + const UInt64 version = initiator_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL + ? DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION + : DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION; writeIntBinary(version, out); writeIntBinary(mode, out); @@ -44,9 +48,7 @@ void ParallelReadRequest::serialize(WriteBuffer & out) const String ParallelReadRequest::describe() const { - String result; - result += fmt::format("replica_num: {} \n", replica_num); - result += fmt::format("min_num_of_marks: {} \n", min_number_of_marks); + String result = fmt::format("replica_num {}, min_num_of_marks {}, ", replica_num, min_number_of_marks); result += description.describe(); return result; } @@ -55,10 +57,12 @@ ParallelReadRequest ParallelReadRequest::deserialize(ReadBuffer & in) { UInt64 version; readIntBinary(version, in); - if (version != DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION) - throw Exception(ErrorCodes::UNKNOWN_PROTOCOL, "Protocol versions for parallel reading "\ - "from replicas differ. Got: {}, supported version: {}", - version, DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION); + if (version < DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION) + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Parallel replicas protocol version is too old. Got: {}, min supported version: {}", + version, + DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION); CoordinationMode mode; size_t replica_num; @@ -72,12 +76,7 @@ ParallelReadRequest ParallelReadRequest::deserialize(ReadBuffer & in) readIntBinary(min_number_of_marks, in); description.deserialize(in); - return ParallelReadRequest( - mode, - replica_num, - min_number_of_marks, - std::move(description) - ); + return ParallelReadRequest(mode, replica_num, min_number_of_marks, std::move(description)); } void ParallelReadRequest::merge(ParallelReadRequest & other) @@ -88,9 +87,14 @@ void ParallelReadRequest::merge(ParallelReadRequest & other) description.merge(other.description); } -void ParallelReadResponse::serialize(WriteBuffer & out) const +void ParallelReadResponse::serialize(WriteBuffer & out, UInt64 replica_protocol_version) const { - UInt64 version = DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION; + /// Previously we didn't maintain backward compatibility and every change was breaking. + /// Particularly, we had an equality check for the version. To work around that code + /// in previous server versions we now have to lie to them about the version. + UInt64 version = replica_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL + ? DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION + : DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION; /// Must be the first writeIntBinary(version, out); @@ -107,44 +111,51 @@ void ParallelReadResponse::deserialize(ReadBuffer & in) { UInt64 version; readIntBinary(version, in); - if (version != DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION) - throw Exception(ErrorCodes::UNKNOWN_PROTOCOL, "Protocol versions for parallel reading " \ - "from replicas differ. Got: {}, supported version: {}", - version, DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION); + if (version < DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION) + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Parallel replicas protocol version is too old. Got: {}, min supported version: {}", + version, + DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION); readBoolText(finish, in); description.deserialize(in); } -void InitialAllRangesAnnouncement::serialize(WriteBuffer & out) const +void InitialAllRangesAnnouncement::serialize(WriteBuffer & out, UInt64 initiator_protocol_version) const { - UInt64 version = DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION; - /// Must be the first + /// Previously we didn't maintain backward compatibility and every change was breaking. + /// Particularly, we had an equality check for the version. To work around that code + /// in previous server versions we now have to lie to them about the version. + UInt64 version = initiator_protocol_version >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL + ? DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION + : DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION; writeIntBinary(version, out); writeIntBinary(mode, out); description.serialize(out); writeIntBinary(replica_num, out); + if (initiator_protocol_version >= DBMS_PARALLEL_REPLICAS_MIN_VERSION_WITH_MARK_SEGMENT_SIZE_FIELD) + writeIntBinary(mark_segment_size, out); } String InitialAllRangesAnnouncement::describe() { - String result; - result += description.describe(); - result += fmt::format("----------\nReceived from {} replica\n", replica_num); - return result; + return fmt::format("replica {}, mode {}, {}", replica_num, mode, description.describe()); } -InitialAllRangesAnnouncement InitialAllRangesAnnouncement::deserialize(ReadBuffer & in) +InitialAllRangesAnnouncement InitialAllRangesAnnouncement::deserialize(ReadBuffer & in, UInt64 replica_protocol_version) { UInt64 version; readIntBinary(version, in); - if (version != DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION) - throw Exception(ErrorCodes::UNKNOWN_PROTOCOL, "Protocol versions for parallel reading " \ - "from replicas differ. Got: {}, supported version: {}", - version, DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION); + if (version < DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION) + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Parallel replicas protocol version is too old. Got: {}, min supported version: {}", + version, + DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION); CoordinationMode mode; RangesInDataPartsDescription description; @@ -156,11 +167,11 @@ InitialAllRangesAnnouncement InitialAllRangesAnnouncement::deserialize(ReadBuffe description.deserialize(in); readIntBinary(replica_num, in); - return InitialAllRangesAnnouncement { - mode, - description, - replica_num - }; + size_t mark_segment_size = 128; + if (replica_protocol_version >= DBMS_PARALLEL_REPLICAS_MIN_VERSION_WITH_MARK_SEGMENT_SIZE_FIELD) + readIntBinary(mark_segment_size, in); + + return InitialAllRangesAnnouncement{mode, description, replica_num, mark_segment_size}; } } diff --git a/src/Storages/MergeTree/RequestResponse.h b/src/Storages/MergeTree/RequestResponse.h index 5f5516a6804d..96b65c45bfae 100644 --- a/src/Storages/MergeTree/RequestResponse.h +++ b/src/Storages/MergeTree/RequestResponse.h @@ -63,7 +63,7 @@ struct ParallelReadRequest /// Contains only data part names without mark ranges. RangesInDataPartsDescription description; - void serialize(WriteBuffer & out) const; + void serialize(WriteBuffer & out, UInt64 initiator_protocol_version) const; String describe() const; static ParallelReadRequest deserialize(ReadBuffer & in); void merge(ParallelReadRequest & other); @@ -78,7 +78,7 @@ struct ParallelReadResponse bool finish{false}; RangesInDataPartsDescription description; - void serialize(WriteBuffer & out) const; + void serialize(WriteBuffer & out, UInt64 replica_protocol_version) const; String describe() const; void deserialize(ReadBuffer & in); }; @@ -93,21 +93,18 @@ struct InitialAllRangesAnnouncement /// No default constructor, you must initialize all fields at once. InitialAllRangesAnnouncement( - CoordinationMode mode_, - RangesInDataPartsDescription description_, - size_t replica_num_) - : mode(mode_) - , description(description_) - , replica_num(replica_num_) + CoordinationMode mode_, RangesInDataPartsDescription description_, size_t replica_num_, size_t mark_segment_size_) + : mode(mode_), description(std::move(description_)), replica_num(replica_num_), mark_segment_size(mark_segment_size_) {} CoordinationMode mode; RangesInDataPartsDescription description; size_t replica_num; + size_t mark_segment_size; - void serialize(WriteBuffer & out) const; + void serialize(WriteBuffer & out, UInt64 initiator_protocol_version) const; String describe(); - static InitialAllRangesAnnouncement deserialize(ReadBuffer & in); + static InitialAllRangesAnnouncement deserialize(ReadBuffer & i, UInt64 replica_protocol_version); }; diff --git a/src/Storages/StorageFromMergeTreeDataPart.cpp b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.cpp similarity index 98% rename from src/Storages/StorageFromMergeTreeDataPart.cpp rename to src/Storages/MergeTree/StorageFromMergeTreeDataPart.cpp index 481d9e669017..280783245a22 100644 --- a/src/Storages/StorageFromMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.cpp @@ -30,7 +30,7 @@ void StorageFromMergeTreeDataPart::read( { query_plan.addStep(MergeTreeDataSelectExecutor(storage).readFromParts( parts, - alter_conversions, + mutations_snapshot, column_names, storage_snapshot, query_info, diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 82ebbf099888..6825752acd7b 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -18,10 +18,12 @@ class StorageFromMergeTreeDataPart final : public IStorage { public: /// Used in part mutation. - explicit StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) + explicit StorageFromMergeTreeDataPart( + const MergeTreeData::DataPartPtr & part_, + const MergeTreeData::MutationsSnapshotPtr & mutations_snapshot_) : IStorage(getIDFromPart(part_)) , parts({part_}) - , alter_conversions({part_->storage.getAlterConversionsForPart(part_)}) + , mutations_snapshot(mutations_snapshot_) , storage(part_->storage) , partition_id(part_->info.partition_id) { @@ -81,7 +83,7 @@ class StorageFromMergeTreeDataPart final : public IStorage private: const MergeTreeData::DataPartsVector parts; - const std::vector alter_conversions; + const MergeTreeData::MutationsSnapshotPtr mutations_snapshot; const MergeTreeData & storage; const String partition_id; const ReadFromMergeTree::AnalysisResultPtr analysis_result_ptr; diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index f736c863eeed..1aa9f5e23f8f 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -268,4 +268,13 @@ bool MutationCommands::containBarrierCommand() const return false; } +NameSet MutationCommands::getAllUpdatedColumns() const +{ + NameSet res; + for (const auto & command : *this) + for (const auto & [column_name, _] : command.column_to_update_expression) + res.insert(column_name); + return res; +} + } diff --git a/src/Storages/MutationCommands.h b/src/Storages/MutationCommands.h index f999aab1f4dc..5ae537bb657e 100644 --- a/src/Storages/MutationCommands.h +++ b/src/Storages/MutationCommands.h @@ -92,6 +92,7 @@ class MutationCommands : public std::vector /// stick with other commands. Commands from one set have already been validated /// to be executed without issues on the creation state. bool containBarrierCommand() const; + NameSet getAllUpdatedColumns() const; }; using MutationCommandsConstPtr = std::shared_ptr; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 0827321e296c..f40aa8ae4e8e 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1625,9 +1625,11 @@ std::tuple StorageMerge::evaluateDatabaseName(cons return {false, ast}; } -bool StorageMerge::supportsTrivialCountOptimization(const StorageSnapshotPtr & storage_snapshot, ContextPtr ctx) const +bool StorageMerge::supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr ctx) const { - return getFirstTable([&](const auto & table) { return !table->supportsTrivialCountOptimization(storage_snapshot, ctx); }) == nullptr; + /// Here we actually need storage snapshot of all nested tables. + /// But to avoid complexity pass nullptr to make more lightweight check in MergeTreeData. + return getFirstTable([&](const auto & table) { return !table->supportsTrivialCountOptimization(nullptr, ctx); }) == nullptr; } std::optional StorageMerge::totalRows(const Settings & settings) const diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 78dbb72c199c..e2a279361a3b 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -512,21 +512,14 @@ Int64 StorageMergeTree::startMutation(const MutationCommands & commands, Context if (txn) txn->addMutation(shared_from_this(), mutation_id); - bool alter_conversions_mutations_updated = updateAlterConversionsMutations(entry.commands, alter_conversions_mutations, /* remove= */ false); - { std::lock_guard lock(currently_processing_in_background_mutex); - bool inserted = current_mutations_by_version.try_emplace(version, std::move(entry)).second; + auto [it, inserted] = current_mutations_by_version.try_emplace(version, std::move(entry)); if (!inserted) - { - if (alter_conversions_mutations_updated) - { - --alter_conversions_mutations; - chassert(alter_conversions_mutations >= 0); - } throw Exception(ErrorCodes::LOGICAL_ERROR, "Mutation {} already exists, it's a bug", version); - } + + incrementMutationsCounters(num_data_mutations_to_apply, num_metadata_mutations_to_apply, *it->second.commands); } LOG_INFO(log, "Added mutation: {}{}", mutation_id, additional_info); @@ -563,7 +556,7 @@ void StorageMergeTree::updateMutationEntriesErrors(FutureMergedMutatedPartPtr re if (static_cast(result_part->part_info.mutation) == it->first) mutation_backoff_policy.removePartFromFailed(failed_part->name); - updateAlterConversionsMutations(it->second.commands, alter_conversions_mutations, /* remove= */ true); + decrementMutationsCounters(num_data_mutations_to_apply, num_metadata_mutations_to_apply, *entry.commands); } } else @@ -763,17 +756,15 @@ std::map StorageMergeTree::getUnfinishedMutationC std::map result; - for (const auto & kv : current_mutations_by_version) + for (const auto & [mutation_version, entry] : current_mutations_by_version) { - Int64 mutation_version = kv.first; - const MergeTreeMutationEntry & entry = kv.second; - const PartVersionWithName needle{mutation_version, ""}; + const PartVersionWithName needle{static_cast(mutation_version), ""}; auto versions_it = std::lower_bound( part_versions_with_names.begin(), part_versions_with_names.end(), needle, comparator); size_t parts_to_do = versions_it - part_versions_with_names.begin(); if (parts_to_do > 0) - result.emplace(entry.file_name, entry.commands); + result.emplace(entry.file_name, *entry.commands); } return result; } @@ -806,7 +797,7 @@ std::vector StorageMergeTree::getMutationsStatus() cons std::map block_numbers_map({{"", entry.block_number}}); - for (const MutationCommand & command : entry.commands) + for (const MutationCommand & command : *entry.commands) { WriteBufferFromOwnString buf; formatAST(*command.ast, buf, false, true); @@ -843,20 +834,15 @@ CancellationCode StorageMergeTree::killMutation(const String & mutation_id) auto it = current_mutations_by_version.find(mutation_version); if (it != current_mutations_by_version.end()) { - bool mutation_finished = true; if (std::optional min_version = getMinPartDataVersion()) - mutation_finished = *min_version > static_cast(mutation_version); - - to_kill.emplace(std::move(it->second)); - - if (!mutation_finished) { - const auto commands = it->second.commands; - current_mutations_by_version.erase(it); - updateAlterConversionsMutations(commands, alter_conversions_mutations, /* remove= */ true); + bool mutation_finished = *min_version > static_cast(mutation_version); + if (!mutation_finished) + decrementMutationsCounters(num_data_mutations_to_apply, num_metadata_mutations_to_apply, *it->second.commands); } - else - current_mutations_by_version.erase(it); + + to_kill.emplace(std::move(it->second)); + current_mutations_by_version.erase(it); } } @@ -904,6 +890,8 @@ void StorageMergeTree::loadDeduplicationLog() void StorageMergeTree::loadMutations() { + std::lock_guard lock(currently_processing_in_background_mutex); + for (const auto & disk : getDisks()) { for (auto it = disk->iterateDirectory(relative_data_path); it->isValid(); it->next()) @@ -912,7 +900,7 @@ void StorageMergeTree::loadMutations() { MergeTreeMutationEntry entry(disk, relative_data_path, it->name()); UInt64 block_number = entry.block_number; - LOG_DEBUG(log, "Loading mutation: {} entry, commands size: {}", it->name(), entry.commands.size()); + LOG_DEBUG(log, "Loading mutation: {} entry, commands size: {}", it->name(), entry.commands->size()); if (!entry.tid.isPrehistoric() && !entry.csn) { @@ -931,10 +919,11 @@ void StorageMergeTree::loadMutations() } } - auto inserted = current_mutations_by_version.try_emplace(block_number, std::move(entry)).second; + auto [entry_it, inserted] = current_mutations_by_version.try_emplace(block_number, std::move(entry)); if (!inserted) throw Exception(ErrorCodes::LOGICAL_ERROR, "Mutation {} already exists, it's a bug", block_number); - updateAlterConversionsMutations(entry.commands, alter_conversions_mutations, /* remove= */ false); + + incrementMutationsCounters(num_data_mutations_to_apply, num_metadata_mutations_to_apply, *entry_it->second.commands); } else if (startsWith(it->name(), "tmp_mutation_")) { @@ -1282,7 +1271,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate( size_t commands_size = 0; MutationCommands commands_for_size_validation; - for (const auto & command : it->second.commands) + for (const auto & command : *it->second.commands) { if (command.type != MutationCommand::Type::DROP_COLUMN && command.type != MutationCommand::Type::DROP_INDEX @@ -1326,11 +1315,11 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate( const auto & single_mutation_commands = it->second.commands; - if (single_mutation_commands.containBarrierCommand()) + if (single_mutation_commands->containBarrierCommand()) { if (commands->empty()) { - commands->insert(commands->end(), single_mutation_commands.begin(), single_mutation_commands.end()); + commands->insert(commands->end(), single_mutation_commands->begin(), single_mutation_commands->end()); last_mutation_to_apply = it; } break; @@ -1338,7 +1327,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate( else { current_ast_elements += commands_size; - commands->insert(commands->end(), single_mutation_commands.begin(), single_mutation_commands.end()); + commands->insert(commands->end(), single_mutation_commands->begin(), single_mutation_commands->end()); last_mutation_to_apply = it; } @@ -2455,32 +2444,80 @@ void StorageMergeTree::attachRestoredParts(MutableDataPartsVector && parts) } } +MutationCommands StorageMergeTree::MutationsSnapshot::getAlterMutationCommandsForPart(const DataPartPtr & part) const +{ + MutationCommands result; + UInt64 part_data_version = part->info.getDataVersion(); + + for (const auto & [mutation_version, commands] : mutations_by_version | std::views::reverse) + { + if (mutation_version <= part_data_version) + break; -MutationCommands StorageMergeTree::getAlterMutationCommandsForPart(const DataPartPtr & part) const + for (const auto & command : *commands | std::views::reverse) + { + if (params.need_data_mutations && AlterConversions::isSupportedDataMutation(command.type)) + result.push_back(command); + else if (AlterConversions::isSupportedMetadataMutation(command.type)) + result.push_back(command); + } + } + + return result; +} + +NameSet StorageMergeTree::MutationsSnapshot::getAllUpdatedColumns() const { - /// NOTE: there is no need to check part metadata_version, since - /// ALTER_METADATA cannot be done asynchronously, like in - /// ReplicatedMergeTree. - chassert(alter_conversions_mutations >= 0); - if (alter_conversions_mutations == 0) + if (!hasDataMutations()) return {}; + NameSet res; + for (const auto & [version, commands] : mutations_by_version) + { + auto names = commands->getAllUpdatedColumns(); + std::move(names.begin(), names.end(), std::inserter(res, res.end())); + } + return res; +} + +MergeTreeData::MutationsSnapshotPtr StorageMergeTree::getMutationsSnapshot(const IMutationsSnapshot::Params & params) const +{ std::lock_guard lock(currently_processing_in_background_mutex); - UInt64 part_data_version = part->info.getDataVersion(); - MutationCommands result; + MutationsSnapshot::Info info + { + .num_data_mutations = num_data_mutations_to_apply, + .num_metadata_mutations = num_metadata_mutations_to_apply, + }; - for (const auto & [mutation_version, entry] : current_mutations_by_version | std::views::reverse) + auto res = std::make_shared(params, std::move(info)); + + bool need_data_mutations = res->hasDataMutations(); + bool need_metadata_mutations = num_metadata_mutations_to_apply > 0; + + if (!need_data_mutations && !need_metadata_mutations) + return res; + + for (const auto & [version, entry] : current_mutations_by_version) { - if (mutation_version <= part_data_version) - break; + bool has_required_command = std::ranges::any_of(*entry.commands, [&](const auto & command) + { + if (need_data_mutations && AlterConversions::isSupportedDataMutation(command.type)) + return true; - for (const auto & command : entry.commands | std::views::reverse) - if (AlterConversions::supportsMutationCommandType(command.type)) - result.emplace_back(command); + if (need_metadata_mutations && AlterConversions::isSupportedMetadataMutation(command.type)) + return true; + + return false; + }); + + /// Copy a pointer to all commands to avoid extracting and copying them. + /// Required commands will be copied later only for specific parts. + if (has_required_command) + res->mutations_by_version.emplace(version, entry.commands); } - return result; + return res; } void StorageMergeTree::startBackgroundMovesIfNeeded() diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 6223b5f50fae..ef333fe3f18b 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -147,8 +147,10 @@ class StorageMergeTree final : public MergeTreeData DataParts currently_merging_mutating_parts; std::map current_mutations_by_version; - /// Unfinished mutations that is required AlterConversions (see getAlterMutationCommandsForPart()) - std::atomic alter_conversions_mutations = 0; + + /// Unfinished mutations that are required for AlterConversions. + Int64 num_data_mutations_to_apply = 0; + Int64 num_metadata_mutations_to_apply = 0; std::atomic shutdown_called {false}; std::atomic flush_called {false}; @@ -308,9 +310,20 @@ class StorageMergeTree final : public MergeTreeData ContextPtr context; }; -protected: - /// Collect mutations that have to be applied on the fly: currently they are only RENAME COLUMN. - MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const override; + struct MutationsSnapshot : public IMutationsSnapshot + { + MutationsSnapshot() = default; + MutationsSnapshot(Params params_, Info info_) : IMutationsSnapshot(std::move(params_), std::move(info_)) {} + + using MutationsByVersion = std::map>; + MutationsByVersion mutations_by_version; + + MutationCommands getAlterMutationCommandsForPart(const MergeTreeData::DataPartPtr & part) const override; + std::shared_ptr cloneEmpty() const override { return std::make_shared(); } + NameSet getAllUpdatedColumns() const override; + }; + + MutationsSnapshotPtr getMutationsSnapshot(const IMutationsSnapshot::Params & params) const override; }; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 14f2163a8d35..171e5b757d66 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5709,7 +5709,8 @@ std::optional StorageReplicatedMergeTree::distributedWriteFromClu { auto connection = std::make_shared( node.host_name, node.port, query_context->getGlobalContext()->getCurrentDatabase(), - node.user, node.password, SSHKey(), /*jwt*/"", node.quota_key, node.cluster, node.cluster_secret, + node.user, node.password, node.proto_send_chunked, node.proto_recv_chunked, + SSHKey(), /*jwt*/"", node.quota_key, node.cluster, node.cluster_secret, "ParallelInsertSelectInititiator", node.compression, node.secure @@ -9211,13 +9212,11 @@ bool StorageReplicatedMergeTree::canUseAdaptiveGranularity() const (!has_non_adaptive_index_granularity_parts && !other_replicas_fixed_granularity)); } - -MutationCommands StorageReplicatedMergeTree::getAlterMutationCommandsForPart(const DataPartPtr & part) const +MergeTreeData::MutationsSnapshotPtr StorageReplicatedMergeTree::getMutationsSnapshot(const IMutationsSnapshot::Params & params) const { - return queue.getAlterMutationCommandsForPart(part); + return queue.getMutationsSnapshot(params); } - void StorageReplicatedMergeTree::startBackgroundMovesIfNeeded() { if (areBackgroundMovesNeeded()) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 4beb5acf0c2f..a8900a4d1d73 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -934,7 +934,7 @@ class StorageReplicatedMergeTree final : public MergeTreeData void waitMutationToFinishOnReplicas( const Strings & replicas, const String & mutation_id) const; - MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const override; + MutationsSnapshotPtr getMutationsSnapshot(const IMutationsSnapshot::Params & params) const override; void startBackgroundMovesIfNeeded() override; diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py index 8bc0f51dfc73..697c094b5525 100644 --- a/tests/ci/ast_fuzzer_check.py +++ b/tests/ci/ast_fuzzer_check.py @@ -16,7 +16,7 @@ from tee_popen import TeePopen from ci_config import CI -IMAGE_NAME = "clickhouse/fuzzer" +IMAGE_NAME = "altinityinfra/fuzzer" def get_run_command( diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 77d91c8400b0..384435311069 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -10,19 +10,20 @@ import docker_images_helper from ci_config import CI -from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH +from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY from git_helper import Git -from pr_info import PRInfo +from pr_info import PRInfo, EventType from report import FAILURE, SUCCESS, JobReport, StatusType from stopwatch import Stopwatch from tee_popen import TeePopen from version_helper import ( ClickHouseVersion, + VersionType, get_version_from_repo, update_version_local, ) -IMAGE_NAME = "clickhouse/binary-builder" +IMAGE_NAME = "altinityinfra/binary-builder" BUILD_LOG_NAME = "build_log.log" @@ -64,6 +65,8 @@ def get_packager_cmd( cmd += " --cache=sccache" cmd += " --s3-rw-access" cmd += f" --s3-bucket={S3_BUILDS_BUCKET}" + cmd += f" --s3-access-key-id={S3_ACCESS_KEY_ID}" + cmd += f" --s3-secret-access-key={S3_SECRET_ACCESS_KEY}" if build_config.additional_pkgs: cmd += " --additional-pkgs" @@ -162,16 +165,31 @@ def main(): version = get_version_from_repo(git=Git(True)) logging.info("Got version from repo %s", version.string) - official_flag = pr_info.number == 0 + # official_flag = pr_info.number == 0 - version_type = "testing" - if is_release_pr(pr_info): - version_type = "stable" - official_flag = True + # version_type = "testing" + # if is_release_pr(pr_info): + # version_type = "stable" + # official_flag = True + + # NOTE(vnemkov): For Altinity Stable builds, version flavor + # (last part of version, like 'altinitystable') is obtained from tag. + # If there is no tag, then version is considered to be 'testing' + version_type = version._flavour = VersionType.TESTING + official_flag = True + + if pr_info.event_type == EventType.PUSH \ + and pr_info.ref.startswith('/ref/tags/'): + tag_name = pr_info.ref.removeprefix('/ref/tags/') + version_type = tag_name.split('.')[-1] + version._flavour = version_type + logging.info("Using version from tag: %s => %s", tag_name, version) + + # TODO(vnemkov): make sure tweak part is incremented by 1 each time we merge a PR update_version_local(version, version_type) - logging.info("Updated local files with version") + logging.info("Updated local files with version %s", version) logging.info("Build short name %s", build_name) diff --git a/tests/ci/changelog.py b/tests/ci/changelog.py index 39e426945d3d..9bac4de4ee74 100755 --- a/tests/ci/changelog.py +++ b/tests/ci/changelog.py @@ -161,7 +161,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--repo", - default="ClickHouse/ClickHouse", + default="Altinity/ClickHouse", help="a repository to query for pull-requests from GitHub", ) parser.add_argument( diff --git a/tests/ci/ci.py b/tests/ci/ci.py index ea1fe872439a..465820a7488f 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -388,7 +388,7 @@ def _pre_action(s3, job_name, batch, indata, pr_info): _get_ext_check_name(job_name), ) ClickHouseHelper().insert_events_into( - db="default", table="checks", events=prepared_events + db="gh-data", table="checks", events=prepared_events ) print(f"Pre action done. Report files [{reports_files}] have been downloaded") @@ -966,7 +966,7 @@ def _add_build_to_version_history( print(f"::notice ::Log Adding record to versions history: {data}") - ch_helper.insert_event_into(db="default", table="version_history", event=data) + ch_helper.insert_event_into(db="gh-data", table="version_history", event=data) def _run_test(job_name: str, run_command: str) -> int: @@ -1302,7 +1302,7 @@ def main() -> int: job_report.check_name or _get_ext_check_name(args.job_name), ) ch_helper.insert_events_into( - db="default", table="checks", events=prepared_events + db="gh-data", table="checks", events=prepared_events ) if "DockerServerImage" in args.job_name and indata is not None: @@ -1376,7 +1376,7 @@ def main() -> int: _get_ext_check_name(args.job_name), ) ClickHouseHelper().insert_events_into( - db="default", table="checks", events=prepared_events + db="gh-data", table="checks", events=prepared_events ) ### POST action: end diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py index 164af72f4be7..ae64e2c44c21 100644 --- a/tests/ci/ci_buddy.py +++ b/tests/ci/ci_buddy.py @@ -104,7 +104,9 @@ def _get_webhooks(): return json_string def post(self, message: str, channels: List[str]) -> None: - print(f"Posting slack message, dry_run [{self.dry_run}]") + print(f"Would've posted slack message, dry_run [{self.dry_run}], message: {message}") + # NOTE(vnemkov): we don't use slack for CI/CD no need to post messages + return if self.dry_run: urls = [self.channels[Channels.DRY_RUN]] else: diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 29c74a9f6ccf..19679441d0d1 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -365,15 +365,12 @@ class CI: ), JobNames.STRESS_TEST_ASAN: CommonJobConfigs.STRESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN], - random_bucket="stress_with_sanitizer", ), JobNames.STRESS_TEST_UBSAN: CommonJobConfigs.STRESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_UBSAN], - random_bucket="stress_with_sanitizer", ), JobNames.STRESS_TEST_MSAN: CommonJobConfigs.STRESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_MSAN], - random_bucket="stress_with_sanitizer", ), JobNames.STRESS_TEST_AZURE_TSAN: CommonJobConfigs.STRESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_TSAN], release_only=True @@ -400,7 +397,7 @@ class CI: required_builds=[BuildNames.PACKAGE_DEBUG], pr_only=True ), JobNames.INTEGRATION_TEST_ASAN: CommonJobConfigs.INTEGRATION_TEST.with_properties( - required_builds=[BuildNames.PACKAGE_ASAN], release_only=True, num_batches=4 + required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4 ), JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN], @@ -419,7 +416,7 @@ class CI: JobNames.INTEGRATION_TEST: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=4, - release_only=True, + #release_only=True, ), JobNames.INTEGRATION_TEST_FLAKY: CommonJobConfigs.INTEGRATION_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN], @@ -469,7 +466,7 @@ class CI: JobNames.STATELESS_TEST_FLAKY_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN], pr_only=True, - timeout=3600, + timeout=3 * 3600, # TODO: approach with reference job names does not work because digest may not be calculated if job skipped in wf # reference_job_name=JobNames.STATELESS_TEST_RELEASE, ), @@ -530,7 +527,7 @@ class CI: JobNames.DOCS_CHECK: JobConfig( digest=DigestConfig( include_paths=["**/*.md", "./docs", "tests/ci/docs_check.py"], - docker=["clickhouse/docs-builder"], + docker=["altinityinfra/docs-builder"], ), run_command="docs_check.py", runner_type=Runners.FUNC_TESTER, @@ -540,7 +537,7 @@ class CI: digest=DigestConfig( include_paths=["./tests/queries/0_stateless/"], exclude_files=[".md"], - docker=["clickhouse/fasttest"], + docker=["altinityinfra/fasttest"], ), timeout=2400, runner_type=Runners.BUILDER, @@ -555,6 +552,14 @@ class CI: timeout=900, runner_type=Runners.STYLE_CHECKER, ), + JobNames.SIGN_RELEASE: JobConfig( + required_builds=[BuildNames.PACKAGE_RELEASE], + runner_type=Runners.STYLE_CHECKER + ), + JobNames.SIGN_AARCH64: JobConfig( + required_builds=[BuildNames.PACKAGE_AARCH64], + runner_type=Runners.STYLE_CHECKER_ARM + ), } @classmethod diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py index 70aba2bc16b0..c10aec14b06b 100644 --- a/tests/ci/ci_definitions.py +++ b/tests/ci/ci_definitions.py @@ -223,6 +223,9 @@ class JobNames(metaclass=WithIter): DOCS_CHECK = "Docs check" BUGFIX_VALIDATE = "Bugfix validation" + SIGN_RELEASE = "Sign release" + SIGN_AARCH64 = "Sign aarch64" + # hack to concatenate Build and non-build jobs under JobNames class for attr_name in dir(BuildNames): @@ -338,7 +341,7 @@ class JobConfig: # label that enables job in CI, if set digest isn't used run_by_label: str = "" # to run always regardless of the job digest or/and label - run_always: bool = False + run_always: bool = True # disables CI await for a given job disable_await: bool = False # if the job needs to be run on the release branch, including master (building packages, docker server). @@ -391,7 +394,7 @@ class CommonJobConfigs: job_name_keyword="compatibility", digest=DigestConfig( include_paths=["./tests/ci/compatibility_check.py"], - docker=["clickhouse/test-old-ubuntu", "clickhouse/test-old-centos"], + docker=["altinityinfra/test-old-ubuntu", "altinityinfra/test-old-centos"], ), run_command="compatibility_check.py", runner_type=Runners.STYLE_CHECKER, @@ -400,7 +403,7 @@ class CommonJobConfigs: job_name_keyword="install", digest=DigestConfig( include_paths=["./tests/ci/install_check.py"], - docker=["clickhouse/install-deb-test", "clickhouse/install-rpm-test"], + docker=["altinityinfra/install-deb-test", "altinityinfra/install-rpm-test"], ), run_command='install_check.py "$CHECK_NAME"', runner_type=Runners.STYLE_CHECKER, @@ -417,7 +420,7 @@ class CommonJobConfigs: "./tests/*.txt", ], exclude_files=[".md"], - docker=["clickhouse/stateless-test"], + docker=["altinityinfra/stateless-test"], ), run_command='functional_test_check.py "$CHECK_NAME"', runner_type=Runners.FUNC_TESTER, @@ -433,7 +436,7 @@ class CommonJobConfigs: "./tests/*.txt", ], exclude_files=[".md"], - docker=["clickhouse/stateful-test"], + docker=["altinityinfra/stateful-test"], ), run_command='functional_test_check.py "$CHECK_NAME"', runner_type=Runners.FUNC_TESTER, @@ -450,7 +453,7 @@ class CommonJobConfigs: "./tests/*.txt", ], exclude_files=[".md"], - docker=["clickhouse/stress-test"], + docker=["altinityinfra/stress-test"], ), run_command="stress_check.py", runner_type=Runners.STRESS_TESTER, @@ -461,7 +464,7 @@ class CommonJobConfigs: digest=DigestConfig( include_paths=["./tests/ci/upgrade_check.py"], exclude_files=[".md"], - docker=["clickhouse/upgrade-check"], + docker=["altinityinfra/upgrade-check"], ), run_command="upgrade_check.py", runner_type=Runners.STRESS_TESTER, @@ -487,7 +490,7 @@ class CommonJobConfigs: include_paths=[ "./tests/ci/ast_fuzzer_check.py", ], - docker=["clickhouse/fuzzer"], + docker=["altinityinfra/fuzzer"], ), run_command="ast_fuzzer_check.py", run_always=True, @@ -498,7 +501,7 @@ class CommonJobConfigs: digest=DigestConfig( include_paths=["./tests/ci/unit_tests_check.py"], exclude_files=[".md"], - docker=["clickhouse/unit-test"], + docker=["altinityinfra/unit-test"], ), run_command="unit_tests_check.py", runner_type=Runners.FUZZER_UNIT_TESTER, @@ -511,7 +514,7 @@ class CommonJobConfigs: "./tests/performance/", ], exclude_files=[".md"], - docker=["clickhouse/performance-comparison"], + docker=["altinityinfra/performance-comparison"], ), run_command="performance_comparison_check.py", runner_type=Runners.STRESS_TESTER, @@ -529,7 +532,7 @@ class CommonJobConfigs: digest=DigestConfig( include_paths=["./tests/ci/sqllogic_test.py"], exclude_files=[".md"], - docker=["clickhouse/sqllogic-test"], + docker=["altinityinfra/sqllogic-test"], ), run_command="sqllogic_test.py", timeout=10800, @@ -541,7 +544,7 @@ class CommonJobConfigs: digest=DigestConfig( include_paths=["./tests/ci/sqltest.py"], exclude_files=[".md"], - docker=["clickhouse/sqltest"], + docker=["altinityinfra/sqltest"], ), run_command="sqltest.py", timeout=10800, @@ -558,7 +561,7 @@ class CommonJobConfigs: DOCKER_SERVER = JobConfig( job_name_keyword="docker", required_on_release_branch=True, - run_command='docker_server.py --check-name "$CHECK_NAME" --tag-type head --allow-build-reuse', + run_command='docker_server.py --check-name "$CHECK_NAME" --tag-type head --allow-build-reuse --push', digest=DigestConfig( include_paths=[ "tests/ci/docker_server.py", @@ -574,7 +577,7 @@ class CommonJobConfigs: include_paths=[ "tests/ci/clickbench.py", ], - docker=["clickhouse/clickbench"], + docker=["altinityinfra/clickbench"], ), run_command='clickbench.py "$CHECK_NAME"', timeout=900, @@ -611,7 +614,7 @@ class CommonJobConfigs: "./tests/performance", ], exclude_files=[".md"], - docker=["clickhouse/binary-builder"], + docker=["altinityinfra/binary-builder"], git_submodules=True, ), run_command="build_check.py $BUILD_NAME", diff --git a/tests/ci/ci_settings.py b/tests/ci/ci_settings.py index 05929179e06e..64de92e289a5 100644 --- a/tests/ci/ci_settings.py +++ b/tests/ci/ci_settings.py @@ -48,7 +48,8 @@ def create_from_pr_message( res = CiSettings() pr_info = PRInfo() if ( - not pr_info.is_pr and not debug_message + (not pr_info.is_pr and not debug_message) + or pr_info.body is None ): # if commit_message is provided it's test/debug scenario - do not return # CI options can be configured in PRs only # if debug_message is provided - it's a test diff --git a/tests/ci/clickbench.py b/tests/ci/clickbench.py index a1988abb1f54..3d995289c530 100644 --- a/tests/ci/clickbench.py +++ b/tests/ci/clickbench.py @@ -22,7 +22,7 @@ def get_image_name() -> str: - return "clickhouse/clickbench" + return "altinityinfra/clickbench" def get_run_command( diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index ef93a07e68cd..029a3f539d6d 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -212,11 +212,11 @@ def prepare_tests_results_for_clickhouse( report_url: str, check_name: str, ) -> List[dict]: - base_ref = pr_info.base_ref - base_repo = pr_info.base_name - head_ref = pr_info.head_ref - head_repo = pr_info.head_name - pull_request_url = f"https://github.com/{GITHUB_REPOSITORY}/commits/{head_ref}" + pull_request_url = "https://github.com/Altinity/ClickHouse/commits/master" + base_ref = "master" + head_ref = "master" + base_repo = pr_info.repo_full_name + head_repo = pr_info.repo_full_name if pr_info.number != 0: pull_request_url = pr_info.pr_html_url diff --git a/tests/ci/compatibility_check.py b/tests/ci/compatibility_check.py index bb0c717160e2..3b161bdcc88e 100644 --- a/tests/ci/compatibility_check.py +++ b/tests/ci/compatibility_check.py @@ -16,8 +16,8 @@ from report import FAILURE, SUCCESS, JobReport, TestResult, TestResults from stopwatch import Stopwatch -IMAGE_UBUNTU = "clickhouse/test-old-ubuntu" -IMAGE_CENTOS = "clickhouse/test-old-centos" +IMAGE_UBUNTU = "altinityinfra/test-old-ubuntu" +IMAGE_CENTOS = "altinityinfra/test-old-centos" DOWNLOAD_RETRIES_COUNT = 5 diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 49d2b63b87ca..c401f218df56 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -275,7 +275,7 @@ def main(): NAME, ) ch_helper = ClickHouseHelper() - ch_helper.insert_events_into(db="default", table="checks", events=prepared_events) + ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) if status == FAILURE: sys.exit(1) diff --git a/tests/ci/docker_images_helper.py b/tests/ci/docker_images_helper.py index f0323145cfa1..9faefef12c24 100644 --- a/tests/ci/docker_images_helper.py +++ b/tests/ci/docker_images_helper.py @@ -20,9 +20,9 @@ def docker_login(relogin: bool = True) -> None: "docker system info | grep --quiet -E 'Username|Registry'" ): Shell.check( # pylint: disable=unexpected-keyword-arg - "docker login --username 'robotclickhouse' --password-stdin", + "docker login --username 'altinityinfra' --password-stdin", strict=True, - stdin_str=get_parameter_from_ssm("dockerhub_robot_password"), + stdin_str=get_parameter_from_ssm("dockerhub-password"), encoding="utf-8", ) diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index 772cbbf1b024..914e93935deb 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -212,7 +212,7 @@ def main(): NAME, ) ch_helper = ClickHouseHelper() - ch_helper.insert_events_into(db="default", table="checks", events=prepared_events) + ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) if status == FAILURE: sys.exit(1) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 3251ec5644e0..ca2732e0af78 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -58,7 +58,7 @@ def parse_args() -> argparse.Namespace: "--version", type=version_arg, default=get_version_from_repo(git=git).string, - help="a version to build, automaticaly got from version_helper, accepts either " + help="a version to build, automatically got from version_helper, accepts either " "tag ('refs/tags/' is removed automatically) or a normal 22.2.2.2 format", ) parser.add_argument( @@ -214,7 +214,7 @@ def build_and_push_image( init_args = ["docker", "buildx", "build"] if push: init_args.append("--push") - init_args.append("--output=type=image,push-by-digest=true") + init_args.append("--output=type=image") init_args.append(f"--tag={image.repo}") else: init_args.append("--output=type=docker") @@ -311,10 +311,10 @@ def main(): assert not args.image_path and not args.image_repo if "server image" in args.check_name: image_path = "docker/server" - image_repo = "clickhouse/clickhouse-server" + image_repo = "altinityinfra/clickhouse-server" elif "keeper image" in args.check_name: image_path = "docker/keeper" - image_repo = "clickhouse/clickhouse-keeper" + image_repo = "altinityinfra/clickhouse-keeper" else: assert False, "Invalid --check-name" else: @@ -332,6 +332,8 @@ def main(): image = DockerImageData(image_path, image_repo, False) tags = gen_tags(args.version, args.tag_type) + tags.append(f'{pr_info.number}-{args.version}') + repo_urls = {} direct_urls: Dict[str, List[str]] = {} diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py index 6bd4ef496755..b850c28ab2b1 100644 --- a/tests/ci/docs_check.py +++ b/tests/ci/docs_check.py @@ -61,7 +61,7 @@ def main(): elif args.force: logging.info("Check the docs because of force flag") - docker_image = pull_image(get_docker_image("clickhouse/docs-builder")) + docker_image = pull_image(get_docker_image("altinityinfra/docs-builder")) test_output = temp_path / "docs_check_log" test_output.mkdir(parents=True, exist_ok=True) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 5217e4035da1..cad781a17445 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -20,20 +20,24 @@ CLOUDFLARE_TOKEN = os.getenv("CLOUDFLARE_TOKEN") GITHUB_EVENT_PATH = os.getenv("GITHUB_EVENT_PATH", "") GITHUB_JOB = os.getenv("GITHUB_JOB_OVERRIDDEN", "") or os.getenv("GITHUB_JOB", "local") -GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse") +GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "Altinity/ClickHouse") GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID", "0") GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") GITHUB_UPSTREAM_REPOSITORY = os.getenv( - "GITHUB_UPSTREAM_REPOSITORY", "ClickHouse/ClickHouse" + "GITHUB_UPSTREAM_REPOSITORY", "Altinity/ClickHouse" ) GITHUB_WORKSPACE = os.getenv("GITHUB_WORKSPACE", git_root) GITHUB_RUN_URL = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}" IMAGES_PATH = os.getenv("IMAGES_PATH", TEMP_PATH) REPO_COPY = os.getenv("REPO_COPY", GITHUB_WORKSPACE) RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp"))) -S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "clickhouse-builds") -S3_BUILDS_BUCKET_PUBLIC = "clickhouse-builds" -S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "clickhouse-test-reports") + +S3_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") +S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "altinity-build-artifacts") +S3_BUILDS_BUCKET_PUBLIC = "altinity-build-artifacts" +S3_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") +S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-build-artifacts") + S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") S3_DOWNLOAD = os.getenv("S3_DOWNLOAD", S3_URL) S3_ARTIFACT_DOWNLOAD_TEMPLATE = ( diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index ed727dd3659f..3e725d8c6a9b 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -80,7 +80,7 @@ def main(): pr_info = PRInfo() - docker_image = pull_image(get_docker_image("clickhouse/fasttest")) + docker_image = pull_image(get_docker_image("altinityinfra/fasttest")) workspace = temp_path / "fasttest-workspace" workspace.mkdir(parents=True, exist_ok=True) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index d08f98fa05fe..0fd8ba4dd638 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -16,7 +16,6 @@ from docker_images_helper import DockerImage, get_docker_image, pull_image from download_release_packages import download_last_release from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH -from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import ( ERROR, @@ -52,8 +51,9 @@ def get_additional_envs( check_name: str, run_by_hash_num: int, run_by_hash_total: int ) -> List[str]: result = [] - azure_connection_string = get_parameter_from_ssm("azure_connection_string") - result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") + # TODO(vnemkov): put proper Azure connection string into SSM and re-enable this one + # azure_connection_string = get_parameter_from_ssm("azure_connection_string") + # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") if "DatabaseReplicated" in check_name: result.append("USE_DATABASE_REPLICATED=1") if "DatabaseOrdinary" in check_name: @@ -67,9 +67,6 @@ def get_additional_envs( result.append("RANDOMIZE_OBJECT_KEY_TYPE=1") if "analyzer" in check_name: result.append("USE_OLD_ANALYZER=1") - if "azure" in check_name: - assert "USE_S3_STORAGE_FOR_MERGE_TREE=1" not in result - result.append("USE_AZURE_STORAGE_FOR_MERGE_TREE=1") if run_by_hash_total != 0: result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}") @@ -80,9 +77,9 @@ def get_additional_envs( def get_image_name(check_name: str) -> str: if "stateless" in check_name.lower(): - return "clickhouse/stateless-test" + return "altinityinfra/stateless-test" if "stateful" in check_name.lower(): - return "clickhouse/stateful-test" + return "altinityinfra/stateful-test" raise ValueError(f"Cannot deduce image name based on check name {check_name}") diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index a4317d5caff5..a218202fd76e 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -59,8 +59,20 @@ def get_parameters_from_ssm( ROBOT_TOKEN = None # type: Optional[Token] +# NOTE(Arthur Passos): Original CI code uses the "_original" version of this method. Each robot token is rate limited +# and the original implementation selects the "best one". To make it simpler and iterate faster, +# we are using only one robot and keeping the method signature. In the future we might reconsider +# having multiple robot tokens +def get_best_robot_token(token_prefix_env_name="github_robot_token"): + # Re-use already fetched token (same as in get_best_robot_token_original) + # except here we assume it is always a string (since we use only one token and don't do token rotation) + global ROBOT_TOKEN + if ROBOT_TOKEN is not None: + return ROBOT_TOKEN + ROBOT_TOKEN = get_parameter_from_ssm(token_prefix_env_name) + return ROBOT_TOKEN -def get_best_robot_token(tokens_path: str = "/github-tokens") -> str: +def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: global ROBOT_TOKEN if ROBOT_TOKEN is not None: return ROBOT_TOKEN.value diff --git a/tests/ci/git_helper.py b/tests/ci/git_helper.py index 6b66bc44d100..ca924a378a45 100644 --- a/tests/ci/git_helper.py +++ b/tests/ci/git_helper.py @@ -11,13 +11,29 @@ logger = logging.getLogger(__name__) +class VersionType: + LTS = "lts" + NEW = "new" + PRESTABLE = "altinityedge" + STABLE = "altinitystable" + TESTING = "altinitytest" + + VALID = (NEW, TESTING, PRESTABLE, STABLE, LTS, + # NOTE (vnemkov): we don't use those directly, but it is used in unit-tests + "stable", + "prestable", + "testing", + ) + # ^ and $ match subline in `multiple\nlines` # \A and \Z match only start and end of the whole string +# NOTE (vnemkov): support both upstream tag style: v22.x.y.z-lts and Altinity tag style: v22.x.y.z.altinitystable +# Because at early release stages there could be no Altinity tag set on commit, only upstream one. RELEASE_BRANCH_REGEXP = r"\A\d+[.]\d+\Z" TAG_REGEXP = ( r"\Av\d{2}" # First two digits of major part r"([.][1-9]\d*){3}" # minor.patch.tweak parts - r"-(new|testing|prestable|stable|lts)\Z" # suffix with a version type + fr"[.-]({'|'.join(VersionType.VALID)})\Z" # suffix with a version type ) SHA_REGEXP = re.compile(r"\A([0-9]|[a-f]){40}\Z") @@ -26,9 +42,9 @@ with tempfile.NamedTemporaryFile("w", delete=False) as f: GIT_KNOWN_HOSTS_FILE = f.name - GIT_PREFIX = ( # All commits to remote are done as robot-clickhouse - "git -c user.email=robot-clickhouse@users.noreply.github.com " - "-c user.name=robot-clickhouse -c commit.gpgsign=false " + GIT_PREFIX = ( # All commits to remote are done as altinity-robot + "git -c user.email=altinity-robot@users.noreply.github.com " + "-c user.name=altinity-robot -c commit.gpgsign=false " "-c core.sshCommand=" f"'ssh -o UserKnownHostsFile={GIT_KNOWN_HOSTS_FILE} " "-o StrictHostKeyChecking=accept-new'" diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index 6c33b1f20660..af556285ecb4 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -17,8 +17,8 @@ from stopwatch import Stopwatch from tee_popen import TeePopen -RPM_IMAGE = "clickhouse/install-rpm-test" -DEB_IMAGE = "clickhouse/install-deb-test" +RPM_IMAGE = "altinityinfra/install-rpm-test" +DEB_IMAGE = "altinityinfra/install-deb-test" TEMP_PATH = Path(TEMP) LOGS_PATH = TEMP_PATH / "tests_logs" diff --git a/tests/ci/integration_test_images.py b/tests/ci/integration_test_images.py index 8148ac611817..d51998cc1549 100644 --- a/tests/ci/integration_test_images.py +++ b/tests/ci/integration_test_images.py @@ -1,19 +1,19 @@ #!/usr/bin/env python3 IMAGES_ENV = { - "clickhouse/dotnet-client": "DOCKER_DOTNET_CLIENT_TAG", - "clickhouse/integration-helper": "DOCKER_HELPER_TAG", - "clickhouse/integration-test": "DOCKER_BASE_TAG", - "clickhouse/integration-tests-runner": "", - "clickhouse/kerberized-hadoop": "DOCKER_KERBERIZED_HADOOP_TAG", - "clickhouse/kerberos-kdc": "DOCKER_KERBEROS_KDC_TAG", - "clickhouse/mysql-golang-client": "DOCKER_MYSQL_GOLANG_CLIENT_TAG", - "clickhouse/mysql-java-client": "DOCKER_MYSQL_JAVA_CLIENT_TAG", - "clickhouse/mysql-js-client": "DOCKER_MYSQL_JS_CLIENT_TAG", - "clickhouse/mysql-php-client": "DOCKER_MYSQL_PHP_CLIENT_TAG", - "clickhouse/nginx-dav": "DOCKER_NGINX_DAV_TAG", - "clickhouse/postgresql-java-client": "DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", - "clickhouse/python-bottle": "DOCKER_PYTHON_BOTTLE_TAG", + "altinityinfra/dotnet-client": "DOCKER_DOTNET_CLIENT_TAG", + "altinityinfra/integration-helper": "DOCKER_HELPER_TAG", + "altinityinfra/integration-test": "DOCKER_BASE_TAG", + "altinityinfra/integration-tests-runner": "", + # "altinityinfra/kerberized-hadoop": "DOCKER_KERBERIZED_HADOOP_TAG", + "altinityinfra/kerberos-kdc": "DOCKER_KERBEROS_KDC_TAG", + "altinityinfra/mysql-golang-client": "DOCKER_MYSQL_GOLANG_CLIENT_TAG", + "altinityinfra/mysql-java-client": "DOCKER_MYSQL_JAVA_CLIENT_TAG", + "altinityinfra/mysql-js-client": "DOCKER_MYSQL_JS_CLIENT_TAG", + "altinityinfra/mysql-php-client": "DOCKER_MYSQL_PHP_CLIENT_TAG", + "altinityinfra/nginx-dav": "DOCKER_NGINX_DAV_TAG", + "altinityinfra/postgresql-java-client": "DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", + "altinityinfra/python-bottle": "DOCKER_PYTHON_BOTTLE_TAG", } IMAGES = list(IMAGES_ENV.keys()) diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py index 6023ad8f0762..476c7b149c20 100755 --- a/tests/ci/integration_tests_runner.py +++ b/tests/ci/integration_tests_runner.py @@ -26,8 +26,8 @@ from stopwatch import Stopwatch from tee_popen import TeePopen -MAX_RETRY = 1 -NUM_WORKERS = 5 +MAX_RETRY = 3 +NUM_WORKERS = 10 SLEEP_BETWEEN_RETRIES = 5 PARALLEL_GROUP_SIZE = 100 CLICKHOUSE_BINARY_PATH = "usr/bin/clickhouse" @@ -318,7 +318,7 @@ def _pre_pull_images(self, repo_path): cmd = ( f"cd {repo_path}/tests/integration && " - f"timeout --signal=KILL 1h ./runner {self._get_runner_opts()} {image_cmd} " + f"timeout --signal=KILL 2h ./runner {self._get_runner_opts()} {image_cmd} " "--pre-pull --command ' echo Pre Pull finished ' " ) @@ -428,7 +428,7 @@ def _get_all_tests(self, repo_path): out_file_full = os.path.join(self.result_path, "runner_get_all_tests.log") cmd = ( f"cd {repo_path}/tests/integration && " - f"timeout --signal=KILL 1h ./runner {runner_opts} {image_cmd} -- --setup-plan " + f"timeout --signal=KILL 2h ./runner {runner_opts} {image_cmd} -- --setup-plan " ) logging.info( @@ -515,7 +515,7 @@ def _get_runner_image_cmd(self, repo_path): "--docker-image-version", ): for img in IMAGES: - if img == "clickhouse/integration-tests-runner": + if img == "altinityinfra/integration-tests-runner": runner_version = self.get_image_version(img) logging.info( "Can run with custom docker image version %s", runner_version @@ -645,7 +645,8 @@ def run_test_group( info_path = os.path.join(repo_path, "tests/integration", info_basename) test_cmd = " ".join([shlex.quote(test) for test in sorted(test_names)]) - parallel_cmd = f" --parallel {num_workers} " if num_workers > 0 else "" + # run in parallel only the first time, re-runs are sequential to give chance to flappy tests to pass. + parallel_cmd = f" --parallel {num_workers} " if num_workers > 0 and i == 0 else "" repeat_cmd = f" --count {repeat_count} " if repeat_count > 0 else "" # -r -- show extra test summary: # -f -- (f)ailed diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py index 772467d42459..85f33374e8a4 100644 --- a/tests/ci/jepsen_check.py +++ b/tests/ci/jepsen_check.py @@ -28,10 +28,10 @@ KEEPER_DESIRED_INSTANCE_COUNT = 3 SERVER_DESIRED_INSTANCE_COUNT = 4 -KEEPER_IMAGE_NAME = "clickhouse/keeper-jepsen-test" +KEEPER_IMAGE_NAME = "altinityinfra/keeper-jepsen-test" KEEPER_CHECK_NAME = CI.JobNames.JEPSEN_KEEPER -SERVER_IMAGE_NAME = "clickhouse/server-jepsen-test" +SERVER_IMAGE_NAME = "altinityinfra/server-jepsen-test" SERVER_CHECK_NAME = CI.JobNames.JEPSEN_SERVER SUCCESSFUL_TESTS_ANCHOR = "# Successful tests" diff --git a/tests/ci/libfuzzer_test_check.py b/tests/ci/libfuzzer_test_check.py index 8f19dd7d0234..984534682224 100644 --- a/tests/ci/libfuzzer_test_check.py +++ b/tests/ci/libfuzzer_test_check.py @@ -109,7 +109,7 @@ def main(): run_by_hash_num = 0 run_by_hash_total = 0 - docker_image = pull_image(get_docker_image("clickhouse/libfuzzer")) + docker_image = pull_image(get_docker_image("altinityinfra/libfuzzer")) fuzzers_path = temp_path / "fuzzers" fuzzers_path.mkdir(parents=True, exist_ok=True) diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index b3488ac0af21..c677ac92bc75 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -31,7 +31,7 @@ from stopwatch import Stopwatch from tee_popen import TeePopen -IMAGE_NAME = "clickhouse/performance-comparison" +IMAGE_NAME = "altinityinfra/performance-comparison" def get_run_command( @@ -100,7 +100,7 @@ def main(): if pr_info.number == 0: pr_link = commit.html_url else: - pr_link = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_info.number}" + pr_link = f"https://github.com/Altinity/ClickHouse/pull/{pr_info.number}" docker_env += ( f' -e CHPC_ADD_REPORT_LINKS="' @@ -188,7 +188,7 @@ def main(): def too_many_slow(msg): match = re.search(r"(|.* )(\d+) slower.*", msg) # This threshold should be synchronized with the value in - # https://github.com/ClickHouse/ClickHouse/blob/master/docker/test/performance-comparison/report.py#L629 + # https://github.com/Altinity/ClickHouse/blob/master/docker/test/performance-comparison/report.py#L629 threshold = 5 return int(match.group(2).strip()) > threshold if match else False diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 09a8cb563a1f..f97ea05e14da 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -80,8 +80,12 @@ def get_pr_for_commit(sha, ref): ref, sha, ) - first_pr = our_prs[0] - return first_pr + if len(our_prs) != 0: + first_pr = our_prs[0] + return first_pr + else: + return None + except Exception as ex: logging.error( "Cannot fetch PR info from commit ref %s, sha %s, exception: %s", @@ -132,8 +136,9 @@ def __init__( ref = github_event.get("ref", "refs/heads/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] + self.ref = ref # type: str e.g. "refs/pull/509/merge" or "refs/tags/v24.3.12.76.altinitystable" # Default values - self.base_ref = "" # type: str + self.base_ref = github_event.get("base_ref","") # type: str self.base_name = "" # type: str self.head_ref = "" # type: str self.head_name = "" # type: str diff --git a/tests/ci/release.py b/tests/ci/release.py index b26d6205f3b2..20a4eff61300 100755 --- a/tests/ci/release.py +++ b/tests/ci/release.py @@ -603,7 +603,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--repo", - default="ClickHouse/ClickHouse", + default="Altinity/ClickHouse", help="repository to create the release", ) parser.add_argument( diff --git a/tests/ci/sign_release.py b/tests/ci/sign_release.py new file mode 100644 index 000000000000..8a5827097c8b --- /dev/null +++ b/tests/ci/sign_release.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import sys +import os +import logging +from env_helper import TEMP_PATH, REPO_COPY, REPORT_PATH +from s3_helper import S3Helper +from pr_info import PRInfo +from build_download_helper import download_builds_filter +import hashlib +from pathlib import Path + +GPG_BINARY_SIGNING_KEY = os.getenv("GPG_BINARY_SIGNING_KEY") +GPG_BINARY_SIGNING_PASSPHRASE = os.getenv("GPG_BINARY_SIGNING_PASSPHRASE") + +CHECK_NAME = os.getenv("CHECK_NAME", "Sign release") + +def hash_file(file_path): + BLOCK_SIZE = 65536 # The size of each read from the file + + file_hash = hashlib.sha256() # Create the hash object, can use something other than `.sha256()` if you wish + with open(file_path, 'rb') as f: # Open the file to read it's bytes + fb = f.read(BLOCK_SIZE) # Read from the file. Take in the amount declared above + while len(fb) > 0: # While there is still data being read from the file + file_hash.update(fb) # Update the hash + fb = f.read(BLOCK_SIZE) # Read the next block from the file + + hash_file_path = file_path + '.sha256' + with open(hash_file_path, 'x') as f: + digest = file_hash.hexdigest() + f.write(digest) + print(f'Hashed {file_path}: {digest}') + + return hash_file_path + +def sign_file(file_path): + priv_key_file_path = 'priv.key' + with open(priv_key_file_path, 'x') as f: + f.write(GPG_BINARY_SIGNING_KEY) + + out_file_path = f'{file_path}.gpg' + + os.system(f'echo {GPG_BINARY_SIGNING_PASSPHRASE} | gpg --batch --import {priv_key_file_path}') + os.system(f'gpg -o {out_file_path} --pinentry-mode=loopback --batch --yes --passphrase {GPG_BINARY_SIGNING_PASSPHRASE} --sign {file_path}') + print(f"Signed {file_path}") + os.remove(priv_key_file_path) + + return out_file_path + +def main(): + reports_path = Path(REPORT_PATH) + + if not os.path.exists(TEMP_PATH): + os.makedirs(TEMP_PATH) + + pr_info = PRInfo() + + logging.info("Repo copy path %s", REPO_COPY) + + s3_helper = S3Helper() + + s3_path_prefix = Path(f"{pr_info.number}/{pr_info.sha}/" + CHECK_NAME.lower().replace( + " ", "_" + ).replace("(", "_").replace(")", "_").replace(",", "_")) + + # downloads `package_release` artifacts generated + download_builds_filter(CHECK_NAME, reports_path, Path(TEMP_PATH)) + + for f in os.listdir(TEMP_PATH): + full_path = os.path.join(TEMP_PATH, f) + if os.path.isdir(full_path): + continue + hashed_file_path = hash_file(full_path) + signed_file_path = sign_file(hashed_file_path) + s3_path = s3_path_prefix / os.path.basename(signed_file_path) + s3_helper.upload_build_file_to_s3(Path(signed_file_path), str(s3_path)) + print(f'Uploaded file {signed_file_path} to {s3_path}') + + # Signed hashes are: + # clickhouse-client_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-keeper_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-client-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-keeper-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-client_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-keeper-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg + # clickhouse-client-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-keeper-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg + # clickhouse-common-static_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-keeper-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-common-static-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-keeper-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-common-static_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-keeper.sha512.gpg + # clickhouse-common-static-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-library-bridge.sha512.gpg + # clickhouse-common-static-dbg_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-odbc-bridge.sha512.gpg + # clickhouse-common-static-dbg-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse-server_22.3.15.2.altinitystable_amd64.deb.sha512.gpg + # clickhouse-common-static-dbg_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg clickhouse-server-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg + # clickhouse-common-static-dbg-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg clickhouse-server_22.3.15.2.altinitystable_x86_64.apk.sha512.gpg + # clickhouse-keeper_22.3.15.2.altinitystable_amd64.deb.sha512.gpg clickhouse-server-22.3.15.2.altinitystable.x86_64.rpm.sha512.gpg + # clickhouse-keeper-22.3.15.2.altinitystable-amd64.tgz.sha512.gpg clickhouse.sha512.gpg + + sys.exit(0) + +if __name__ == "__main__": + main() diff --git a/tests/ci/sqlancer_check.py b/tests/ci/sqlancer_check.py index a68db8b97919..1c173eb55519 100644 --- a/tests/ci/sqlancer_check.py +++ b/tests/ci/sqlancer_check.py @@ -14,7 +14,7 @@ from tee_popen import TeePopen from ci_config import CI -IMAGE_NAME = "clickhouse/sqlancer-test" +IMAGE_NAME = "altinityinfra/sqlancer-test" def get_run_command(download_url: str, workspace_path: Path, image: DockerImage) -> str: diff --git a/tests/ci/sqllogic_test.py b/tests/ci/sqllogic_test.py index 7fe44c235c7b..27ab9441826b 100755 --- a/tests/ci/sqllogic_test.py +++ b/tests/ci/sqllogic_test.py @@ -26,7 +26,7 @@ from tee_popen import TeePopen NO_CHANGES_MSG = "Nothing to run" -IMAGE_NAME = "clickhouse/sqllogic-test" +IMAGE_NAME = "altinityinfra/sqllogic-test" def get_run_command( diff --git a/tests/ci/sqltest.py b/tests/ci/sqltest.py index 8e6ca6ff87fa..35fa6fbaa199 100644 --- a/tests/ci/sqltest.py +++ b/tests/ci/sqltest.py @@ -14,7 +14,7 @@ from stopwatch import Stopwatch from ci_config import CI -IMAGE_NAME = "clickhouse/sqltest" +IMAGE_NAME = "altinityinfra/sqltest" def get_run_command(pr_number, sha, download_url, workspace_path, image): diff --git a/tests/ci/stress.py b/tests/ci/stress.py index a3cdd8b3d6f5..4e86f551b309 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -78,7 +78,7 @@ def get_options(i: int, upgrade_check: bool) -> str: client_options.append("ignore_drop_queries_probability=0.5") if random.random() < 0.2: - client_options.append("allow_experimental_parallel_reading_from_replicas=1") + client_options.append("enable_parallel_replicas=1") client_options.append("max_parallel_replicas=3") client_options.append("cluster_for_parallel_replicas='parallel_replicas'") client_options.append("parallel_replicas_for_non_replicated_merge_tree=1") diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 85da601e379c..48a4e2f87798 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -13,7 +13,6 @@ from clickhouse_helper import CiLogsCredentials from docker_images_helper import DockerImage, get_docker_image, pull_image from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH -from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import ERROR, JobReport, TestResults, read_test_results from stopwatch import Stopwatch @@ -34,14 +33,13 @@ def format(self, record): def get_additional_envs(check_name: str) -> List[str]: result = [] - azure_connection_string = get_parameter_from_ssm("azure_connection_string") - result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") + # TODO(vnemkov): put proper Azure connection string into SSM and re-enable this one + # azure_connection_string = get_parameter_from_ssm("azure_connection_string") + # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") # some cloud-specific features require feature flags enabled # so we need this ENV to be able to disable the randomization # of feature flags result.append("RANDOMIZE_KEEPER_FEATURE_FLAGS=1") - if "azure" in check_name: - result.append("USE_AZURE_STORAGE_FOR_MERGE_TREE=1") if "s3" in check_name: result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") @@ -208,4 +206,4 @@ def run_stress_test(docker_image_name: str) -> None: if __name__ == "__main__": - run_stress_test("clickhouse/stress-test") + run_stress_test("altinityinfra/stress-test") diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 36620d44a2d0..ba5aef398a19 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -168,7 +168,7 @@ def main(): run_shell_check = any(is_shell(file) for file in pr_info.changed_files) run_python_check = any(is_python(file) for file in pr_info.changed_files) - IMAGE_NAME = "clickhouse/style-test" + IMAGE_NAME = "altinityinfra/style-test" image = pull_image(get_docker_image(IMAGE_NAME)) docker_command = ( f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " diff --git a/tests/ci/test_git.py b/tests/ci/test_git.py index 60cd95b6869d..833582e01bd7 100644 --- a/tests/ci/test_git.py +++ b/tests/ci/test_git.py @@ -56,16 +56,16 @@ def setUp(self): self.git.commits_since_new = 0 def test_tags(self): - self.git.new_tag = "v21.12.333.22222-stable" - self.git.latest_tag = "v21.12.333.22222-stable" + self.git.new_tag = "v21.12.333.22222-altinitystable" + self.git.latest_tag = "v21.12.333.22222-altinitystable" for tag_attr in ("new_tag", "latest_tag"): - self.assertEqual(getattr(self.git, tag_attr), "v21.12.333.22222-stable") + self.assertEqual(getattr(self.git, tag_attr), "v21.12.333.22222-altinitystable") setattr(self.git, tag_attr, "") self.assertEqual(getattr(self.git, tag_attr), "") for tag in ( - "v21.12.333-stable", + "v21.12.333-altinitystable", "v21.12.333-prestable", - "21.12.333.22222-stable", + "21.12.333.22222-altinitystable", "v21.12.333.22222-production", ): with self.assertRaises(Exception): diff --git a/tests/ci/test_version.py b/tests/ci/test_version.py index c4f12091ec09..30f7dbc6a762 100644 --- a/tests/ci/test_version.py +++ b/tests/ci/test_version.py @@ -18,13 +18,14 @@ def test_version_arg(self): ("1.1.1.2", vh.get_version_from_string("1.1.1.2")), ("v11.1.1.2-lts", vh.get_version_from_string("11.1.1.2")), ("v01.1.1.2-prestable", vh.get_version_from_string("1.1.1.2")), - ("v21.1.1.2-stable", vh.get_version_from_string("21.1.1.2")), + ("v21.1.1.2-altinitystable", vh.get_version_from_string("21.1.1.2")), ("v31.1.1.2-testing", vh.get_version_from_string("31.1.1.2")), ("refs/tags/v31.1.1.2-testing", vh.get_version_from_string("31.1.1.2")), ) - for test_case in cases: - version = vh.version_arg(test_case[0]) - self.assertEqual(test_case[1], version) + for i, test_case in enumerate(cases): + with self.subTest(test_case, i=i): + version = vh.version_arg(test_case[0]) + self.assertEqual(test_case[1], version) error_cases = ( "0.0.0", "1.1.1.a", @@ -48,44 +49,52 @@ class TestCase: expected: CHV cases = ( + # TestCase( + # "v24.6.1.1-new", + # 15, + # "v24.4.1.2088-stable", + # 415, + # CHV(24, 5, 1, 54487, None, 415), + # ), + # TestCase( + # "v24.6.1.1-testing", + # 15, + # "v24.4.1.2088-stable", + # 415, + # CHV(24, 5, 1, 54487, None, 15), + # ), + # TestCase( + # "v24.6.1.1-stable", + # 15, + # "v24.4.1.2088-stable", + # 415, + # CHV(24, 5, 1, 54487, None, 15), + # ), + # TestCase( + # "v24.5.1.1-stable", + # 15, + # "v24.4.1.2088-stable", + # 415, + # CHV(24, 5, 1, 54487, None, 15), + # ), TestCase( - "v24.6.1.1-new", - 15, + "v24.5.1.100-stable", + 0, "v24.4.1.2088-stable", 415, - CHV(24, 5, 1, 54487, None, 415), - ), - TestCase( - "v24.6.1.1-testing", - 15, - "v24.4.1.2088-stable", - 415, - CHV(24, 5, 1, 54487, None, 16), - ), - TestCase( - "v24.6.1.1-stable", - 15, - "v24.4.1.2088-stable", - 415, - CHV(24, 5, 1, 54487, None, 15), - ), - TestCase( - "v24.5.1.1-stable", - 15, - "v24.4.1.2088-stable", - 415, - CHV(24, 5, 1, 54487, None, 15), + CHV(24, 5, 1, 54487, None, 100), ), ) git = Git(True) - for tc in cases: - git.latest_tag = tc.latest_tag - git.commits_since_latest = tc.commits_since_latest - git.new_tag = tc.new_tag - git.commits_since_new = tc.commits_since_new - self.assertEqual( - vh.get_version_from_repo( - Path("tests/ci/tests/autogenerated_versions.txt"), git - ), - tc.expected, - ) + for i, tc in enumerate(cases): + with self.subTest(tc, i=i): + git.latest_tag = tc.latest_tag + git.commits_since_latest = tc.commits_since_latest + git.new_tag = tc.new_tag + git.commits_since_new = tc.commits_since_new + self.assertEqual( + vh.get_version_from_repo( + Path("tests/ci/tests/autogenerated_versions.txt"), git + ), + tc.expected, + ) diff --git a/tests/ci/tests/autogenerated_versions.txt b/tests/ci/tests/autogenerated_versions.txt index 10028bf50c81..675fc161a2fc 100644 --- a/tests/ci/tests/autogenerated_versions.txt +++ b/tests/ci/tests/autogenerated_versions.txt @@ -6,6 +6,7 @@ SET(VERSION_REVISION 54487) SET(VERSION_MAJOR 24) SET(VERSION_MINOR 5) SET(VERSION_PATCH 1) +SET(VERSION_TWEAK 15) SET(VERSION_GITHASH 70a1d3a63d47f0be077d67b8deb907230fc7cfb0) SET(VERSION_DESCRIBE v24.5.1.1-testing) SET(VERSION_STRING 24.5.1.1) diff --git a/tests/ci/tests/docker_images_for_tests.json b/tests/ci/tests/docker_images_for_tests.json index 70db87605616..5faa3522e7ec 100644 --- a/tests/ci/tests/docker_images_for_tests.json +++ b/tests/ci/tests/docker_images_for_tests.json @@ -1,120 +1,120 @@ { "docker/packager/deb": { - "name": "clickhouse/deb-builder", + "name": "altinityinfra/deb-builder", "dependent": [] }, "docker/packager/binary": { - "name": "clickhouse/binary-builder", + "name": "altinityinfra/binary-builder", "dependent": [ "docker/test/codebrowser" ] }, "docker/test/compatibility/centos": { - "name": "clickhouse/test-old-centos", + "name": "altinityinfra/test-old-centos", "dependent": [] }, "docker/test/compatibility/ubuntu": { - "name": "clickhouse/test-old-ubuntu", + "name": "altinityinfra/test-old-ubuntu", "dependent": [] }, "docker/test/integration/base": { - "name": "clickhouse/integration-test", + "name": "altinityinfra/integration-test", "dependent": [] }, "docker/test/fuzzer": { - "name": "clickhouse/fuzzer", + "name": "altinityinfra/fuzzer", "dependent": [] }, "docker/test/performance-comparison": { - "name": "clickhouse/performance-comparison", + "name": "altinityinfra/performance-comparison", "dependent": [] }, "docker/test/util": { - "name": "clickhouse/test-util", + "name": "altinityinfra/test-util", "dependent": [ "docker/test/base", "docker/test/fasttest" ] }, "docker/test/stateless": { - "name": "clickhouse/stateless-test", + "name": "altinityinfra/stateless-test", "dependent": [ "docker/test/stateful", "docker/test/unit" ] }, "docker/test/stateful": { - "name": "clickhouse/stateful-test", + "name": "altinityinfra/stateful-test", "dependent": [ "docker/test/stress" ] }, "docker/test/unit": { - "name": "clickhouse/unit-test", + "name": "altinityinfra/unit-test", "dependent": [] }, "docker/test/stress": { - "name": "clickhouse/stress-test", + "name": "altinityinfra/stress-test", "dependent": [] }, "docker/test/codebrowser": { - "name": "clickhouse/codebrowser", + "name": "altinityinfra/codebrowser", "dependent": [] }, "docker/test/integration/runner": { - "name": "clickhouse/integration-tests-runner", + "name": "altinityinfra/integration-tests-runner", "dependent": [] }, "docker/test/fasttest": { - "name": "clickhouse/fasttest", + "name": "altinityinfra/fasttest", "dependent": [] }, "docker/test/style": { - "name": "clickhouse/style-test", + "name": "altinityinfra/style-test", "dependent": [] }, "docker/test/integration/s3_proxy": { - "name": "clickhouse/s3-proxy", + "name": "altinityinfra/s3-proxy", "dependent": [] }, "docker/test/integration/resolver": { - "name": "clickhouse/python-bottle", + "name": "altinityinfra/python-bottle", "dependent": [] }, "docker/test/integration/helper_container": { - "name": "clickhouse/integration-helper", + "name": "altinityinfra/integration-helper", "dependent": [] }, "docker/test/integration/mysql_golang_client": { - "name": "clickhouse/mysql-golang-client", + "name": "altinityinfra/mysql-golang-client", "dependent": [] }, "docker/test/integration/dotnet_client": { - "name": "clickhouse/dotnet-client", + "name": "altinityinfra/dotnet-client", "dependent": [] }, "docker/test/integration/mysql_java_client": { - "name": "clickhouse/mysql-java-client", + "name": "altinityinfra/mysql-java-client", "dependent": [] }, "docker/test/integration/mysql_js_client": { - "name": "clickhouse/mysql-js-client", + "name": "altinityinfra/mysql-js-client", "dependent": [] }, "docker/test/integration/mysql_php_client": { - "name": "clickhouse/mysql-php-client", + "name": "altinityinfra/mysql-php-client", "dependent": [] }, "docker/test/integration/postgresql_java_client": { - "name": "clickhouse/postgresql-java-client", + "name": "altinityinfra/postgresql-java-client", "dependent": [] }, "docker/test/integration/kerberos_kdc": { - "name": "clickhouse/kerberos-kdc", + "name": "altinityinfra/kerberos-kdc", "dependent": [] }, "docker/test/base": { - "name": "clickhouse/test-base", + "name": "altinityinfra/test-base", "dependent": [ "docker/test/stateless", "docker/test/integration/base", @@ -123,20 +123,16 @@ "docker/test/sqltest" ] }, - "docker/test/integration/kerberized_hadoop": { - "name": "clickhouse/kerberized-hadoop", - "dependent": [] - }, "docker/test/sqlancer": { - "name": "clickhouse/sqlancer-test", + "name": "altinityinfra/sqlancer-test", "dependent": [] }, "docker/test/keeper-jepsen": { - "name": "clickhouse/keeper-jepsen-test", + "name": "altinityinfra/keeper-jepsen-test", "dependent": [] }, "docker/docs/builder": { - "name": "clickhouse/docs-builder", + "name": "altinityinfra/docs-builder", "only_amd64": true, "dependent": [ "docker/docs/check", @@ -144,19 +140,19 @@ ] }, "docker/docs/check": { - "name": "clickhouse/docs-check", + "name": "altinityinfra/docs-check", "dependent": [] }, "docker/docs/release": { - "name": "clickhouse/docs-release", + "name": "altinityinfra/docs-release", "dependent": [] }, "docker/test/sqllogic": { - "name": "clickhouse/sqllogic-test", + "name": "altinityinfra/sqllogic-test", "dependent": [] }, "docker/test/sqltest": { - "name": "clickhouse/sqltest", + "name": "altinityinfra/sqltest", "dependent": [] } } diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index 9cc8ec379bfe..7aabe009bc99 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -15,7 +15,7 @@ from stopwatch import Stopwatch from tee_popen import TeePopen -IMAGE_NAME = "clickhouse/unit-test" +IMAGE_NAME = "altinityinfra/unit-test" def get_test_name(line): diff --git a/tests/ci/upgrade_check.py b/tests/ci/upgrade_check.py index 83b6f9e299fd..f84451cad81d 100644 --- a/tests/ci/upgrade_check.py +++ b/tests/ci/upgrade_check.py @@ -1,4 +1,4 @@ import stress_check if __name__ == "__main__": - stress_check.run_stress_test("clickhouse/upgrade-check") + stress_check.run_stress_test("altinityinfra/upgrade-check") diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 07a7a9601c04..559af989540f 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Literal, Optional, Set, Tuple, Union -from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix +from git_helper import TWEAK, Git, get_tags, git_runner, removeprefix, VersionType FILE_WITH_VERSION_PATH = "cmake/autogenerated_versions.txt" CHANGELOG_IN_PATH = "debian/changelog.in" @@ -29,6 +29,7 @@ SET(VERSION_MINOR {minor}) SET(VERSION_PATCH {patch}) SET(VERSION_GITHASH {githash}) +SET(VERSION_TWEAK {tweak}) SET(VERSION_DESCRIBE {describe}) SET(VERSION_STRING {string}) # end of autochange @@ -48,6 +49,7 @@ def __init__( revision: Union[int, str], git: Optional[Git], tweak: Optional[Union[int, str]] = None, + flavour: Optional[str] = None, ): self._major = int(major) self._minor = int(minor) @@ -61,6 +63,7 @@ def __init__( self._tweak = self._git.tweak self._describe = "" self._description = "" + self._flavour = flavour def update(self, part: PART_TYPE) -> "ClickHouseVersion": """If part is valid, returns a new version""" @@ -152,9 +155,12 @@ def description(self) -> str: @property def string(self): - return ".".join( + version_as_string = ".".join( (str(self.major), str(self.minor), str(self.patch), str(self.tweak)) ) + if self._flavour: + version_as_string = f"{version_as_string}.{self._flavour}" + return version_as_string @property def is_lts(self) -> bool: @@ -185,7 +191,10 @@ def with_description(self, version_type): if version_type not in VersionType.VALID: raise ValueError(f"version type {version_type} not in {VersionType.VALID}") self._description = version_type - self._describe = f"v{self.string}-{version_type}" + if version_type == self._flavour: + self._describe = f"v{self.string}" + else: + self._describe = f"v{self.string}-{version_type}" return self def copy(self) -> "ClickHouseVersion": @@ -242,21 +251,12 @@ def __repr__(self): ClickHouseVersions = List[ClickHouseVersion] - -class VersionType: - LTS = "lts" - NEW = "new" - PRESTABLE = "prestable" - STABLE = "stable" - TESTING = "testing" - VALID = (NEW, TESTING, PRESTABLE, STABLE, LTS) - - def validate_version(version: str) -> None: + # NOTE(vnemkov): minor but important fixes, so versions with 'flavour' are threated as valid (e.g. 22.8.8.4.altinitystable) parts = version.split(".") - if len(parts) != 4: + if len(parts) < 4: raise ValueError(f"{version} does not contain 4 parts") - for part in parts: + for part in parts[:4]: int(part) @@ -297,18 +297,36 @@ def get_version_from_repo( versions["patch"], versions["revision"], git, + # Explicitly use tweak value from version file + tweak=versions.get("tweak", versions["revision"]), + flavour=versions.get("flavour", None) ) - # Since 24.5 we have tags like v24.6.1.1-new, and we must check if the release - # branch already has it's own commit. It's necessary for a proper tweak version + + # if this commit is tagged, use tag's version instead of something stored in cmake if git is not None and git.latest_tag: version_from_tag = get_version_from_tag(git.latest_tag) - if ( - version_from_tag.description == VersionType.NEW - and cmake_version < version_from_tag - ): - # We are in a new release branch without existing release. - # We should change the tweak version to a `tweak_to_new` - cmake_version.tweak = git.tweak_to_new + logging.debug(f'Git latest tag: {git.latest_tag} ({git.commits_since_latest} commits ago)\n' + f'"new" tag: {git.new_tag} ({git.commits_since_new})\n' + f'current commit: {git.sha}\n' + f'current brach: {git.branch}' + ) + if git.commits_since_latest == 0: + # Tag has a priority over the version written in CMake. + # Version must match (except tweak, flavour, description, etc.) to avoid accidental mess. + if not (version_from_tag.major == cmake_version.major \ + and version_from_tag.minor == cmake_version.minor \ + and version_from_tag.patch == cmake_version.patch): + raise RuntimeError(f"Version generated from tag ({version_from_tag}) should have same major, minor, and patch values as version generated from cmake ({cmake_version})") + + # Don't need to reset version completely, mostly because revision part is not set in tag, but must be preserved + logging.debug(f"Resetting TWEAK and FLAVOUR of version from cmake {cmake_version} to values from tag: {version_from_tag.tweak}.{version_from_tag._flavour}") + cmake_version._flavour = version_from_tag._flavour + cmake_version.tweak = version_from_tag.tweak + else: + # We've had some number of commits since the latest tag. + logging.debug(f"Bumping the TWEAK of version from cmake {cmake_version} by {git.commits_since_latest}") + cmake_version.tweak = cmake_version.tweak + git.commits_since_latest + return cmake_version @@ -316,15 +334,31 @@ def get_version_from_string( version: str, git: Optional[Git] = None ) -> ClickHouseVersion: validate_version(version) - parts = version.split(".") - return ClickHouseVersion(parts[0], parts[1], parts[2], -1, git, parts[3]) + # dict for simple handling of missing parts with parts.get(index, default) + parts = dict(enumerate(version.split("."))) + return ClickHouseVersion( + parts[0], + parts[1], + parts[2], + -1, + git, + parts.get(3, None), + parts.get(4, None) + ) def get_version_from_tag(tag: str) -> ClickHouseVersion: Git.check_tag(tag) - tag, description = tag[1:].split("-", 1) - version = get_version_from_string(tag) - version.with_description(description) + tag = tag[1:] # strip initial 'v' + if '-' in tag: + # Upstream tags with dash + tag, description = tag.split("-", 1) + version = get_version_from_string(tag) + version.with_description(description) + else: + # Altinity's tags, with dots as separators between parts (handled properly down the road) + version = get_version_from_string(tag) + return version @@ -424,7 +458,7 @@ def update_contributors( get_abs_path(relative_contributors_path).write_text(content, encoding="utf-8") -def update_version_local(version, version_type="testing"): +def update_version_local(version : ClickHouseVersion, version_type="testing"): update_contributors() version.with_description(version_type) update_cmake_version(version) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index a3d7e0e922de..665800f8b716 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -912,6 +912,7 @@ class SettingsRandomizer: "min_external_table_block_size_bytes": lambda: random.choice([0, 1, 100000000]), "max_parsing_threads": lambda: random.choice([0, 1, 10]), "optimize_functions_to_subcolumns": lambda: random.randint(0, 1), + "parallel_replicas_local_plan": lambda: random.randint(0, 1), } @staticmethod diff --git a/tests/config/config.d/azure_storage_conf.xml b/tests/config/config.d/azure_storage_conf.xml index f24b62b87b10..412d40111a70 100644 --- a/tests/config/config.d/azure_storage_conf.xml +++ b/tests/config/config.d/azure_storage_conf.xml @@ -4,10 +4,13 @@ object_storage azure + http://localhost:10000/devstoreaccount1 + cont false + + devstoreaccount1 + Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== 33554432 - openbucketforpublicci - cache diff --git a/tests/config/config.d/azure_storage_policy_by_default.xml b/tests/config/config.d/azure_storage_policy_by_default.xml deleted file mode 100644 index cab8a106f1b8..000000000000 --- a/tests/config/config.d/azure_storage_policy_by_default.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - azure_cache - - diff --git a/tests/config/config.d/clusters.xml b/tests/config/config.d/clusters.xml index af26565d7e6a..ff42b5828e36 100644 --- a/tests/config/config.d/clusters.xml +++ b/tests/config/config.d/clusters.xml @@ -115,10 +115,6 @@ false - - 127.0.0.1 - 9000 - 127.0.0.2 9000 @@ -147,6 +143,10 @@ 127.0.0.8 9000 + + 127.0.0.1 + 9000 + 127.0.0.9 9000 diff --git a/tests/config/config.d/database_replicated.xml b/tests/config/config.d/database_replicated.xml index d8bed2f08feb..4c6f262ddc5e 100644 --- a/tests/config/config.d/database_replicated.xml +++ b/tests/config/config.d/database_replicated.xml @@ -44,10 +44,19 @@ 5000 information false + 1 1000000000000000 + + 1 + 1 + 1 + 1 + 1 + + 1 diff --git a/tests/integration/compose/docker_compose_clickhouse.yml b/tests/integration/compose/docker_compose_clickhouse.yml index fdd124ede91a..ff4523c5b0d7 100644 --- a/tests/integration/compose/docker_compose_clickhouse.yml +++ b/tests/integration/compose/docker_compose_clickhouse.yml @@ -2,4 +2,4 @@ version: '2.3' # Used to pre-pull images with docker-compose services: clickhouse1: - image: clickhouse/integration-test + image: altinityinfra/integration-test diff --git a/tests/integration/compose/docker_compose_dotnet_client.yml b/tests/integration/compose/docker_compose_dotnet_client.yml index b63dac51522c..e5746fa209fb 100644 --- a/tests/integration/compose/docker_compose_dotnet_client.yml +++ b/tests/integration/compose/docker_compose_dotnet_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: dotnet1: - image: clickhouse/dotnet-client:${DOCKER_DOTNET_CLIENT_TAG:-latest} + image: altinityinfra/dotnet-client:${DOCKER_DOTNET_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/tests/integration/compose/docker_compose_jdbc_bridge.yml b/tests/integration/compose/docker_compose_jdbc_bridge.yml index b3686adc21c4..3f8edfb88268 100644 --- a/tests/integration/compose/docker_compose_jdbc_bridge.yml +++ b/tests/integration/compose/docker_compose_jdbc_bridge.yml @@ -1,6 +1,7 @@ version: '2.3' services: bridge1: + # NOTE(vnemkov): not produced by CI/CD, so must not be replaced with altinityinfra/jdbc-bridge image: clickhouse/jdbc-bridge command: | /bin/bash -c 'cat << EOF > config/datasources/self.json @@ -24,4 +25,4 @@ services: volumes: - type: ${JDBC_BRIDGE_FS:-tmpfs} source: ${JDBC_BRIDGE_LOGS:-} - target: /app/logs \ No newline at end of file + target: /app/logs diff --git a/tests/integration/compose/docker_compose_keeper.yml b/tests/integration/compose/docker_compose_keeper.yml index 91010c4aa83d..fba5bc728f88 100644 --- a/tests/integration/compose/docker_compose_keeper.yml +++ b/tests/integration/compose/docker_compose_keeper.yml @@ -1,7 +1,7 @@ version: '2.3' services: zoo1: - image: ${image:-clickhouse/integration-test} + image: ${image:-altinityinfra/integration-test} restart: always user: ${user:-} volumes: @@ -37,7 +37,7 @@ services: - inet6 - rotate zoo2: - image: ${image:-clickhouse/integration-test} + image: ${image:-altinityinfra/integration-test} restart: always user: ${user:-} volumes: @@ -73,7 +73,7 @@ services: - inet6 - rotate zoo3: - image: ${image:-clickhouse/integration-test} + image: ${image:-altinityinfra/integration-test} restart: always user: ${user:-} volumes: diff --git a/tests/integration/compose/docker_compose_kerberized_hdfs.yml b/tests/integration/compose/docker_compose_kerberized_hdfs.yml index e955a14eb3df..58d321177c0d 100644 --- a/tests/integration/compose/docker_compose_kerberized_hdfs.yml +++ b/tests/integration/compose/docker_compose_kerberized_hdfs.yml @@ -4,7 +4,7 @@ services: kerberizedhdfs1: cap_add: - DAC_READ_SEARCH - image: clickhouse/kerberized-hadoop:${DOCKER_KERBERIZED_HADOOP_TAG:-latest} + image: altinityinfra/kerberized-hadoop:${DOCKER_KERBERIZED_HADOOP_TAG:-latest} hostname: kerberizedhdfs1 restart: always volumes: @@ -24,7 +24,7 @@ services: net.ipv4.ip_local_port_range: '55000 65535' hdfskerberos: - image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} + image: altinityinfra/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} hostname: hdfskerberos volumes: - ${KERBERIZED_HDFS_DIR}/secrets:/tmp/keytab diff --git a/tests/integration/compose/docker_compose_kerberized_kafka.yml b/tests/integration/compose/docker_compose_kerberized_kafka.yml index 49d4c1db90fe..7ae1011b1876 100644 --- a/tests/integration/compose/docker_compose_kerberized_kafka.yml +++ b/tests/integration/compose/docker_compose_kerberized_kafka.yml @@ -52,7 +52,7 @@ services: net.ipv4.ip_local_port_range: '55000 65535' kafka_kerberos: - image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} + image: altinityinfra/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} hostname: kafka_kerberos volumes: - ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab diff --git a/tests/integration/compose/docker_compose_kerberos_kdc.yml b/tests/integration/compose/docker_compose_kerberos_kdc.yml index 3ce9a6df1fb6..062bdace6e9c 100644 --- a/tests/integration/compose/docker_compose_kerberos_kdc.yml +++ b/tests/integration/compose/docker_compose_kerberos_kdc.yml @@ -2,7 +2,7 @@ version: '2.3' services: kerberoskdc: - image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} + image: altinityinfra/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} hostname: kerberoskdc volumes: - ${KERBEROS_KDC_DIR}/secrets:/tmp/keytab diff --git a/tests/integration/compose/docker_compose_minio.yml b/tests/integration/compose/docker_compose_minio.yml index 40098d05b04b..b709161296d9 100644 --- a/tests/integration/compose/docker_compose_minio.yml +++ b/tests/integration/compose/docker_compose_minio.yml @@ -19,14 +19,14 @@ services: # HTTP proxies for Minio. proxy1: - image: clickhouse/s3-proxy + image: altinityinfra/s3-proxy expose: - "8080" # Redirect proxy port - "80" # Reverse proxy port - "443" # Reverse proxy port (secure) proxy2: - image: clickhouse/s3-proxy + image: altinityinfra/s3-proxy expose: - "8080" - "80" @@ -34,7 +34,7 @@ services: # Empty container to run proxy resolver. resolver: - image: clickhouse/python-bottle:${DOCKER_PYTHON_BOTTLE_TAG:-latest} + image: altinityinfra/python-bottle:${DOCKER_PYTHON_BOTTLE_TAG:-latest} expose: - "8080" tty: true diff --git a/tests/integration/compose/docker_compose_mysql_golang_client.yml b/tests/integration/compose/docker_compose_mysql_golang_client.yml index 56cc04105740..09154b584244 100644 --- a/tests/integration/compose/docker_compose_mysql_golang_client.yml +++ b/tests/integration/compose/docker_compose_mysql_golang_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: golang1: - image: clickhouse/mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest} + image: altinityinfra/mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/tests/integration/compose/docker_compose_mysql_java_client.yml b/tests/integration/compose/docker_compose_mysql_java_client.yml index 529974dd4bfe..ffda321958cf 100644 --- a/tests/integration/compose/docker_compose_mysql_java_client.yml +++ b/tests/integration/compose/docker_compose_mysql_java_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: java1: - image: clickhouse/mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest} + image: altinityinfra/mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest} # to keep container running command: sleep 1d diff --git a/tests/integration/compose/docker_compose_mysql_js_client.yml b/tests/integration/compose/docker_compose_mysql_js_client.yml index 90939449c5f3..b46eb2706c47 100644 --- a/tests/integration/compose/docker_compose_mysql_js_client.yml +++ b/tests/integration/compose/docker_compose_mysql_js_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: mysqljs1: - image: clickhouse/mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest} + image: altinityinfra/mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/tests/integration/compose/docker_compose_mysql_php_client.yml b/tests/integration/compose/docker_compose_mysql_php_client.yml index 408b8ff089a9..662783a00a1f 100644 --- a/tests/integration/compose/docker_compose_mysql_php_client.yml +++ b/tests/integration/compose/docker_compose_mysql_php_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: php1: - image: clickhouse/mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest} + image: altinityinfra/mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/tests/integration/compose/docker_compose_nginx.yml b/tests/integration/compose/docker_compose_nginx.yml index 38d2a6d84c84..9d4403f283fb 100644 --- a/tests/integration/compose/docker_compose_nginx.yml +++ b/tests/integration/compose/docker_compose_nginx.yml @@ -5,7 +5,7 @@ services: # Files will be put into /usr/share/nginx/files. nginx: - image: clickhouse/nginx-dav:${DOCKER_NGINX_DAV_TAG:-latest} + image: altinityinfra/nginx-dav:${DOCKER_NGINX_DAV_TAG:-latest} restart: always ports: - 80:80 diff --git a/tests/integration/compose/docker_compose_postgresql_java_client.yml b/tests/integration/compose/docker_compose_postgresql_java_client.yml index 904bfffdfd5b..5c8673ae3eeb 100644 --- a/tests/integration/compose/docker_compose_postgresql_java_client.yml +++ b/tests/integration/compose/docker_compose_postgresql_java_client.yml @@ -1,6 +1,6 @@ version: '2.2' services: java: - image: clickhouse/postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest} + image: altinityinfra/postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 3022a91b6865..d802a2166a3d 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -75,7 +75,8 @@ # Minimum version we use in integration tests to check compatibility with old releases # Keep in mind that we only support upgrading between releases that are at most 1 year different. # This means that this minimum need to be, at least, 1 year older than the current release -CLICKHOUSE_CI_MIN_TESTED_VERSION = "23.3" +# NOTE(vnemkov): this is a docker tag, make sure it doesn't include initial 'v' +CLICKHOUSE_CI_MIN_TESTED_VERSION = "23.3.19.33.altinitystable" ZOOKEEPER_CONTAINERS = ("zoo1", "zoo2", "zoo3") @@ -1074,7 +1075,7 @@ def setup_keeper_cmd(self, instance, env_variables, docker_compose_yml_dir): env_variables["keeper_binary"] = binary_path env_variables["keeper_cmd_prefix"] = keeper_cmd_prefix - env_variables["image"] = "clickhouse/integration-test:" + self.docker_base_tag + env_variables["image"] = "altinityinfra/integration-test:" + self.docker_base_tag env_variables["user"] = str(os.getuid()) env_variables["keeper_fs"] = "bind" for i in range(1, 4): @@ -1724,7 +1725,7 @@ def add_instance( hostname=None, env_variables=None, instance_env_variables=False, - image="clickhouse/integration-test", + image="altinityinfra/integration-test", tag=None, stay_alive=False, ipv4_address=None, @@ -3429,7 +3430,7 @@ def __init__( hostname=None, env_variables=None, instance_env_variables=False, - image="clickhouse/integration-test", + image="altinityinfra/integration-test", tag="latest", stay_alive=False, ipv4_address=None, @@ -4072,7 +4073,7 @@ def contains_in_log( exclusion_substring="", ): if from_host: - # We check fist file exists but want to look for all rotated logs as well + # We check first file exists but want to look for all rotated logs as well result = subprocess_check_call( [ "bash", @@ -4695,6 +4696,9 @@ def write_embedded_config(name, dest_dir, fix_log_level=False): if self.with_kerberized_hdfs: depends_on.append("kerberizedhdfs1") + if self.with_ldap: + depends_on.append("openldap") + if self.with_rabbitmq: depends_on.append("rabbitmq1") diff --git a/tests/integration/helpers/mongo_cert.pem b/tests/integration/helpers/mongo_cert.pem index b9c3b83e4733..9444d19a3d2a 100644 --- a/tests/integration/helpers/mongo_cert.pem +++ b/tests/integration/helpers/mongo_cert.pem @@ -1,49 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC3uaPiZMfjPBBE -yDEYJsJIoriu0SaC80uTmPM7bFpnOOXOBvbT4wD2q+uVaLQifKtPTgZAkP5Y3rX8 -S5TOzaLsNp68S1Ja/EzxQUolOSgb4A948TTiUTrTjfMxsPRhmxXTjozWV8CFtL9P -Lg6H+55oyQOJedWYe1kSWRJQayXSweBK5qjOPi2qDF/xdFRQuMivpBUar/b/E9GQ -RKpIaoqMYsl/WF/tReb4N658UxkVlFdR8s48UoA9LfJLMPr4N+QDTfvtcT2bYlpT -4a9b6IXa9BQKCw3AKfTqEPO1XunH//iLNkt1bLtqgZNyT/tY0tLY3EKMXIDuRBVn -KCbfVJ1RAgMBAAECggEAJFCjXiqBgB7tMEtJuPZgTK8tRhC9RgEFHUWMPmCqdeC/ -O7wQqc0i8Z8Fz+CESpTN370Sa0y9mZ9b5WSjI0VuQLaDJcDVpHpeUwmOuFDV5ryh -EkzLITjhIdPbECVkCK7433o7yFpMCaGydtopsSNBKoEhG9ljKOKotoG4pwCm10N5 -K9Qepj82OjRhLkpmuiMFb4/vvOm5dglYmkq5+n/fdUYFtrYr3NvMSCTlietPHDgV -Wb3strvk1g9ARWfa2j7Q6moF2sbyob9zVLoRiD9VgmNB60v7QAJxDctVkbOoDgKp -uN2fkxTHwlOPAO6Zhgnie11jnZr1711TFxmEfMkSKQKBgQDqpB8m0hSJsWLKWxQK -yx+5Xgs+Cr8gb0AYHJQ87obj2XqwXLpBSMrkzTn6vIGRv+NMSfiM/755RUm5aJPY -om+7F68JEIL26ZA7bIfjHhV5o9fvpo+6N6cJyR08Q/KkF8Tej9K4qQec0W/jtKeZ -KAJ1k7/BBuN82iTtEJ3GWBaaRwKBgQDIcwQrGlyyXqnBK25gl/E1Ng+V3p/2sy98 -1BpEshxen4KorHEXCJArydELtvK/ll6agil6QebrJN5dtYOOgvcDTu1mQjdUPN3C -VXpSQ0L8XxfyTNYQTWON9wJGL1pzlTiyHvlSrQFsFWMUoxrqndWIIRtrXjap1npp -HDrcqy2/pwKBgB5fHhUlTjlAd7wfq+l1v2Z8ENJ4C6NEIzS7xkhYy6cEiIf5iLZY -mMKi+eVFrzPRdbdzP7Poipwh5tgT/EcnR3UdLK/srjcNpni6pKA2TatQFOxVT/dX -qsxudtVNKkQpO3dfgHQclPqsdWIxCRye/CqB9Gkk3h9UEUGKTBHXZx2TAoGAF0tG -cLvfidr2Xzxs10zQ+x4NMZ1teX3ZRuhfJRyNr3FZ/cAMZGDaYDxTzsiz7Q/Mbqgx -qcN+0lS2gq1VXHpbukaxz/Bh/agVHUBRtr2aSznBzqafOcXEi/roiL94A3aT4B85 -WiJAyA60NPG/bwRojClMxm1sbNA/6XceYAaEioECgYEA3m88G3UwizfJAsfT5H5K -3HXNYzQ1XGrA8shI0kxeqfNP5qmTfH5q/K2VMWeShT3F/9Ytgc+H8c9XP1wKq7Zl -6AtmdDOeLzHkgwVK0p20/Wh2Qjw4ikJLdM+y8wnfMiwCXWQxoh1X905EwNtyBc2Z -9S3G5CXldFHC4NGdx0vetiE= +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCUnkHzBu8XdPpD +3oWJRLBMNEptQwdOUl1QhMyF9wxQ+lya3fHnlsJc2GgWrZHMI/vMr6rJHtNl/92M +KcychYswKiSKCqpeXOWIotsR9UvTxT9Mvi6Ca3vnkM4h8jt97XnnnXhjSdOUptmQ +M43zEO0XYekvoHSDuGNdNokEecq9mk64e7bu15DQo8Ck6jDL80opD9UX7diehAop +vjI2AgrR/+MlxnhCeXqVOLd8ukNtwdmRIWPrKlHcICvabvKHuJpGdUL8GpHPUINd +qMEUg4wQWKz74IV1aZCdiZICk6yACM/bWed3g1RzV21jAKLaL4xJkUqBiEtcs8Jk +PT3cVMGHAgMBAAECggEAAul6qiHchB+uQMCWyC5xTeRqAXR3tAv4Tj4fGJjkXY4Z +OrAjr9Kp38EvX1amgvUWV3FT3NMevDf5xd9OdzAA0g0uJIF+mAhYFW48i1FnQcHQ +mOf0zmiZR7l8o7ROb3JvooXHxW+ba/qjGPVwC801gJvruehgbOCRxh9DTRp7sH5K +BmcddhULhKBEQjWUmYNEM3A2axpdi3g1aYKERRLn8J0DXcItTwbxuxbNcs3erl8W +3yyv/JKmqnWF5sNyX3wEWuQcDEZZy+W7Hn4KPMxyU+WA5el5nJ8kFlxhpInmajwu +8Ytn6IEyThyXutVomosVBuP16QORl2Nad0hnQO9toQKBgQDDgiehXr3k2wfVaVOD +PocW4leXausIU2XcCn6FxTG9vLUDMPANw0MxgenC2nrjaUU9J9UjdRYgMcFGWrl4 +E27wEn5e0nZ/Y7F2cfhuOc9vNmZ+eHm2KQRyfAjIVL5Hpldqk2jXyCnLBNeWGHSw +kPQMU+FLqmrOFUvXlD2my+OSHwKBgQDCmgS9r+xFh4BCB9dY6eyQJF/jYmAQHs26 +80WJ6gAhbUw1O71uDtS9/3PZVXwwNCOHrcc49BPrpJdxGPHGvd2Q5y+j5LDDbQSZ +aLTiCZ2B0RM5Bd2dXD8gEHN4WCX7pJ/o4kDi4zONBmp5mg/tFfer5z5IU/1P7Wak +1Mu0JIHzmQKBgDNaNoqeVgaMuYwGtFbez6DlJtiwzrdLIJAheYYte5k4vdruub8D +sNyKIRp7RJgDCJq9obBEiuE98GRIZDrz78nDMco6QcHIL87KtNRO/vtZMKa7gkyk +jXR8u9nS2H/9YyytN3amLsQSq4XTOqM+D7xFNAIp6w/ibB9d4quzFj1FAoGBAKTE +x/LcO897NWuzO/D6z+QUCGR87R15F3SNenmVedrTskz4ciH3yMW+v5ZrPSWLX/IH +f8GHWD6TM+780eoW5L1GIh5BCjHN4rEJ6O3iekxqfD4x6zzL2F8Lztk8uZxh/Uuw +FoSFHybvIcQoYAe8K+KPfzq6cqb0OY6i5n920dkxAoGAJkw6ADqsJfH3NR+bQfgF +oEA1KqriMxyEJm44Y7E80C+iF4iNALF+Er9TSnr4mDxX5e/dW9d1YeS9o0nOfkpF +MaBmJfxqo4QQJLPRaxYQ2Jhfn7irir4BroxeNXQgNNhgSuKIvkfRyGYwl7P0AT4v +8H8rkZGneMD3gLB5MfnRhGk= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIDazCCAlOgAwIBAgIUO9pfiBMsADdk9nBMHs10n8kaIr8wDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjA0MTIwOTQxNDVaFw0yNTAx -MDUwOTQxNDVaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw -HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQC3uaPiZMfjPBBEyDEYJsJIoriu0SaC80uTmPM7bFpn -OOXOBvbT4wD2q+uVaLQifKtPTgZAkP5Y3rX8S5TOzaLsNp68S1Ja/EzxQUolOSgb -4A948TTiUTrTjfMxsPRhmxXTjozWV8CFtL9PLg6H+55oyQOJedWYe1kSWRJQayXS -weBK5qjOPi2qDF/xdFRQuMivpBUar/b/E9GQRKpIaoqMYsl/WF/tReb4N658UxkV -lFdR8s48UoA9LfJLMPr4N+QDTfvtcT2bYlpT4a9b6IXa9BQKCw3AKfTqEPO1XunH -//iLNkt1bLtqgZNyT/tY0tLY3EKMXIDuRBVnKCbfVJ1RAgMBAAGjUzBRMB0GA1Ud -DgQWBBSx7Tx8W4c6wjW0qkeG7CAMLY7YkjAfBgNVHSMEGDAWgBSx7Tx8W4c6wjW0 -qkeG7CAMLY7YkjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAb -/Up/LEIdwhiN/S3HolxY2D2BrTpKHLQuggBN4+gZlK5OksCkM46LYlP/ruHXCxbR -mQoRhmooj4TvkKyBwzvKq76O+OuRtBhXzRipnBbNTqFPLf9enJUrut8lsFrI+pdl -Nn4PSGGbFPpQ5vFRCktczwwYh0zLuZ/1DbFsbRWlDnZdvoWZdfV0qsvcBRK2DXDI -29xSfw897OpITIkaryZigQVsKv8TXhfsaq9PUuH0/z84S82QG5fR6FzULofgkylb -wXvwaSdcu3k4Lo8j77BEAEvlH8Ynja0eojx5Avl9h4iw/IOQKE4GAg56CzcequLv -clPlaBBWoD6yn+q4NhLF +MIIEGzCCAwOgAwIBAgIUaoGlyuJAyvs6yowFXymfu7seEiUwDQYJKoZIhvcNAQEL +BQAwgZwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDbGlja0hvdXNlMRMwEQYDVQQH +DApDbGlja0hvdXNlMREwDwYDVQQKDAhQZXJzb25hbDETMBEGA1UECwwKQ2xpY2tI +b3VzZTEkMCIGCSqGSIb3DQEJARYVY2xpY2tob3VzZUBjbGlja2hvdXNlMRUwEwYD +VQQDDAxtb25nb19zZWN1cmUwHhcNMjQwNTI2MTYwMDMxWhcNMzQwNTI0MTYwMDMx +WjCBnDELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNsaWNrSG91c2UxEzARBgNVBAcM +CkNsaWNrSG91c2UxETAPBgNVBAoMCFBlcnNvbmFsMRMwEQYDVQQLDApDbGlja0hv +dXNlMSQwIgYJKoZIhvcNAQkBFhVjbGlja2hvdXNlQGNsaWNraG91c2UxFTATBgNV +BAMMDG1vbmdvX3NlY3VyZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AJSeQfMG7xd0+kPehYlEsEw0Sm1DB05SXVCEzIX3DFD6XJrd8eeWwlzYaBatkcwj ++8yvqske02X/3YwpzJyFizAqJIoKql5c5Yii2xH1S9PFP0y+LoJre+eQziHyO33t +eeedeGNJ05Sm2ZAzjfMQ7Rdh6S+gdIO4Y102iQR5yr2aTrh7tu7XkNCjwKTqMMvz +SikP1Rft2J6ECim+MjYCCtH/4yXGeEJ5epU4t3y6Q23B2ZEhY+sqUdwgK9pu8oe4 +mkZ1Qvwakc9Qg12owRSDjBBYrPvghXVpkJ2JkgKTrIAIz9tZ53eDVHNXbWMAotov +jEmRSoGIS1yzwmQ9PdxUwYcCAwEAAaNTMFEwHQYDVR0OBBYEFJyz3Kt5XBDg5cvI +0v1ioqejqX+CMB8GA1UdIwQYMBaAFJyz3Kt5XBDg5cvI0v1ioqejqX+CMA8GA1Ud +EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAHAQFA5VMYvaQFnKtKfHg9TF +qfJ4uM3YsGdgsgmGWgflD1S4Z290H6Q2QvyZAEceTrlJxqArlWlVp5DAU6EeXjEh +QMAgdkJHF1Hg2jsZKPtdkb88UtuzwAME357T8NtEJSHzNE5QqYwlVM71JkWpdqvA +UUdOJbWhhJfowIf4tMmL1DUuIy2qYpoP/tEBXEw9uwpmZqb7KELwT3lRyOMaGFN7 +RHVwbvJWlHiu83QDNaWz6ijQkWl3tCN6TWcFD1qc1x8GpMzjbsAAYbCx7fbHM2LD +9kGSCiyv5K0MLNK5u67RtUFfPHtyD8RA0TtxIZ4PEN/eFANKS2/5NEi1ZuZ5/Pk= -----END CERTIFICATE----- diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index e6e79dc79478..3689bb409d15 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -243,7 +243,7 @@ def __init__( def _ensure_container(self): if self._container is None or self._container_expire_time <= time.time(): - image_name = "clickhouse/integration-helper:" + os.getenv( + image_name = "altinityinfra/integration-helper:" + os.getenv( "DOCKER_HELPER_TAG", "latest" ) for i in range(5): diff --git a/tests/integration/runner b/tests/integration/runner index 0667541b196d..c8f23ba1258c 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -29,7 +29,7 @@ CONFIG_DIR_IN_REPO = "programs/server" INTEGRATION_DIR_IN_REPO = "tests/integration" UTILS_DIR_IN_REPO = "utils" -DIND_INTEGRATION_TESTS_IMAGE_NAME = "clickhouse/integration-tests-runner" +DIND_INTEGRATION_TESTS_IMAGE_NAME = "altinityinfra/integration-tests-runner" def check_args_and_update_paths(args): diff --git a/tests/integration/test_backward_compatibility/configs/clusters.xml b/tests/integration/test_backward_compatibility/configs/clusters.xml new file mode 100644 index 000000000000..ac773152df92 --- /dev/null +++ b/tests/integration/test_backward_compatibility/configs/clusters.xml @@ -0,0 +1,20 @@ + + + + + + node0 + 9000 + + + node1 + 9000 + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 7de5f51921b7..953ae47c109d 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -6,7 +6,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py index 6b385bf84020..d794d53a8401 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -6,7 +6,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, ) diff --git a/tests/integration/test_backward_compatibility/test_aggregate_function_state.py b/tests/integration/test_backward_compatibility/test_aggregate_function_state.py index 9878c1ed70ea..7775bbcb6153 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_function_state.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_function_state.py @@ -6,7 +6,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -14,7 +14,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_convert_ordinary.py b/tests/integration/test_backward_compatibility/test_convert_ordinary.py index f5d0c0666007..7011a384bde8 100644 --- a/tests/integration/test_backward_compatibility/test_convert_ordinary.py +++ b/tests/integration/test_backward_compatibility/test_convert_ordinary.py @@ -4,7 +4,7 @@ cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_zookeeper=True, diff --git a/tests/integration/test_backward_compatibility/test_cte_distributed.py b/tests/integration/test_backward_compatibility/test_cte_distributed.py index e0be009e8744..73330ac04732 100644 --- a/tests/integration/test_backward_compatibility/test_cte_distributed.py +++ b/tests/integration/test_backward_compatibility/test_cte_distributed.py @@ -7,7 +7,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py index 202a741bfb59..af13c53ab8f2 100644 --- a/tests/integration/test_backward_compatibility/test_functions.py +++ b/tests/integration/test_backward_compatibility/test_functions.py @@ -12,7 +12,7 @@ upstream = cluster.add_instance("upstream", use_old_analyzer=True) backward = cluster.add_instance( "backward", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, ) diff --git a/tests/integration/test_backward_compatibility/test_insert_profile_events.py b/tests/integration/test_backward_compatibility/test_insert_profile_events.py index a90453d045be..d3955f5cf3c0 100644 --- a/tests/integration/test_backward_compatibility/test_insert_profile_events.py +++ b/tests/integration/test_backward_compatibility/test_insert_profile_events.py @@ -10,7 +10,7 @@ upstream_node = cluster.add_instance("upstream_node", use_old_analyzer=True) old_node = cluster.add_instance( "old_node", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, ) diff --git a/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py b/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py index 4752a589a44f..0b4178451986 100644 --- a/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py +++ b/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py @@ -6,7 +6,7 @@ # Version 21.6.3.14 has incompatible partition id for tables with UUID in partition key. node = cluster.add_instance( "node", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py index b13e6c975e80..c090b0f91be7 100644 --- a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py @@ -6,7 +6,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -14,7 +14,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py index 83be0e4c5a34..de8fcb3a940f 100644 --- a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py +++ b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py @@ -7,7 +7,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_parallel_replicas_protocol.py b/tests/integration/test_backward_compatibility/test_parallel_replicas_protocol.py new file mode 100644 index 000000000000..e1b9049ef5dc --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_parallel_replicas_protocol.py @@ -0,0 +1,64 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) +cluster_name = "parallel_replicas" +nodes = [ + cluster.add_instance( + f"node{num}", + main_configs=["configs/clusters.xml"], + with_zookeeper=False, + image="clickhouse/clickhouse-server", + tag="23.11", # earlier versions lead to "Not found column sum(a) in block." exception 🤷 + stay_alive=True, + use_old_analyzer=True, + with_installed_binary=True, + ) + for num in range(2) +] + [ + cluster.add_instance( + "node2", + main_configs=["configs/clusters.xml"], + with_zookeeper=False, + use_old_analyzer=True, + ) +] + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_backward_compatability(start_cluster): + for node in nodes: + node.query("create table t (a UInt64) engine = MergeTree order by tuple()") + node.query("insert into t select number % 100000 from numbers_mt(1000000)") + + # all we want is the query to run without errors + for node in nodes: + assert ( + node.query( + """ + select sum(a) + from t + """, + settings={ + "cluster_for_parallel_replicas": "parallel_replicas", + "max_parallel_replicas": 3, + "allow_experimental_parallel_reading_from_replicas": 1, + "parallel_replicas_for_non_replicated_merge_tree": 1, + }, + ) + == "49999500000\n" + ) + + for node in nodes: + node.query("drop table t") diff --git a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py index cbe147dc07bd..ba5d797982a3 100644 --- a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py +++ b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py @@ -7,7 +7,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py index ad573c7ffe06..97ee2b01c3f1 100644 --- a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py @@ -7,16 +7,18 @@ node1 = cluster.add_instance( "node1", with_zookeeper=False, - image="clickhouse/clickhouse-server", - tag="24.1" if is_arm() else CLICKHOUSE_CI_MIN_TESTED_VERSION, + # NOTE(vnemkov): here and below upstream uses clickhouse/clickhouse-server:24.1, + # the first Altinity Stable release after that was 24.3.5.47.altinitystable + image="altinity/clickhouse-server", + tag="24.3.5.47.altinitystable" if is_arm() else CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, ) node2 = cluster.add_instance( "node2", with_zookeeper=False, - image="clickhouse/clickhouse-server", - tag="24.1" if is_arm() else CLICKHOUSE_CI_MIN_TESTED_VERSION, + image="altinity/clickhouse-server", + tag="24.3.5.47.altinitystable" if is_arm() else CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, ) diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index e36c3310e4ad..9401091821d8 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -6,7 +6,7 @@ node_old = cluster.add_instance( "node1", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_disk_over_web_server/test.py b/tests/integration/test_disk_over_web_server/test.py index 891ee8f00f5f..9573eec6b5a6 100644 --- a/tests/integration/test_disk_over_web_server/test.py +++ b/tests/integration/test_disk_over_web_server/test.py @@ -37,7 +37,7 @@ def cluster(): with_nginx=True, stay_alive=True, with_installed_binary=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, ) cluster.add_instance( diff --git a/tests/integration/test_distributed_insert_backward_compatibility/test.py b/tests/integration/test_distributed_insert_backward_compatibility/test.py index 9e794555d49d..ed786eccc117 100644 --- a/tests/integration/test_distributed_insert_backward_compatibility/test.py +++ b/tests/integration/test_distributed_insert_backward_compatibility/test.py @@ -10,7 +10,7 @@ node_dist = cluster.add_instance( "node2", main_configs=["configs/remote_servers.xml"], - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index d74cf97e5c66..7f5c4c7841c8 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -17,6 +17,7 @@ def make_instance(name, *args, **kwargs): main_configs.append("configs/remote_servers.xml") user_configs = kwargs.pop("user_configs", []) user_configs.append("configs/users.xml") + return cluster.add_instance( name, with_zookeeper=True, diff --git a/tests/integration/test_hedged_requests/test.py b/tests/integration/test_hedged_requests/test.py index 02ecf3c1367d..0d72f7c45b10 100644 --- a/tests/integration/test_hedged_requests/test.py +++ b/tests/integration/test_hedged_requests/test.py @@ -333,7 +333,7 @@ def test_receive_timeout2(started_cluster): # in packet receiving but there are replicas in process of # connection establishing. update_configs( - node_1_sleep_in_send_data=4000, + node_1_sleep_in_send_data=5000, node_2_sleep_in_send_tables_status=2000, node_3_sleep_in_send_tables_status=2000, ) diff --git a/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt new file mode 100644 index 000000000000..05ce3ef8ee42 --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDtjCCAp6gAwIBAgIUdOfco+b8/fQZQOafHgghkEYL3YkwDQYJKoZIhvcNAQEL +BQAwazELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5 +MRAwDgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0 +Y2x1c3RlciBSb290IENBMB4XDTI0MTIwMjE0MjkyNloXDTM0MTEzMDE0MjkyNlow +ajELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5MRAw +DgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRswGQYDVQQDDBJjbGlja2hv +dXNlLWtlZXBlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUXmnn +Cv7sY9lbrS1Q3c5q7ok9R4XEPq/jWBFkIEnJR7vSjCEnOjLxg+1MdUItjbqODf9N +5vFbHiiqWQVkGrmg8/CTSme0qyNr7FcmG1hO4bzK/dvIyK1R7YISqZpXoCTVzEnU +IjU7f+PkX2uAiSypxM4zpNyC7++j6ah8xYNRfR9AS5c7e1dvNKBNMmNipYxVgaEo +pIke40m12ezIzLOtkL/rGlsnM2Tv/0Wv1xQE+OjHByyQE08vuliatFfweTXLF48m +4S4NdOq5dh2WX8xLPr8BxRLjXzs08wgKVFpWkIOR2uEInjuVQAGMuZeOqUuQGdar +GMH4M/3tDl0eJ7mbAgMBAAGjUzBRMB0GA1UdDgQWBBTr9ldBtTB0vatq2yhQgYtt +zMNhJDAfBgNVHSMEGDAWgBTGSqv6LHbQlKrpPWtYEVoX+/c5cTAPBgNVHRMBAf8E +BTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBYc03AV8n0D43xm07MxpgqDvNEZC9u +Q2LnP89UBnBmXD5FwMz4XhA/iupyAeYItZ8R17caIpLHgwOUrh3oHxVW5V144Q0p +hBBp/im8WQ8NnS3z52CusxE1Zu5AMjoZtxY8FTvgs6vuJZYds/dgtUg5bBawR2LX +A5FsPLyYpwCjoPTM622uXkuPfRMc5SC5edwHa1RyoG8Poz8B6Y63iKQydOXin9Q1 +9rQ7mqM7D2dCURx4gVoN9y+fLkXgQEzTMBT4wuVJl+CXnxcmKsoROAy7g2mL0RMw +P3cl+Bod3NrabhjAqG01nHsQzy0uJ/aJHbqoR3OtYo8DdsoKrBRramiG +-----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key new file mode 100644 index 000000000000..ac20648bb8b1 --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDUXmnnCv7sY9lb +rS1Q3c5q7ok9R4XEPq/jWBFkIEnJR7vSjCEnOjLxg+1MdUItjbqODf9N5vFbHiiq +WQVkGrmg8/CTSme0qyNr7FcmG1hO4bzK/dvIyK1R7YISqZpXoCTVzEnUIjU7f+Pk +X2uAiSypxM4zpNyC7++j6ah8xYNRfR9AS5c7e1dvNKBNMmNipYxVgaEopIke40m1 +2ezIzLOtkL/rGlsnM2Tv/0Wv1xQE+OjHByyQE08vuliatFfweTXLF48m4S4NdOq5 +dh2WX8xLPr8BxRLjXzs08wgKVFpWkIOR2uEInjuVQAGMuZeOqUuQGdarGMH4M/3t +Dl0eJ7mbAgMBAAECggEAJwzxZlXESI2Xw17VzV/r/Ae+3rDPLSXly+U+1W2Gg+eX +5wBzfDYcdgKvWPba42uDWWnDf3yu9vVVvvU9o4myhqE0pLDy3ur1SXwdDlnK5D5o +K9+AUaxtCnqlB29+fQxqmZHGJabgqP88VZsiNnGC7/jLff2buswKAdcOb1sWaZ5F +p5YBL6TmUzpg3Pdbs2N/OVsZGh5Y+d6m6hJgjsbdcWC4LrFurj9UyQoCGIRruelN +Ra3ft4QAC3biD0/hEc4WyjzZdlMvvEqaeQUmqK2TJLELwyH33W1Ek2FsLm6LTP9i +7kW2+9684GWBXo5ge87BXASVwAXnFE+pnXV5QGLMqQKBgQDpo2ZM/R/lWLR8Hpww +WKZZFfWPpsv0d6DbwVWowdRoojPue62nIby6+LBk08j9UwqkC0TK8pWCjL4hhSfU +JjLuGuzl+RFfstpt42qh2zgi8aedLtGHjFyjHp1jE3rb9l95YXUpNm7syEMbLR8V +NR09na41ftFCBPBNYFsdxrzKPwKBgQDosd6qwhrTL+ndtiJIeonRvunjo4yXn22/ +qWqRy3WJmZpDKWpsGWOmlJ4G7+10Q3zOMpb+nUOEjJNI3EdnLrVTeJo8WNuNXHyl +axgWV3TR8JT6GIG+zavGEI51JjTH3X1eGzm9T4Di8mj34FzyK1af6atiiRj5sIuk +NG71CUjhpQKBgQCDJ90n3vjm0LMQ8kYPxdQsMm2VZLcd14IPmyqw/45z5opsmDVV +m1TNSQoMr+8mdlWE3WaS3zcbAFNDkfJX39G7ZJYUS4t7Q3XnNkEH934975Z+YGfz +RdJDJ86GbcsMa/QQuasBpbMDbTBusxe92gE+M6Q2F6j0/LzBUxQTVRtqFQKBgBFZ +IXatnf5cthzXdVrd9+RxTVKxYMv1EOOXJ+DSwGKP1xZmwg5pHirPLbDqtlNSrL1a +vDMjWmNJb7mg4pnou5ALj8QsA8JYQNq8T0FrJ8R3IUQ8C4BEKShNF7HYNVspQi1/ +7iAVC1DgLb89NPDBFmY5r5NbEUecR+zoE9Wk6ZAZAoGBAIHTifNzf7/qDtHI2+Cc +YGiudMlWWwNqTUr1BjPQx1au1VImpDAB6eaz3DV0oIS0fpREte6SRrcwvtrRqp4M +AT4uCjiVOaXW/MwybfS6BIivTvuTkyPuNCBIWTH1JBQ3CEcEIIV5YcpFDQs5FQ0M +GPtHFxbKMUmLmJVW5nbKUUhr +-----END PRIVATE KEY----- diff --git a/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt new file mode 100644 index 000000000000..686cd6455548 --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDtjCCAp6gAwIBAgIUP0g0uMpZSD2OOtjFXz/anI4EU+swDQYJKoZIhvcNAQEL +BQAwazELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5 +MRAwDgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0 +Y2x1c3RlciBSb290IENBMB4XDTI0MTIwMjE0MjkyNloXDTM0MTEzMDE0MjkyNlow +ajELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5MRAw +DgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRswGQYDVQQDDBJjbGlja2hv +dXNlLWtlZXBlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCgflyz +Kg1deXEXFJIzoyLIAfRPs8MpOsKt06DPVvyZp2ct+g2GCcZlwV4L/GunIV2sugeX +ZHcJ+B06gKSgouxOMFjTnBEdlygLegMeyrJI6TKREiiWMYYxfUVabpC0DtKeZxc/ +D9BY4qLjngxbdRwS7l4eKv74jV9dowDfCNZxXLtzP3uj+AFlLuWk0LP6qFmJMUii +tM7f3oLzxURxIddBASjz12dyQGdm/6v6UcVWnqSDXApozb9LPmapUiJM9axcEvjM +C/Qr14021OEgLVGGEeAAA4JHWZPCqQjbgaDHm5xa61KAMnwDxk/GbMX/TFSwgV4x +pDChT9GKzVMHNf7PAgMBAAGjUzBRMB0GA1UdDgQWBBRZ8QY0I9WoGyFlwyGOs4ZY +cf5tEDAfBgNVHSMEGDAWgBTGSqv6LHbQlKrpPWtYEVoX+/c5cTAPBgNVHRMBAf8E +BTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB6hUw6IrDGBvGN3AIVatO/6xZX5LZM +Lp5B4uL5rz+6BXf+hZFsj3o4uvyxaEW12m+/bPPOA4EBdShtUfydfMoDHJsnrE8k +D6aVq04f7vjffGeFzvQhfAEnK5/rutWDyq9rXlqcKcPFLhl2Pozk7ty3V+Wz7i3+ +0n2uDTxAfcdlkeSlzPpXP/JOMFN6BwmzrgsyLHyPeIhjfv/lFMoAOblpF6tDFvlY +sXTk0P3Eh9zQ9vT2HI3ZVkqXUe3qQZhUOkKezy0J/OK/6wlvRoO3GXr8/gJO+lJp +ATwurpIc/za08toWbziOOL4xhY4RA+7S9uK3Uz+2a8AoRyUurMP4AHpx +-----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key new file mode 100644 index 000000000000..c4e956e31777 --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCgflyzKg1deXEX +FJIzoyLIAfRPs8MpOsKt06DPVvyZp2ct+g2GCcZlwV4L/GunIV2sugeXZHcJ+B06 +gKSgouxOMFjTnBEdlygLegMeyrJI6TKREiiWMYYxfUVabpC0DtKeZxc/D9BY4qLj +ngxbdRwS7l4eKv74jV9dowDfCNZxXLtzP3uj+AFlLuWk0LP6qFmJMUiitM7f3oLz +xURxIddBASjz12dyQGdm/6v6UcVWnqSDXApozb9LPmapUiJM9axcEvjMC/Qr1402 +1OEgLVGGEeAAA4JHWZPCqQjbgaDHm5xa61KAMnwDxk/GbMX/TFSwgV4xpDChT9GK +zVMHNf7PAgMBAAECgf9+Cl08oHSJPWifSeoBfvgCfJKel5pj60fT5kcO10Ghy6Vt +IMKisi8dKET8wz/IGcFe8RtmpR4UVK4NXB6YguDhALWEtwwntwfURKiokiWJ+HVD +8s4Fbyht/m1UTqUk23MG2xPgcorkBlWc/pqdaXOWjwpnUXVNXAJketvrKBHTbgq+ +XBvWlYNm+8ThGxm6Ryf3v6K04m5bVAzdQljGBUXRixqEZ71qkQ1TwEBaphbEWPLZ +7nod0yKxQ38ydYrrqqjoINvNLr4OBrlsjuYFAXbwwJmY0L5EleleVDjuiKmbCLUX +CKFaT+YlvHKY5m313ohVFldjRqThjuNt2VtuOFECgYEA0cyCH+sBTaMrYCgi8+l2 +nB+w9zVUqI70naT9zpBlei3DWgklMlNoGRbiIrvFyS2eH3GMq+WZub2/Ci0AU0RG +QYcvp4dfJth6IoKif8Un+RSMW1rN4pPDA88YIr+BKlnkjwfj+71ldBuX75UHXut1 +8z8ThxrmpMCOFql6S5odkSsCgYEAw9ZANtjTBpo2Dff8uPE+Ml3rilRWk3RjX+iU +UbKrsNWwU0QXQu4RbxHaCmJEPFP7bL6W/DkWSvzrxxnnmtQ2UXvXu1jf5D8asliz ++HMIbX3beiKec+C0xbt5xbOYkO4pfurS8V9fYsJTFyDDiMk9cDlHwKfH71yO2Qgp +Zw4Szu0CgYEAtb8awxfMyzsdanGaxf5r+BgkMCQNMPCWzLKQBRBmOI/IegkOJijH +N2TxhfFxCDTylH7DxG5k29ma0+/kJj4xNrcr+090iKxkMd1FdLaRSGAar1Fcpnon +KPeRCxknhk7Vh2rof761Uv5MgwpxljMYvR7ZheMyB2ugK9Wp0jCyiH0CgYAus9B6 +g/jHUU1kxWgKftWTU1yRj41Z+t6cB64fUZmqQTucj9dwSa/0qfAym76kGG8UPtto +6QBM/8YGpEHcZZFSm1MWRZqXJwlp0MeSj3RKEEKf/NOG1OanZQ8kO7E9lt5kewG1 +OEZaGfeQw2p+G2fAdJiM9DY0+gDC9zRQdEW9/QKBgQCIlZ6ToPdBJPuhKTtV0+c/ +e3k0+bKy3OIoW+laaRS3JMMyvveQECaLKMsVMYCppJfKCV/NOUnTIw/g1vXuom6I +QZHGK10aWB8bunblKFrSFxfBStgDCPeZklb4ECQP4+QfLb7Xi7fAvfSi0hFP4LNo +Ea+Ttp/shik8d0sy217IEQ== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml index 986b503ebe35..bc3d2a44ce29 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml @@ -1,5 +1,6 @@ + 0 9181 1 /var/lib/clickhouse/coordination/log @@ -10,6 +11,7 @@ 10000 75 trace + 1000 diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml index 652b1992f463..39eec44d0944 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml @@ -1,5 +1,6 @@ + 0 9181 2 /var/lib/clickhouse/coordination/log @@ -10,6 +11,7 @@ 10000 75 trace + 1000 diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml index 6507f97473b8..88a3d3674334 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml @@ -1,5 +1,6 @@ + 0 9181 3 /var/lib/clickhouse/coordination/log @@ -10,6 +11,7 @@ 10000 75 trace + 1000 diff --git a/tests/integration/test_keeper_internal_secure/configs/rootCA.pem b/tests/integration/test_keeper_internal_secure/configs/rootCA.pem index ec16533d98a2..208259f8f85e 100644 --- a/tests/integration/test_keeper_internal_secure/configs/rootCA.pem +++ b/tests/integration/test_keeper_internal_secure/configs/rootCA.pem @@ -1,21 +1,22 @@ -----BEGIN CERTIFICATE----- -MIIDazCCAlOgAwIBAgIUUiyhAav08YhTLfUIXLN/0Ln09n4wDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMTA0MTIxMTQ1MjBaFw0yMTA1 -MTIxMTQ1MjBaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw -HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQDK0Ww4voPlkePBPS2MsEi7e1ePS+CDxTdDuOwWWEA7 -JiOyqIGqdyL6AE2EqjL3sSdVFVxytpGQWDuM6JHXdb01AnMngBuql9Jkiln7i267 -v54HtMWdm8o3rik/b/mB+kkn/sP715tI49Ybh/RobtvtK16ZgHr1ombkq6rXiom2 -8GmSmpYFwZtZsXtm2JwbZVayupQpWwdu3KrTXKBtVyKVvvWdgkf47DWYtWDS3vqE -cShM1H97G4DvI+4RX1WtQevQ0yCx1aFTg4xMHFkpUxlP8iW6mQaQPqy9rnI57e3L -RHc2I/B56xa43R3GmQ2S7bE4hvm1SrZDtVgrZLf4nvwNAgMBAAGjUzBRMB0GA1Ud -DgQWBBQ4+o0x1FzK7nRbcnm2pNLwaywCdzAfBgNVHSMEGDAWgBQ4+o0x1FzK7nRb -cnm2pNLwaywCdzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQDE -YmM8MH6RKcaqMqCBefWLj0LTcZ/Wm4G/eCFC51PkAIsf7thnzViemBHRXUSF8wzc -1MBPD6II6OB1F0i7ntGjtlhnL2WcPYbo2Np59p7fo9SMbYwF49OZ40twsuKeeoAp -pfow+y/EBZqa99MY2q6FU6FDA3Rpv0Sdk+/5PHdsSP6cgeMszFBUS0tCQEvEl83n -FJUb0vjEX4x3J64XO/0DKXyCxFyF77OwHG2ZV5BeCpIhGXu+d/e221LJkGI2orKR -kgsaUwrkS8HQt3Hd0gYpLI1Opx/JlRpB0VLYLzRGj7kDpbAcTj3SMEUp/FAZmlXR -Iiebt73eE3rOWVFgyY9f +MIIDtzCCAp+gAwIBAgIUeJXILNkZb1FYvV7YnFYDB1OUrB4wDQYJKoZIhvcNAQEL +BQAwazELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5 +MRAwDgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0 +Y2x1c3RlciBSb290IENBMB4XDTI0MTIwMjE0MjkyNloXDTM0MTEzMDE0MjkyNlow +azELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5MRAw +DgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0Y2x1 +c3RlciBSb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvhBQ +1W4Swyw4g8VROLzDCYieR+6tyvUwkP/KyH9UapuCnQtZCaNhz6uCCouWONIV8LEx +Mk6YnHJKkBfsWx2C2dKQo3PFyroDa+9J08eFglZCCUcYqYJSzHne07fniIug37w2 +hekFWPbl8dYaNrnRNVUqkAHkFcxJA7JHnEPx+N0V58+2OJrq8bucTVA35oCq6Cjj +wBDJI9/puwtRpwTa3dcZ6bGFKArRKTKO5Nd6gufQKd2MrwXOOGFCltrPDbAUCbKU +UpphEmZIB7rPhCl3qkUgiFM8obgVGgw7E6UD1BKkCS42SFlONAdxnVKNTghN7RK3 +dmAmUYvtT1O7tj4BPQIDAQABo1MwUTAdBgNVHQ4EFgQUxkqr+ix20JSq6T1rWBFa +F/v3OXEwHwYDVR0jBBgwFoAUxkqr+ix20JSq6T1rWBFaF/v3OXEwDwYDVR0TAQH/ +BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAEyO49djQOI9qaHc0tvuAtio+qRqT +zQN5H8FJS4T7RDZSc3sXaUElY8hD2ecGZPDtxmFZy+IMtSGZcMfKlEr3pugrYwRh +571dFp1+o8wEmyOU0NHsGmSxCLZOk9nMxZEhLvc722B6oKHTIm3rvxkQqKpdfliE +oFW2QZVBteZ04A4AKOs0mkZptycZKMLiht5I2s1gzlbK9084huGmnayW9a3pKWyV +zpgnugfxEiwjKh7HpF9Mc2M4Z3f483bj/f6+G8Z7668dORQUFUwv/ohxN8w8zfgA +pfrQNknzYfihuFam6/CFzOsT9Nndtuz14N/LKI9csd2ixWTknHPLyMGrVA== -----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_internal_secure/configs/server.crt b/tests/integration/test_keeper_internal_secure/configs/server.crt deleted file mode 100644 index dfa32da54448..000000000000 --- a/tests/integration/test_keeper_internal_secure/configs/server.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDETCCAfkCFHL+gKBQnU0P73/nrFrGaVPauTPmMA0GCSqGSIb3DQEBCwUAMEUx -CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjEwNDEyMTE0NzI5WhcNMjEwNTEyMTE0 -NzI5WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE -CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEA1iPeYn1Vy4QnQi6uNVqQnFLr0u3qdrMjGEBNAOuGmtIdhIn8 -rMCzaehNr3y2YTMRbZAqmv28P/wOXpzR1uQaFlQzTOjmsn/HOZ9JX2hv5sBUv7SU -UiPJS7UtptKDPbLv3N/v1dOXbY+vVyzo8U1Q9OS1J5yhYW6KtxP++hfSrOsFu669 -d1pqWFWaNBsmf0zF+ETvi6lywhyTFA1/PazcStP5GntcDL7eDvGq+DDsRC40oRpy -S4xRQRSteCTtGGmWpx+Jmt+90wFnLgruUbWT0veCoLxLvz0tJUk3ueUVnMkrxBQG -Fz+IWm+SQppNU5LlAcBcu9wJfo3h34BXp0NFNQIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQCUnvQsv+GsPwGnIWqH9iiFVhgDx5QbSTW94Fyqk8dcIJBzWAiCshmLBWPJ -pfy4y2nxJbzovFsd9DA49pxqqILeLjue99yma2DVKeo+XDLDN3OX5faIMTBd7AnL -0MKqW7gUSLRUZrNOvFciAY8xRezgBQQBo4mcmmMbAbk5wKndGY6ZZOcY+JwXlqGB -5hyi6ishO8ciiZi3GMFNWWk9ViSfo27IqjKdSkQq1pr3FULvepd6SkdX+NvfZTAH -rG+CSoFGiJcOBbhDkvpY32cAJEnJOA1vHpFxfnGP8/1haeVZHqSwH1cySD78HVtF -fBs000wGHzBYWNI2KkwjNtYf06P4 ------END CERTIFICATE----- diff --git a/tests/integration/test_keeper_internal_secure/configs/server.key b/tests/integration/test_keeper_internal_secure/configs/server.key deleted file mode 100644 index 7e57c8b6b347..000000000000 --- a/tests/integration/test_keeper_internal_secure/configs/server.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA1iPeYn1Vy4QnQi6uNVqQnFLr0u3qdrMjGEBNAOuGmtIdhIn8 -rMCzaehNr3y2YTMRbZAqmv28P/wOXpzR1uQaFlQzTOjmsn/HOZ9JX2hv5sBUv7SU -UiPJS7UtptKDPbLv3N/v1dOXbY+vVyzo8U1Q9OS1J5yhYW6KtxP++hfSrOsFu669 -d1pqWFWaNBsmf0zF+ETvi6lywhyTFA1/PazcStP5GntcDL7eDvGq+DDsRC40oRpy -S4xRQRSteCTtGGmWpx+Jmt+90wFnLgruUbWT0veCoLxLvz0tJUk3ueUVnMkrxBQG -Fz+IWm+SQppNU5LlAcBcu9wJfo3h34BXp0NFNQIDAQABAoIBAHYDso2o8V2F6XTp -8QxqawQcFudaQztDonW9CjMVmks8vRPMUDqMwNP/OMEcBA8xa8tsBm8Ao3zH1suB -tYuujkn8AYHDYVDCZvN0u6UfE3yiRpKYXJ2gJ1HX+d7UaYvZT6P0rmKzh+LTqxhq -Ib7Kk3FDkirQgYgGueAH3x/JfUvaAGvFrq+HvvlhHOs7M7iFU4nJA8jNfBolpTnG -v5MMI+f8/GHGreVICJUoclE+4V/4LDHUlrc3l1kQk0keeD6ECw/pl48TNL6ncXKu -baez1rfKbMPjhLUy2q5UZa93oW+olchEOXs1nUNKUhIOOr0f0YweYhUHNTineVM9 -yTecMIkCgYEA7CFQMyeLVeBA6C9AHBe8Zf/k64cMPyr0lUz6548ulil580PNPbvW -kd2vIKfUMgCO5lMA47ArL4bXZ7cjTvJmPYE1Yv8z+F0Tk03fnTrudHOSBEiGXAu3 -MPTxCDU7Se5Dwj0Fq81aFRtCHl8Rrss+WiBD8eRoxb/vwXKFc6VUAWMCgYEA6CjZ -XrZz11lySBhjkyVXcdLj89hDZ+bPxA7b3VB7TfCxsn5xVck7U3TFkg5Z9XwEQ7Ob -XFAPuwT9GKm7QPp6L8T2RltoJ3ys40UH1RtcNLz2aIo/xSP7lopPdAfWHef5r4y9 -kRw+Gh4NP/l5wefXsRz/D0jY3+t+QnwnhuCKbocCgYEAiR6bPOlkvzyXVH1DxEyA -Sdb8b00f7nqaRyzJsrfxvJ9fQsWHpKa0ZkYOUW9ECLlMQjHHHXEK0vGBmqe9qDWY -63RhtRgvbLVYDb018k7rc9I846Hd7AudmJ9UbIjE4hyrWlsnNOntur32ej6IvTEn -Bx0fd5NEyDi6GGLRXiOOkbMCgYAressLE/yqDlR68CZl/o5cAPU0TAKDyRSMUYQX -9OTC+hstpMSxHlkADlSaQBnVAf8CdvbX2R65FfwYzGEHkGGl5KuDDcd57b2rathG -rzMbpXA4r/u1fkG2Nf0fbABL5ZA7so4mSTXQSmSM4LpO+I7K2vVh9XC4rzAcX4g/ -mHoUrQKBgBf3rxp5h9P3HWoZYjzBDo2FqXUjKLLjE9ed5e/VqecqfHIkmueuNHlN -xifHr7lpsYu6IXkTnlK14pvLoPuwP59dCIOUYwAFz8RlH4MSUGNhYeGA8cqRrhmJ -tYk3OKExuN/+O12kUPVTy6BMH1hBdRJP+7y7lapWsRhZt18y+8Za ------END RSA PRIVATE KEY----- diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.xml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.xml deleted file mode 100644 index 37eb2624b1b6..000000000000 --- a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - /etc/clickhouse-server/config.d/server.crt - /etc/clickhouse-server/config.d/server.key - /etc/clickhouse-server/config.d/rootCA.pem - true - none - true - sslv2,sslv3 - true - - - diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml new file mode 100644 index 000000000000..0a0a8d87507b --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml @@ -0,0 +1,18 @@ +openSSL: + server: + certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' + privateKeyFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.key' + caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' + loadDefaultCAFile: true + verificationMode: 'none' + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true + client: + certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' + caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' + loadDefaultCAFile: true + verificationMode: 'none' + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml new file mode 100644 index 000000000000..d55d729b3460 --- /dev/null +++ b/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml @@ -0,0 +1,22 @@ +openSSL: + server: + certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' + privateKeyFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.key' + caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' + privateKeyPassphraseHandler: + name: KeyFileHandler + options: + password: 'PASSWORD' + loadDefaultCAFile: true + verificationMode: 'none' + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true + client: + certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' + caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' + loadDefaultCAFile: true + verificationMode: 'none' + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true diff --git a/tests/integration/test_keeper_internal_secure/test.py b/tests/integration/test_keeper_internal_secure/test.py index 2d45e95e4fff..c08df4db856d 100644 --- a/tests/integration/test_keeper_internal_secure/test.py +++ b/tests/integration/test_keeper_internal_secure/test.py @@ -2,44 +2,55 @@ import pytest from helpers.cluster import ClickHouseCluster -import random -import string +import helpers.keeper_utils as ku +from multiprocessing.dummy import Pool import os -import time +CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance( - "node1", - main_configs=[ - "configs/enable_secure_keeper1.xml", - "configs/ssl_conf.xml", - "configs/server.crt", - "configs/server.key", - "configs/rootCA.pem", - ], -) -node2 = cluster.add_instance( - "node2", - main_configs=[ - "configs/enable_secure_keeper2.xml", - "configs/ssl_conf.xml", - "configs/server.crt", - "configs/server.key", - "configs/rootCA.pem", - ], -) -node3 = cluster.add_instance( - "node3", - main_configs=[ - "configs/enable_secure_keeper3.xml", - "configs/ssl_conf.xml", - "configs/server.crt", - "configs/server.key", - "configs/rootCA.pem", - ], -) - -from kazoo.client import KazooClient, KazooState +nodes = [ + cluster.add_instance( + "node1", + main_configs=[ + "configs/enable_secure_keeper1.xml", + "configs/ssl_conf.yml", + "configs/WithoutPassPhrase.crt", + "configs/WithoutPassPhrase.key", + "configs/WithPassPhrase.crt", + "configs/WithPassPhrase.key", + "configs/rootCA.pem", + ], + stay_alive=True, + ), + cluster.add_instance( + "node2", + main_configs=[ + "configs/enable_secure_keeper2.xml", + "configs/ssl_conf.yml", + "configs/WithoutPassPhrase.crt", + "configs/WithoutPassPhrase.key", + "configs/WithPassPhrase.crt", + "configs/WithPassPhrase.key", + "configs/rootCA.pem", + ], + stay_alive=True, + ), + cluster.add_instance( + "node3", + main_configs=[ + "configs/enable_secure_keeper3.xml", + "configs/ssl_conf.yml", + "configs/WithoutPassPhrase.crt", + "configs/WithoutPassPhrase.key", + "configs/WithPassPhrase.crt", + "configs/WithPassPhrase.key", + "configs/rootCA.pem", + ], + stay_alive=True, + ), +] + +from kazoo.client import KazooClient @pytest.fixture(scope="module") @@ -61,23 +72,117 @@ def get_fake_zk(nodename, timeout=30.0): return _fake_zk_instance -def test_secure_raft_works(started_cluster): +def run_test(): + node_zks = [] try: - node1_zk = get_fake_zk("node1") - node2_zk = get_fake_zk("node2") - node3_zk = get_fake_zk("node3") + for node in nodes: + node_zks.append(get_fake_zk(node.name)) - node1_zk.create("/test_node", b"somedata1") - node2_zk.sync("/test_node") - node3_zk.sync("/test_node") + node_zks[0].create("/test_node", b"somedata1") + node_zks[1].sync("/test_node") + node_zks[2].sync("/test_node") - assert node1_zk.exists("/test_node") is not None - assert node2_zk.exists("/test_node") is not None - assert node3_zk.exists("/test_node") is not None + for node_zk in node_zks: + assert node_zk.exists("/test_node") is not None finally: try: - for zk_conn in [node1_zk, node2_zk, node3_zk]: + for zk_conn in node_zks: + if zk_conn is None: + continue zk_conn.stop() zk_conn.close() except: pass + + +def setupSsl(node, filename, password): + if password is None: + node.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs/ssl_conf.yml"), + "/etc/clickhouse-server/config.d/ssl_conf.yml", + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/ssl_conf.yml", + "WithoutPassPhrase", + filename, + ) + return + + node.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs/ssl_conf_password.yml"), + "/etc/clickhouse-server/config.d/ssl_conf.yml", + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/ssl_conf.yml", + "WithoutPassPhrase", + filename, + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/ssl_conf.yml", + "PASSWORD", + password, + ) + + +def stop_all_clickhouse(): + for node in nodes: + node.stop_clickhouse() + + for node in nodes: + node.exec_in_container(["rm", "-rf", "/var/lib/clickhouse/coordination"]) + + +def start_clickhouse(node): + node.start_clickhouse() + + +def start_all_clickhouse(): + p = Pool(3) + waiters = [] + + for node in nodes: + waiters.append(p.apply_async(start_clickhouse, args=(node,))) + + for waiter in waiters: + waiter.wait() + + for node in nodes: + ku.wait_until_connected(cluster, node) + + +def check_valid_configuration(filename, password): + stop_all_clickhouse() + for node in nodes: + setupSsl(node, filename, password) + start_all_clickhouse() + nodes[0].wait_for_log_line( + "Raft ASIO listener initiated on :::9234, SSL enabled", look_behind_lines=1000 + ) + run_test() + + +def check_invalid_configuration(filename, password): + stop_all_clickhouse() + for node in nodes: + setupSsl(node, filename, password) + + nodes[0].start_clickhouse() + nodes[0].wait_for_log_line( + "Raft ASIO listener initiated on :::9234, SSL enabled", look_behind_lines=1000 + ) + nodes[0].wait_for_log_line("failed to connect to peer.*Connection refused") + + +def test_secure_raft_works(started_cluster): + check_valid_configuration("WithoutPassPhrase", None) + + +def test_secure_raft_works_with_password(started_cluster): + check_valid_configuration("WithoutPassPhrase", "unusedpassword") + check_invalid_configuration("WithPassPhrase", "wrongpassword") + check_invalid_configuration("WithPassPhrase", "") + check_valid_configuration("WithPassPhrase", "test") + check_invalid_configuration("WithPassPhrase", None) diff --git a/tests/integration/test_ldap_external_user_directory/configs/remote_servers.xml b/tests/integration/test_ldap_external_user_directory/configs/remote_servers.xml new file mode 100644 index 000000000000..cf1bdf9dcb19 --- /dev/null +++ b/tests/integration/test_ldap_external_user_directory/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + + instance1 + 9000 + + + + + instance2 + 9000 + + + + + \ No newline at end of file diff --git a/tests/integration/test_ldap_external_user_directory/test.py b/tests/integration/test_ldap_external_user_directory/test.py index 39753794d633..0315c02b532e 100644 --- a/tests/integration/test_ldap_external_user_directory/test.py +++ b/tests/integration/test_ldap_external_user_directory/test.py @@ -7,8 +7,22 @@ LDAP_ADMIN_PASSWORD = "clickhouse" cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance( - "instance", main_configs=["configs/ldap_with_role_mapping.xml"], with_ldap=True + +instance1 = cluster.add_instance( + "instance1", + main_configs=["configs/ldap_with_role_mapping.xml", "configs/remote_servers.xml"], + macros={"shard": 1, "replica": "instance1"}, + stay_alive=True, + with_ldap=True, + with_zookeeper=True, +) + +instance2 = cluster.add_instance( + "instance2", + main_configs=["configs/remote_servers.xml"], + macros={"shard": 1, "replica": "instance2"}, + stay_alive=True, + with_zookeeper=True, ) @@ -48,48 +62,143 @@ def add_ldap_group(ldap_cluster, group_cn, member_cn): assert code == 0 +def delete_ldap_group(ldap_cluster, group_cn): + code, (stdout, stderr) = ldap_cluster.ldap_container.exec_run( + [ + "sh", + "-c", + """ldapdelete -r 'cn={group_cn},dc=example,dc=org' \ +-H ldap://{host}:{port} -D "{admin_bind_dn}" -x -w {admin_password} + """.format( + host=ldap_cluster.ldap_host, + port=ldap_cluster.ldap_port, + admin_bind_dn=LDAP_ADMIN_BIND_DN, + admin_password=LDAP_ADMIN_PASSWORD, + group_cn=group_cn, + ), + ], + demux=True, + ) + logging.debug( + f"test_ldap_external_user_directory code:{code} stdout:{stdout}, stderr:{stderr}" + ) + assert code == 0 + + def test_authentication_pass(): - assert instance.query( - "select currentUser()", user="janedoe", password="qwerty" + assert instance1.query( + "SELECT currentUser()", user="janedoe", password="qwerty" ) == TSV([["janedoe"]]) def test_authentication_fail(): # User doesn't exist. - assert "doesnotexist: Authentication failed" in instance.query_and_get_error( + assert "doesnotexist: Authentication failed" in instance1.query_and_get_error( "SELECT currentUser()", user="doesnotexist" ) # Wrong password. - assert "janedoe: Authentication failed" in instance.query_and_get_error( + assert "janedoe: Authentication failed" in instance1.query_and_get_error( "SELECT currentUser()", user="janedoe", password="123" ) def test_role_mapping(ldap_cluster): - instance.query("CREATE ROLE role_1") - instance.query("CREATE ROLE role_2") + instance1.query("DROP ROLE IF EXISTS role_1") + instance1.query("DROP ROLE IF EXISTS role_2") + instance1.query("DROP ROLE IF EXISTS role_3") + instance1.query("CREATE ROLE role_1") + instance1.query("CREATE ROLE role_2") add_ldap_group(ldap_cluster, group_cn="clickhouse-role_1", member_cn="johndoe") add_ldap_group(ldap_cluster, group_cn="clickhouse-role_2", member_cn="johndoe") - assert instance.query( + assert instance1.query( "select currentUser()", user="johndoe", password="qwertz" ) == TSV([["johndoe"]]) - assert instance.query( + assert instance1.query( "select role_name from system.current_roles ORDER BY role_name", user="johndoe", password="qwertz", ) == TSV([["role_1"], ["role_2"]]) - instance.query("CREATE ROLE role_3") + instance1.query("CREATE ROLE role_3") add_ldap_group(ldap_cluster, group_cn="clickhouse-role_3", member_cn="johndoe") # Check that non-existing role in ClickHouse is ignored during role update # See https://github.com/ClickHouse/ClickHouse/issues/54318 add_ldap_group(ldap_cluster, group_cn="clickhouse-role_4", member_cn="johndoe") - assert instance.query( + assert instance1.query( "select role_name from system.current_roles ORDER BY role_name", user="johndoe", password="qwertz", ) == TSV([["role_1"], ["role_2"], ["role_3"]]) + + instance1.query("DROP ROLE role_1") + instance1.query("DROP ROLE role_2") + instance1.query("DROP ROLE role_3") + + delete_ldap_group(ldap_cluster, group_cn="clickhouse-role_1") + delete_ldap_group(ldap_cluster, group_cn="clickhouse-role_2") + delete_ldap_group(ldap_cluster, group_cn="clickhouse-role_3") + delete_ldap_group(ldap_cluster, group_cn="clickhouse-role_4") + + +def test_push_role_to_other_nodes(ldap_cluster): + instance1.query("DROP TABLE IF EXISTS distributed_table SYNC") + instance1.query("DROP TABLE IF EXISTS local_table SYNC") + instance2.query("DROP TABLE IF EXISTS local_table SYNC") + instance1.query("DROP ROLE IF EXISTS role_read") + + instance1.query("CREATE ROLE role_read") + instance1.query("GRANT SELECT ON *.* TO role_read") + + add_ldap_group(ldap_cluster, group_cn="clickhouse-role_read", member_cn="johndoe") + + assert instance1.query( + "select currentUser()", user="johndoe", password="qwertz" + ) == TSV([["johndoe"]]) + + instance1.query( + "CREATE TABLE IF NOT EXISTS local_table (id UInt32) ENGINE = MergeTree() ORDER BY id" + ) + instance2.query( + "CREATE TABLE IF NOT EXISTS local_table (id UInt32) ENGINE = MergeTree() ORDER BY id" + ) + instance2.query("INSERT INTO local_table VALUES (1), (2), (3)") + instance1.query( + "CREATE TABLE IF NOT EXISTS distributed_table AS local_table ENGINE = Distributed(test_ldap_cluster, default, local_table)" + ) + + result = instance1.query( + "SELECT sum(id) FROM distributed_table", user="johndoe", password="qwertz" + ) + assert result.strip() == "6" + + instance1.query("DROP TABLE IF EXISTS distributed_table SYNC") + instance1.query("DROP TABLE IF EXISTS local_table SYNC") + instance2.query("DROP TABLE IF EXISTS local_table SYNC") + instance2.query("DROP ROLE IF EXISTS role_read") + + delete_ldap_group(ldap_cluster, group_cn="clickhouse-role_read") + + +def test_remote_query_user_does_not_exist_locally(ldap_cluster): + """ + Check that even if user does not exist locally, using it to execute remote queries is still possible + """ + instance2.query("DROP USER IF EXISTS non_local") + instance2.query("DROP TABLE IF EXISTS test_table sync") + + instance2.query("CREATE USER non_local") + instance2.query("CREATE TABLE test_table (id Int16) ENGINE=Memory") + instance2.query("INSERT INTO test_table VALUES (123)") + instance2.query("GRANT SELECT ON default.test_table TO non_local") + + result = instance1.query( + "SELECT * FROM remote('instance2', 'default.test_table', 'non_local')" + ) + assert result.strip() == "123" + + instance2.query("DROP USER IF EXISTS non_local") + instance2.query("DROP TABLE IF EXISTS test_table SYNC") diff --git a/tests/integration/test_old_versions/test.py b/tests/integration/test_old_versions/test.py index a5e62a380bfd..39ea7a78e815 100644 --- a/tests/integration/test_old_versions/test.py +++ b/tests/integration/test_old_versions/test.py @@ -6,7 +6,7 @@ cluster = ClickHouseCluster(__file__) node_oldest = cluster.add_instance( "node_oldest", - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"], diff --git a/tests/integration/test_parallel_replicas_all_marks_read/test.py b/tests/integration/test_parallel_replicas_all_marks_read/test.py index 7776ccb0c095..e67da7fe02de 100644 --- a/tests/integration/test_parallel_replicas_all_marks_read/test.py +++ b/tests/integration/test_parallel_replicas_all_marks_read/test.py @@ -54,7 +54,7 @@ def _get_result_without_parallel_replicas(query): return nodes[0].query( query, settings={ - "allow_experimental_parallel_reading_from_replicas": 0, + "enable_parallel_replicas": 0, }, ) @@ -65,7 +65,7 @@ def _get_result_with_parallel_replicas( return nodes[0].query( query, settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "max_parallel_replicas": 6, "cluster_for_parallel_replicas": f"{cluster_name}", "parallel_replicas_mark_segment_size": parallel_replicas_mark_segment_size, diff --git a/tests/integration/test_parallel_replicas_custom_key/test.py b/tests/integration/test_parallel_replicas_custom_key/test.py index 375fe58d7417..39fd7ff292e6 100644 --- a/tests/integration/test_parallel_replicas_custom_key/test.py +++ b/tests/integration/test_parallel_replicas_custom_key/test.py @@ -37,13 +37,13 @@ def insert_data(table_name, row_num, all_nodes=False): @pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"]) -@pytest.mark.parametrize("filter_type", ["default", "range"]) +@pytest.mark.parametrize("parallel_replicas_mode", ["custom_key_sampling"]) @pytest.mark.parametrize( "cluster", ["test_multiple_shards_multiple_replicas", "test_single_shard_multiple_replicas"], ) def test_parallel_replicas_custom_key_distributed( - start_cluster, cluster, custom_key, filter_type + start_cluster, cluster, custom_key, parallel_replicas_mode ): for node in nodes: node.rotate_logs() @@ -84,7 +84,8 @@ def test_parallel_replicas_custom_key_distributed( settings={ "max_parallel_replicas": 4, "parallel_replicas_custom_key": custom_key, - "parallel_replicas_custom_key_filter_type": filter_type, + "enable_parallel_replicas": 1, + "parallel_replicas_mode": parallel_replicas_mode, "prefer_localhost_replica": 0, }, ) @@ -100,13 +101,15 @@ def test_parallel_replicas_custom_key_distributed( @pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"]) -@pytest.mark.parametrize("filter_type", ["default", "range"]) +@pytest.mark.parametrize( + "parallel_replicas_mode", ["custom_key_sampling", "custom_key_range"] +) @pytest.mark.parametrize( "cluster", ["test_single_shard_multiple_replicas"], ) def test_parallel_replicas_custom_key_mergetree( - start_cluster, cluster, custom_key, filter_type + start_cluster, cluster, custom_key, parallel_replicas_mode ): for node in nodes: node.rotate_logs() @@ -130,8 +133,9 @@ def test_parallel_replicas_custom_key_mergetree( "SELECT key, count() FROM test_table_for_mt GROUP BY key ORDER BY key", settings={ "max_parallel_replicas": 4, + "enable_parallel_replicas": 1, "parallel_replicas_custom_key": custom_key, - "parallel_replicas_custom_key_filter_type": filter_type, + "parallel_replicas_mode": parallel_replicas_mode, "parallel_replicas_for_non_replicated_merge_tree": 1, "cluster_for_parallel_replicas": cluster, }, @@ -141,13 +145,15 @@ def test_parallel_replicas_custom_key_mergetree( @pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"]) -@pytest.mark.parametrize("filter_type", ["default", "range"]) +@pytest.mark.parametrize( + "parallel_replicas_mode", ["custom_key_sampling", "custom_key_range"] +) @pytest.mark.parametrize( "cluster", ["test_single_shard_multiple_replicas"], ) def test_parallel_replicas_custom_key_replicatedmergetree( - start_cluster, cluster, custom_key, filter_type + start_cluster, cluster, custom_key, parallel_replicas_mode ): for node in nodes: node.rotate_logs() @@ -174,8 +180,9 @@ def test_parallel_replicas_custom_key_replicatedmergetree( "SELECT key, count() FROM test_table_for_rmt GROUP BY key ORDER BY key", settings={ "max_parallel_replicas": 4, + "enable_parallel_replicas": 1, "parallel_replicas_custom_key": custom_key, - "parallel_replicas_custom_key_filter_type": filter_type, + "parallel_replicas_mode": parallel_replicas_mode, "cluster_for_parallel_replicas": cluster, }, ) diff --git a/tests/integration/test_parallel_replicas_custom_key_failover/test.py b/tests/integration/test_parallel_replicas_custom_key_failover/test.py index f24a24f32387..d46f7955588c 100644 --- a/tests/integration/test_parallel_replicas_custom_key_failover/test.py +++ b/tests/integration/test_parallel_replicas_custom_key_failover/test.py @@ -52,13 +52,15 @@ def create_tables(cluster, table_name): @pytest.mark.parametrize("use_hedged_requests", [1, 0]) @pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"]) -@pytest.mark.parametrize("filter_type", ["default", "range"]) +@pytest.mark.parametrize( + "parallel_replicas_mode", ["custom_key_sampling", "custom_key_range"] +) @pytest.mark.parametrize("prefer_localhost_replica", [0, 1]) def test_parallel_replicas_custom_key_failover( start_cluster, use_hedged_requests, custom_key, - filter_type, + parallel_replicas_mode, prefer_localhost_replica, ): cluster_name = "test_single_shard_multiple_replicas" @@ -77,8 +79,9 @@ def test_parallel_replicas_custom_key_failover( settings={ "log_comment": log_comment, "max_parallel_replicas": 4, + "enable_parallel_replicas": 1, "parallel_replicas_custom_key": custom_key, - "parallel_replicas_custom_key_filter_type": filter_type, + "parallel_replicas_mode": parallel_replicas_mode, "use_hedged_requests": use_hedged_requests, "prefer_localhost_replica": prefer_localhost_replica, # avoid considering replica delay on connection choice diff --git a/tests/integration/test_parallel_replicas_custom_key_load_balancing/test.py b/tests/integration/test_parallel_replicas_custom_key_load_balancing/test.py index ddfd07a08645..3f462312967b 100644 --- a/tests/integration/test_parallel_replicas_custom_key_load_balancing/test.py +++ b/tests/integration/test_parallel_replicas_custom_key_load_balancing/test.py @@ -56,12 +56,14 @@ def create_tables(table_name): @pytest.mark.parametrize("use_hedged_requests", [1, 0]) @pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"]) -@pytest.mark.parametrize("filter_type", ["default", "range"]) +@pytest.mark.parametrize( + "parallel_replicas_mode", ["custom_key_sampling", "custom_key_range"] +) def test_parallel_replicas_custom_key_load_balancing( start_cluster, use_hedged_requests, custom_key, - filter_type, + parallel_replicas_mode, ): cluster_name = "test_single_shard_multiple_replicas" table = "test_table" @@ -80,8 +82,9 @@ def test_parallel_replicas_custom_key_load_balancing( "log_comment": log_comment, "prefer_localhost_replica": 0, "max_parallel_replicas": 4, + "enable_parallel_replicas": 1, "parallel_replicas_custom_key": custom_key, - "parallel_replicas_custom_key_filter_type": filter_type, + "parallel_replicas_mode": parallel_replicas_mode, "use_hedged_requests": use_hedged_requests, # avoid considering replica delay on connection choice # otherwise connection can be not distributed evenly among available nodes diff --git a/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py b/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py index 0f54d053ce1d..0c7e820114ac 100644 --- a/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py +++ b/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py @@ -85,7 +85,7 @@ def test_skip_unavailable_shards(start_cluster, prefer_localhost_replica): node1.query( f"SELECT count(), min(key), max(key), sum(key) FROM {table_name}_d", settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "max_parallel_replicas": 3, "prefer_localhost_replica": prefer_localhost_replica, "skip_unavailable_shards": 1, @@ -119,7 +119,7 @@ def test_error_on_unavailable_shards(start_cluster, prefer_localhost_replica): node1.query( f"SELECT count(), min(key), max(key), sum(key) FROM {table_name}_d", settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "max_parallel_replicas": 3, "prefer_localhost_replica": prefer_localhost_replica, "skip_unavailable_shards": 0, @@ -154,7 +154,7 @@ def test_no_unavailable_shards(start_cluster, skip_unavailable_shards): node1.query( f"SELECT count(), min(key), max(key), sum(key) FROM {table_name}_d", settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "max_parallel_replicas": 3, "prefer_localhost_replica": 0, "skip_unavailable_shards": skip_unavailable_shards, diff --git a/tests/integration/test_parallel_replicas_failover/test.py b/tests/integration/test_parallel_replicas_failover/test.py index bf25136bff7f..340aa273811c 100644 --- a/tests/integration/test_parallel_replicas_failover/test.py +++ b/tests/integration/test_parallel_replicas_failover/test.py @@ -1,4 +1,5 @@ import pytest +import uuid from helpers.cluster import ClickHouseCluster @@ -72,7 +73,7 @@ def test_skip_replicas_without_table(start_cluster): node1.query( f"SELECT key, count() FROM {table_name} GROUP BY key ORDER BY key", settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "max_parallel_replicas": 3, "cluster_for_parallel_replicas": cluster_name, "log_comment": log_comment, @@ -84,7 +85,7 @@ def test_skip_replicas_without_table(start_cluster): node1.query("SYSTEM FLUSH LOGS") assert ( node1.query( - f"SELECT ProfileEvents['DistributedConnectionMissingTable'], ProfileEvents['ParallelReplicasUnavailableCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '{log_comment}' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0" + f"SELECT ProfileEvents['DistributedConnectionMissingTable'], ProfileEvents['ParallelReplicasUnavailableCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '{log_comment}' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS enable_parallel_replicas=0" ) == "1\t1\n" ) @@ -105,7 +106,7 @@ def test_skip_unresponsive_replicas(start_cluster): node1.query( f"SELECT key, count() FROM {table_name} GROUP BY key ORDER BY key", settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "max_parallel_replicas": 3, "cluster_for_parallel_replicas": cluster_name, }, diff --git a/tests/integration/test_parallel_replicas_invisible_parts/test.py b/tests/integration/test_parallel_replicas_invisible_parts/test.py index cab3fb46fe92..bc0c96fc56c1 100644 --- a/tests/integration/test_parallel_replicas_invisible_parts/test.py +++ b/tests/integration/test_parallel_replicas_invisible_parts/test.py @@ -81,7 +81,7 @@ def _get_result_with_parallel_replicas( return nodes[0].query( query, settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "max_parallel_replicas": len(nodes), "cluster_for_parallel_replicas": f"{cluster_name}", "parallel_replicas_mark_segment_size": parallel_replicas_mark_segment_size, diff --git a/tests/integration/test_parallel_replicas_no_replicas/test.py b/tests/integration/test_parallel_replicas_no_replicas/test.py index 62d4b005d94c..4c25315202c5 100644 --- a/tests/integration/test_parallel_replicas_no_replicas/test.py +++ b/tests/integration/test_parallel_replicas_no_replicas/test.py @@ -45,9 +45,10 @@ def test_skip_all_replicas( initiator.query( f"SELECT key, count() FROM {table_name} GROUP BY key ORDER BY key", settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "max_parallel_replicas": max_parallel_replicas, "cluster_for_parallel_replicas": cluster_name, "skip_unavailable_shards": skip_unavailable_shards, + "parallel_replicas_local_plan": 0, }, ) diff --git a/tests/integration/test_parallel_replicas_over_distributed/test.py b/tests/integration/test_parallel_replicas_over_distributed/test.py index 3d35ec3fa8e1..2752f1cb00f5 100644 --- a/tests/integration/test_parallel_replicas_over_distributed/test.py +++ b/tests/integration/test_parallel_replicas_over_distributed/test.py @@ -137,7 +137,7 @@ def test_parallel_replicas_over_distributed( node.query( f"SELECT count(), min(key), max(key), sum(key) FROM {table_name}_d", settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "prefer_localhost_replica": prefer_localhost_replica, "max_parallel_replicas": max_parallel_replicas, }, @@ -150,7 +150,7 @@ def test_parallel_replicas_over_distributed( node.query( f"SELECT count(), min(key), max(key), sum(key) FROM {table_name}_d", settings={ - "allow_experimental_parallel_reading_from_replicas": 0, + "enable_parallel_replicas": 0, }, ) == expected_result diff --git a/tests/integration/test_parallel_replicas_protocol/__init__.py b/tests/integration/test_parallel_replicas_protocol/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_parallel_replicas_protocol/configs/remote_servers.xml b/tests/integration/test_parallel_replicas_protocol/configs/remote_servers.xml new file mode 100644 index 000000000000..9fc2f119898a --- /dev/null +++ b/tests/integration/test_parallel_replicas_protocol/configs/remote_servers.xml @@ -0,0 +1,21 @@ + + + + + false + + node0 + 9000 + + + node1 + 9000 + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_parallel_replicas_protocol/test.py b/tests/integration/test_parallel_replicas_protocol/test.py new file mode 100644 index 000000000000..c43226c5ecc9 --- /dev/null +++ b/tests/integration/test_parallel_replicas_protocol/test.py @@ -0,0 +1,74 @@ +import pytest +import re + +from helpers.cluster import ClickHouseCluster +from random import randint + +cluster = ClickHouseCluster(__file__) + +nodes = [ + cluster.add_instance( + f"node{num}", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, + macros={"replica": f"node{num}", "shard": "shard"}, + ) + for num in range(3) +] + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def _create_tables(table_name): + nodes[0].query( + f"DROP TABLE IF EXISTS {table_name} ON CLUSTER 'parallel_replicas'", + settings={"database_atomic_wait_for_drop_and_detach_synchronously": True}, + ) + + # big number of granules + low total size in bytes = super tiny granules = big min_marks_per_task + # => big mark_segment_size will be chosen. it is not required to be big, just not equal to the default + nodes[0].query( + f""" + CREATE TABLE {table_name} ON CLUSTER 'parallel_replicas' (value Int64) + Engine=ReplicatedMergeTree('/test_parallel_replicas/shard/{table_name}', '{{replica}}') + ORDER BY () + SETTINGS index_granularity = 1 + """ + ) + + nodes[0].query(f"INSERT INTO {table_name} SELECT 42 FROM numbers(1000)") + nodes[0].query(f"SYSTEM SYNC REPLICA ON CLUSTER 'parallel_replicas' {table_name}") + + +# now mark_segment_size is part of the protocol and is communicated to the initiator. +# let's check that the correct value is actually used by the coordinator +def test_mark_segment_size_communicated_correctly(start_cluster): + table_name = "t" + _create_tables(table_name) + + for local_plan in [0, 1]: + query_id = f"query_id_{randint(0, 1000000)}" + nodes[0].query( + f"SELECT sum(value) FROM {table_name}", + settings={ + "allow_experimental_parallel_reading_from_replicas": 2, + "max_parallel_replicas": 100, + "cluster_for_parallel_replicas": "parallel_replicas", + "parallel_replicas_mark_segment_size": 0, + "parallel_replicas_local_plan": local_plan, + "query_id": query_id, + }, + ) + + nodes[0].query("SYSTEM FLUSH LOGS") + log_line = nodes[0].grep_in_log( + f"{query_id}.*Reading state is fully initialized" + ) + assert re.search(r"mark_segment_size: (\d+)", log_line).group(1) == "16384" diff --git a/tests/integration/test_parallel_replicas_skip_shards/test.py b/tests/integration/test_parallel_replicas_skip_shards/test.py index a18c82a53a94..0a0a16915428 100644 --- a/tests/integration/test_parallel_replicas_skip_shards/test.py +++ b/tests/integration/test_parallel_replicas_skip_shards/test.py @@ -25,7 +25,7 @@ def test_skip_unavailable_shards(start_cluster): node1.query( "SELECT hostName() as h FROM clusterAllReplicas('two_shards', system.one) order by h", settings={ - "allow_experimental_parallel_reading_from_replicas": 0, + "enable_parallel_replicas": 0, "skip_unavailable_shards": 1, }, ) @@ -36,7 +36,7 @@ def test_skip_unavailable_shards(start_cluster): node1.query( "SELECT hostName() as h FROM clusterAllReplicas('two_shards', system.one) order by h", settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "max_parallel_replicas": 3, "skip_unavailable_shards": 1, # "async_socket_for_remote" : 0, @@ -53,7 +53,7 @@ def test_error_on_unavailable_shards(start_cluster): node1.query( "SELECT hostName() as h FROM clusterAllReplicas('two_shards', system.one) order by h", settings={ - "allow_experimental_parallel_reading_from_replicas": 0, + "enable_parallel_replicas": 0, "skip_unavailable_shards": 0, }, ) @@ -62,7 +62,7 @@ def test_error_on_unavailable_shards(start_cluster): node1.query( "SELECT hostName() as h FROM clusterAllReplicas('two_shards', system.one) order by h", settings={ - "allow_experimental_parallel_reading_from_replicas": 2, + "enable_parallel_replicas": 2, "max_parallel_replicas": 3, "skip_unavailable_shards": 0, }, diff --git a/tests/integration/test_parallel_replicas_snapshot_from_initiator/__init__.py b/tests/integration/test_parallel_replicas_snapshot_from_initiator/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_parallel_replicas_snapshot_from_initiator/configs/remote_servers.xml b/tests/integration/test_parallel_replicas_snapshot_from_initiator/configs/remote_servers.xml new file mode 100644 index 000000000000..734acf5f3639 --- /dev/null +++ b/tests/integration/test_parallel_replicas_snapshot_from_initiator/configs/remote_servers.xml @@ -0,0 +1,33 @@ + + + + + false + + node0 + 9000 + + + node1 + 9000 + + + node2 + 9000 + + + node3 + 9000 + + + node4 + 9000 + + + node5 + 9000 + + + + + diff --git a/tests/integration/test_parallel_replicas_snapshot_from_initiator/test.py b/tests/integration/test_parallel_replicas_snapshot_from_initiator/test.py new file mode 100644 index 000000000000..a7e7e99455bc --- /dev/null +++ b/tests/integration/test_parallel_replicas_snapshot_from_initiator/test.py @@ -0,0 +1,73 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +nodes = [ + cluster.add_instance( + f"node{num}", main_configs=["configs/remote_servers.xml"], with_zookeeper=True + ) + for num in range(6) +] + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def _create_tables(table_name): + for idx, node in enumerate(nodes): + node.query( + f"DROP TABLE IF EXISTS {table_name}", + settings={"database_atomic_wait_for_drop_and_detach_synchronously": True}, + ) + + node.query( + f""" + CREATE TABLE {table_name} (value Int64) + Engine=ReplicatedMergeTree('/test_parallel_replicas/shard/{table_name}', '{idx}') + ORDER BY () + """ + ) + + nodes[0].query( + f"INSERT INTO {table_name} SELECT * FROM numbers(1000)", + settings={"insert_deduplicate": 0}, + ) + nodes[0].query(f"SYSTEM SYNC REPLICA ON CLUSTER 'parallel_replicas' {table_name}") + + for idx, node in enumerate(nodes): + node.query("SYSTEM STOP REPLICATED SENDS") + # the same data on all nodes except for a single value + node.query( + f"INSERT INTO {table_name} VALUES ({idx})", + settings={"insert_deduplicate": 0}, + ) + + +# check that we use the state of data parts from the initiator node (for some sort of determinism of what is been read). +# currently it is implemented only when we build local plan for the initiator node (we aim to make this behavior default) +def test_initiator_snapshot_is_used_for_reading(start_cluster): + table_name = "t" + _create_tables(table_name) + + for idx, node in enumerate(nodes): + expected = 499500 + idx # sum of all integers 0..999 + idx + assert ( + node.query( + f"SELECT sum(value) FROM {table_name}", + settings={ + "allow_experimental_parallel_reading_from_replicas": 2, + "max_parallel_replicas": 100, + "cluster_for_parallel_replicas": "parallel_replicas", + "parallel_replicas_local_plan": True, + }, + ) + == f"{expected}\n" + ) diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index 2b30170b203e..c3b8b14560c4 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -365,7 +365,7 @@ def test_different_part_types_on_replicas(start_cluster, table, part_type): "node7", user_configs=["configs_old/users.d/not_optimize_count.xml"], with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_replicated_merge_tree_compatibility/test.py b/tests/integration/test_replicated_merge_tree_compatibility/test.py index a70f3234c1eb..22de74d073e4 100644 --- a/tests/integration/test_replicated_merge_tree_compatibility/test.py +++ b/tests/integration/test_replicated_merge_tree_compatibility/test.py @@ -5,7 +5,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -13,7 +13,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml b/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml index 30a0b6276b76..469e76528c5f 100644 --- a/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml +++ b/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml @@ -2,7 +2,8 @@ 1 - 1 + 1 + 1 default 100 0 diff --git a/tests/integration/test_replicating_constants/test.py b/tests/integration/test_replicating_constants/test.py index af8916dd625b..8da83038d146 100644 --- a/tests/integration/test_replicating_constants/test.py +++ b/tests/integration/test_replicating_constants/test.py @@ -8,8 +8,8 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, - image="clickhouse/clickhouse-server", - tag="23.3", + image="altinity/clickhouse-server", + tag="23.3.19.33.altinitystable", with_installed_binary=True, ) diff --git a/tests/integration/test_storage_s3_queue/configs/users_old.xml b/tests/integration/test_storage_s3_queue/configs/users_old.xml new file mode 100644 index 000000000000..227dbe9bde5b --- /dev/null +++ b/tests/integration/test_storage_s3_queue/configs/users_old.xml @@ -0,0 +1,11 @@ + + + + 1 + 1 + + + 1 + + + diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 87a707ee8711..ff041ce16b70 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -107,7 +107,9 @@ def started_cluster(): ) cluster.add_instance( "old_instance", + user_configs=["configs/users_old.xml"], with_zookeeper=True, + # NOTE (vnemkov) Can't use altinity/clickhouse here since 23.8 doesn't hve S3Queue (and associated settings yet) image="clickhouse/clickhouse-server", tag="23.12", stay_alive=True, diff --git a/tests/integration/test_trace_log_build_id/test.py b/tests/integration/test_trace_log_build_id/test.py index 8d654aef342d..19c26a3a5bf9 100644 --- a/tests/integration/test_trace_log_build_id/test.py +++ b/tests/integration/test_trace_log_build_id/test.py @@ -11,7 +11,7 @@ node = cluster.add_instance( "node", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 538322473ee3..21b5885ffb49 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -16,7 +16,7 @@ node4 = cluster.add_instance( "node4", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -28,7 +28,7 @@ node5 = cluster.add_instance( "node5", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, @@ -39,7 +39,7 @@ node6 = cluster.add_instance( "node6", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, diff --git a/tests/integration/test_version_update/test.py b/tests/integration/test_version_update/test.py index b386a79c932f..f42d791d1bda 100644 --- a/tests/integration/test_version_update/test.py +++ b/tests/integration/test_version_update/test.py @@ -10,7 +10,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, stay_alive=True, diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 9365498f89d4..73a01a51ce74 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -9,7 +9,7 @@ node1 = cluster.add_instance( "node1", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, stay_alive=True, @@ -20,7 +20,7 @@ node2 = cluster.add_instance( "node2", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, stay_alive=True, @@ -31,7 +31,7 @@ node3 = cluster.add_instance( "node3", with_zookeeper=True, - image="clickhouse/clickhouse-server", + image="altinity/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, stay_alive=True, diff --git a/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql b/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql index 901b818cbc06..fa713fc8d43a 100644 --- a/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql +++ b/tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql @@ -1,7 +1,9 @@ -- Tags: replica, distributed -SET allow_experimental_parallel_reading_from_replicas = 0; +SET enable_parallel_replicas = 1; +SET parallel_replicas_mode = 'sampling_key'; SET max_parallel_replicas = 2; +SET parallel_replicas_for_non_replicated_merge_tree = 1; DROP TABLE IF EXISTS report; set allow_deprecated_syntax_for_merge_tree=1; diff --git a/tests/queries/0_stateless/00193_parallel_replicas.sql b/tests/queries/0_stateless/00193_parallel_replicas.sql index 2549ada2a78a..b19d9cfdefe4 100644 --- a/tests/queries/0_stateless/00193_parallel_replicas.sql +++ b/tests/queries/0_stateless/00193_parallel_replicas.sql @@ -13,12 +13,14 @@ INSERT INTO parallel_replicas (x, u, s) VALUES (41, 42, 'U'),(43, 44, 'V'),(45, INSERT INTO parallel_replicas (x, u, s) VALUES (51, 52, 'Z'); /* - * Проверяем, что: - * - на каждой реплике таблица не пустая; - * - объединение данных всех реплик совпадает с содержанием таблицы parallel_replicas. - */ +* Check that: +* - the table is not empty on each replica; +* - combining the data of all replicas coincides with the contents of the parallel_replicas table. +*/ -/* Две реплики */ +/* Two replicas */ + +SET enable_parallel_replicas=1, parallel_replicas_mode='sampling_key', max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree = 1; CREATE TABLE parallel_replicas_backup(d Date DEFAULT today(), x UInt32, u UInt64, s String) ENGINE = Memory; @@ -38,7 +40,7 @@ SELECT x, u, s FROM parallel_replicas_backup ORDER BY x, u, s ASC; DROP TABLE parallel_replicas_backup; CREATE TABLE parallel_replicas_backup(d Date DEFAULT today(), x UInt32, u UInt64, s String) ENGINE = Memory; -/* Три реплики */ +/* Three replicas */ SET parallel_replicas_count = 3; diff --git a/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql b/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql index 86c84427297d..566660f67c63 100644 --- a/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql +++ b/tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql @@ -1,7 +1,5 @@ -- Tags: replica, distributed -set allow_experimental_parallel_reading_from_replicas=0; - drop table if exists test_max_parallel_replicas_lr; -- If you wonder why the table is named with "_lr" suffix in this test. @@ -10,7 +8,10 @@ drop table if exists test_max_parallel_replicas_lr; CREATE TABLE test_max_parallel_replicas_lr (timestamp UInt64) ENGINE = MergeTree ORDER BY (intHash32(timestamp)) SAMPLE BY intHash32(timestamp); INSERT INTO test_max_parallel_replicas_lr select number as timestamp from system.numbers limit 100; +SET enable_parallel_replicas = 1; +SET parallel_replicas_mode='sampling_key'; SET max_parallel_replicas = 2; +SET parallel_replicas_for_non_replicated_merge_tree = 1; select count() FROM remote('127.0.0.{2|3}', currentDatabase(), test_max_parallel_replicas_lr) PREWHERE timestamp > 0; drop table test_max_parallel_replicas_lr; diff --git a/tests/queries/0_stateless/01034_sample_final_distributed.sql b/tests/queries/0_stateless/01034_sample_final_distributed.sql index 64bafd17b2d9..bbb1b0dcf691 100644 --- a/tests/queries/0_stateless/01034_sample_final_distributed.sql +++ b/tests/queries/0_stateless/01034_sample_final_distributed.sql @@ -1,6 +1,9 @@ -- Tags: distributed -set allow_experimental_parallel_reading_from_replicas = 0; +set enable_parallel_replicas = 1; +set parallel_replicas_mode = 'sampling_key'; +set max_parallel_replicas = 3; +set parallel_replicas_for_non_replicated_merge_tree = 1; drop table if exists sample_final; create table sample_final (CounterID UInt32, EventDate Date, EventTime DateTime, UserID UInt64, Sign Int8) engine = CollapsingMergeTree(Sign) order by (CounterID, EventDate, intHash32(UserID), EventTime) sample by intHash32(UserID) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'; diff --git a/tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql b/tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql index ec650551bafb..f31929306998 100644 --- a/tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql +++ b/tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql @@ -2,7 +2,7 @@ -- set distributed_foreground_insert = 1; -- see https://github.com/ClickHouse/ClickHouse/issues/18971 -SET allow_experimental_parallel_reading_from_replicas = 0; -- see https://github.com/ClickHouse/ClickHouse/issues/34525 +SET enable_parallel_replicas = 0; -- see https://github.com/ClickHouse/ClickHouse/issues/34525 SET prefer_localhost_replica = 1; DROP TABLE IF EXISTS local_01099_a; diff --git a/tests/queries/0_stateless/01517_select_final_distributed.sql b/tests/queries/0_stateless/01517_select_final_distributed.sql index 701828b0b38f..d2c6e63a861a 100644 --- a/tests/queries/0_stateless/01517_select_final_distributed.sql +++ b/tests/queries/0_stateless/01517_select_final_distributed.sql @@ -1,16 +1,16 @@ -- Tags: distributed -SET allow_experimental_parallel_reading_from_replicas = 0; +SET enable_parallel_replicas = 0; DROP TABLE IF EXISTS test5346; -CREATE TABLE test5346 (`Id` String, `Timestamp` DateTime, `updated` DateTime) +CREATE TABLE test5346 (`Id` String, `Timestamp` DateTime, `updated` DateTime) ENGINE = ReplacingMergeTree(updated) PARTITION BY tuple() ORDER BY (Timestamp, Id); INSERT INTO test5346 VALUES('1',toDateTime('2020-01-01 00:00:00'),toDateTime('2020-01-01 00:00:00')); -SELECT Id, Timestamp -FROM remote('localhost,127.0.0.1,127.0.0.2',currentDatabase(),'test5346') FINAL +SELECT Id, Timestamp +FROM remote('localhost,127.0.0.1,127.0.0.2',currentDatabase(),'test5346') FINAL ORDER BY Timestamp; SELECT Id, Timestamp diff --git a/tests/queries/0_stateless/01557_max_parallel_replicas_no_sample.sql b/tests/queries/0_stateless/01557_max_parallel_replicas_no_sample.sql index 04777f5b31ce..3704119c989b 100644 --- a/tests/queries/0_stateless/01557_max_parallel_replicas_no_sample.sql +++ b/tests/queries/0_stateless/01557_max_parallel_replicas_no_sample.sql @@ -1,6 +1,9 @@ -- Tags: replica -SET allow_experimental_parallel_reading_from_replicas=0; +SET enable_parallel_replicas=1; +SET max_parallel_replicas=3; +SET parallel_replicas_mode='sampling_key'; +SET parallel_replicas_for_non_replicated_merge_tree = 1; DROP TABLE IF EXISTS t; CREATE TABLE t (x String) ENGINE = MergeTree ORDER BY x; diff --git a/tests/queries/0_stateless/02221_parallel_replicas_bug.sh b/tests/queries/0_stateless/02221_parallel_replicas_bug.sh index 3c44a2a7ba7f..5c801af531de 100755 --- a/tests/queries/0_stateless/02221_parallel_replicas_bug.sh +++ b/tests/queries/0_stateless/02221_parallel_replicas_bug.sh @@ -4,4 +4,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} --allow_experimental_parallel_reading_from_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 -nm < "$CURDIR"/01099_parallel_distributed_insert_select.sql > /dev/null +${CLICKHOUSE_CLIENT} --enable_parallel_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 -m < "$CURDIR"/01099_parallel_distributed_insert_select.sql > /dev/null diff --git a/tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.sh b/tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.sh index bc90f4b2c111..347293036e17 100755 --- a/tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.sh +++ b/tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.sh @@ -14,7 +14,7 @@ as select * from numbers(1); # # Logical error: 'Coordinator for parallel reading from replicas is not initialized'. opts=( - --allow_experimental_parallel_reading_from_replicas 1 + --enable_parallel_replicas 1 --parallel_replicas_for_non_replicated_merge_tree 1 --max_parallel_replicas 3 --cluster_for_parallel_replicas parallel_replicas diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.reference b/tests/queries/0_stateless/02404_memory_bound_merging.reference index 0409c48f846a..e02684c3b95d 100644 --- a/tests/queries/0_stateless/02404_memory_bound_merging.reference +++ b/tests/queries/0_stateless/02404_memory_bound_merging.reference @@ -100,7 +100,7 @@ select a, count() from dist_t_different_dbs group by a, b order by a limit 5 off 504 2000 1000000 -- { echoOn } -- -explain pipeline select a from pr_t group by a order by a limit 5 offset 500; +explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings parallel_replicas_local_plan=0; (Expression) ExpressionTransform (Limit) @@ -119,6 +119,31 @@ ExpressionTransform Resize 1 → 4 GroupingAggregatedTransform 3 → 1 (ReadFromRemoteParallelReplicas) +explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings allow_experimental_analyzer=1, parallel_replicas_local_plan=1; +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + MergingSortedTransform 4 → 1 + MergeSortingTransform × 4 + LimitsCheckingTransform × 4 + PartialSortingTransform × 4 + (Expression) + ExpressionTransform × 4 + (MergingAggregated) + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 3 → 1 + (Union) + (Aggregating) + FinalizeAggregatedTransform + AggregatingInOrderTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeSelect(pool: ReadPoolParallelReplicasInOrder, algorithm: InOrder) 0 → 1 + (ReadFromRemoteParallelReplicas) select a, count() from pr_t group by a order by a limit 5 offset 500; 500 1000 501 1000 diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.sql b/tests/queries/0_stateless/02404_memory_bound_merging.sql index a2de19dff8a7..866f49717aba 100644 --- a/tests/queries/0_stateless/02404_memory_bound_merging.sql +++ b/tests/queries/0_stateless/02404_memory_bound_merging.sql @@ -63,17 +63,17 @@ drop table if exists pr_t; create table pr_t(a UInt64, b UInt64) engine=MergeTree order by a; insert into pr_t select number % 1000, number % 1000 from numbers_mt(1e6); -set allow_experimental_parallel_reading_from_replicas = 1; +set enable_parallel_replicas = 1; set parallel_replicas_for_non_replicated_merge_tree = 1; set max_parallel_replicas = 3; -set use_hedged_requests = 0; set cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; set distributed_aggregation_memory_efficient=1; select count() from pr_t; -- { echoOn } -- -explain pipeline select a from pr_t group by a order by a limit 5 offset 500; +explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings parallel_replicas_local_plan=0; +explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings allow_experimental_analyzer=1, parallel_replicas_local_plan=1; select a, count() from pr_t group by a order by a limit 5 offset 500; select a, count() from pr_t group by a, b order by a limit 5 offset 500; diff --git a/tests/queries/0_stateless/02532_send_logs_level_test.reference b/tests/queries/0_stateless/02532_send_logs_level_test.reference index 72f4ea06184f..e69de29bb2d1 100644 --- a/tests/queries/0_stateless/02532_send_logs_level_test.reference +++ b/tests/queries/0_stateless/02532_send_logs_level_test.reference @@ -1,4 +0,0 @@ - MergeTreeReadPoolBase: Will use min_marks_per_task=24 - MergeTreeMarksLoader: Loading marks from path data.cmrk3 - MergeTreeRangeReader: First reader returned: num_rows: 1, columns: 1, total_rows_per_granule: 1, no filter, column[0]: Int32(size = 1), requested columns: key - MergeTreeRangeReader: read() returned num_rows: 1, columns: 1, total_rows_per_granule: 1, no filter, column[0]: Int32(size = 1), sample block key diff --git a/tests/queries/0_stateless/02532_send_logs_level_test.sh b/tests/queries/0_stateless/02532_send_logs_level_test.sh index a50539311cba..a91e49ddd229 100755 --- a/tests/queries/0_stateless/02532_send_logs_level_test.sh +++ b/tests/queries/0_stateless/02532_send_logs_level_test.sh @@ -18,6 +18,10 @@ $CLICKHOUSE_CLIENT -m -q " # instead of "last" value, hence you cannot simply append another # --send_logs_level here. CLICKHOUSE_CLIENT_CLEAN=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=test/g') -$CLICKHOUSE_CLIENT_CLEAN -q "select * from data SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" |& grep -o -e '.*' -e '.*' -$CLICKHOUSE_CLIENT -q "drop table data" +set -e + +trap '$CLICKHOUSE_CLIENT -q "drop table data"' EXIT + +$CLICKHOUSE_CLIENT_CLEAN -q "select * from data SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" |& (! grep -q -o -e '.*') +$CLICKHOUSE_CLIENT_CLEAN -q "select * from data SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;" |& grep -q -o -e '.*' diff --git a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_mt.reference b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_mt.reference index 1bb07f0d9168..4759240cfae0 100644 --- a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_mt.reference +++ b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_mt.reference @@ -1,176 +1,176 @@ query='SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_mt)' with custom_key='sipHash64(x)' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 Hello -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 Hello -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 Hello -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 Hello -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 Hello -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 Hello query='SELECT * FROM 02535_custom_key_mt' with custom_key='sipHash64(x)' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 Hello -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 Hello -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 Hello -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 Hello -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 Hello -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 Hello query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_mt) GROUP BY y ORDER BY y' with custom_key='y' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_mt) GROUP BY y ORDER BY y' with custom_key='cityHash64(y)' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_mt) GROUP BY y ORDER BY y' with custom_key='cityHash64(y) + 1' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 query='SELECT y, count() FROM 02535_custom_key_mt GROUP BY y ORDER BY y' with custom_key='y' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 query='SELECT y, count() FROM 02535_custom_key_mt GROUP BY y ORDER BY y' with custom_key='cityHash64(y)' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 query='SELECT y, count() FROM 02535_custom_key_mt GROUP BY y ORDER BY y' with custom_key='cityHash64(y) + 1' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 diff --git a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_mt.sh b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_mt.sh index fad43ea90706..aa27af090638 100755 --- a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_mt.sh +++ b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_mt.sh @@ -7,12 +7,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function run_with_custom_key { echo "query='$1' with custom_key='$2'" - for filter_type in 'default' 'range'; do + for filter_type in 'custom_key_sampling' 'custom_key_range'; do for max_replicas in {1..3}; do echo "filter_type='$filter_type' max_replicas=$max_replicas" query="$1 SETTINGS max_parallel_replicas=$max_replicas\ +, enable_parallel_replicas='1' \ +, parallel_replicas_mode ='$filter_type'\ , parallel_replicas_custom_key='$2'\ -, parallel_replicas_custom_key_filter_type='$filter_type'\ , parallel_replicas_for_non_replicated_merge_tree=1 \ , cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'" $CLICKHOUSE_CLIENT --query="$query" diff --git a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_rmt.reference b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_rmt.reference index c6526b506d34..170a51f1f255 100644 --- a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_rmt.reference +++ b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_rmt.reference @@ -1,176 +1,176 @@ query='SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_rmt)' with custom_key='sipHash64(x)' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 Hello -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 Hello -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 Hello -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 Hello -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 Hello -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 Hello query='SELECT * FROM 02535_custom_key_rmt' with custom_key='sipHash64(x)' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 Hello -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 Hello -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 Hello -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 Hello -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 Hello -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 Hello query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_rmt_hash) GROUP BY y ORDER BY y' with custom_key='y' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_rmt_hash) GROUP BY y ORDER BY y' with custom_key='cityHash64(y)' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_rmt_hash) GROUP BY y ORDER BY y' with custom_key='cityHash64(y) + 1' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 query='SELECT y, count() FROM 02535_custom_key_rmt_hash GROUP BY y ORDER BY y' with custom_key='y' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 query='SELECT y, count() FROM 02535_custom_key_rmt_hash GROUP BY y ORDER BY y' with custom_key='cityHash64(y)' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 query='SELECT y, count() FROM 02535_custom_key_rmt_hash GROUP BY y ORDER BY y' with custom_key='cityHash64(y) + 1' -filter_type='default' max_replicas=1 +filter_type='custom_key_sampling' max_replicas=1 0 334 1 333 2 333 -filter_type='default' max_replicas=2 +filter_type='custom_key_sampling' max_replicas=2 0 334 1 333 2 333 -filter_type='default' max_replicas=3 +filter_type='custom_key_sampling' max_replicas=3 0 334 1 333 2 333 -filter_type='range' max_replicas=1 +filter_type='custom_key_range' max_replicas=1 0 334 1 333 2 333 -filter_type='range' max_replicas=2 +filter_type='custom_key_range' max_replicas=2 0 334 1 333 2 333 -filter_type='range' max_replicas=3 +filter_type='custom_key_range' max_replicas=3 0 334 1 333 2 333 diff --git a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_rmt.sh b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_rmt.sh index 6350f5027f90..87899aab274d 100755 --- a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_rmt.sh +++ b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_rmt.sh @@ -7,12 +7,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function run_with_custom_key { echo "query='$1' with custom_key='$2'" - for filter_type in 'default' 'range'; do + for filter_type in 'custom_key_sampling' 'custom_key_range'; do for max_replicas in {1..3}; do echo "filter_type='$filter_type' max_replicas=$max_replicas" query="$1 SETTINGS max_parallel_replicas=$max_replicas\ +, enable_parallel_replicas='1' \ +, parallel_replicas_mode ='$filter_type'\ , parallel_replicas_custom_key='$2'\ -, parallel_replicas_custom_key_filter_type='$filter_type'\ , parallel_replicas_for_non_replicated_merge_tree=1 \ , cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'" $CLICKHOUSE_CLIENT --query="$query" diff --git a/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.reference b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.reference index 492b12dba563..3d385dcfb465 100644 --- a/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.reference +++ b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.reference @@ -12,3 +12,4 @@ 10 10 10 +10 diff --git a/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.sh b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.sh index d00026d516a3..26ff30495abc 100755 --- a/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.sh +++ b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.sh @@ -10,6 +10,7 @@ set -o pipefail $CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='none'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" $CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='lz4'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" $CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='snappy'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='snappy', output_format_parquet_use_custom_encoder=0" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" $CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='zstd'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" $CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='brotli'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" $CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='gzip'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" diff --git a/tests/queries/0_stateless/02708_parallel_replicas_not_found_column.sql b/tests/queries/0_stateless/02708_parallel_replicas_not_found_column.sql index ff7b53ce01ff..4179b0491fdd 100644 --- a/tests/queries/0_stateless/02708_parallel_replicas_not_found_column.sql +++ b/tests/queries/0_stateless/02708_parallel_replicas_not_found_column.sql @@ -1,4 +1,4 @@ CREATE TABLE IF NOT EXISTS t_02708(x DateTime) ENGINE = MergeTree ORDER BY tuple(); SET send_logs_level='error'; -SELECT count() FROM t_02708 SETTINGS allow_experimental_parallel_reading_from_replicas=1; +SELECT count() FROM t_02708 SETTINGS enable_parallel_replicas=1; DROP TABLE t_02708; diff --git a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference index 125fa524c4ae..c6a3bbc3dfb1 100644 --- a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference +++ b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference @@ -31,10 +31,10 @@ 29 2j&S)ba?XG QuQj 17163829389637435056 3 UlI+1 14144472852965836438 =============== QUERIES EXECUTED BY PARALLEL INNER QUERY ALONE =============== -0 3 SELECT `__table1`.`key` AS `key`, `__table1`.`value1` AS `value1`, `__table1`.`value2` AS `value2`, toUInt64(min(`__table1`.`time`)) AS `start_ts` FROM `default`.`join_inner_table` AS `__table1` PREWHERE (`__table1`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table1`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table1`.`key`, `__table1`.`value1`, `__table1`.`value2` ORDER BY `__table1`.`key` ASC, `__table1`.`value1` ASC, `__table1`.`value2` ASC LIMIT _CAST(10, \'UInt64\') +0 2 SELECT `__table1`.`key` AS `key`, `__table1`.`value1` AS `value1`, `__table1`.`value2` AS `value2`, toUInt64(min(`__table1`.`time`)) AS `start_ts` FROM `default`.`join_inner_table` AS `__table1` PREWHERE (`__table1`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table1`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table1`.`key`, `__table1`.`value1`, `__table1`.`value2` ORDER BY `__table1`.`key` ASC, `__table1`.`value1` ASC, `__table1`.`value2` ASC LIMIT _CAST(10, \'UInt64\') 0 3 SELECT `key`, `value1`, `value2`, toUInt64(min(`time`)) AS `start_ts` FROM `default`.`join_inner_table` PREWHERE (`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`number` > toUInt64(\'1610517366120\')) GROUP BY `key`, `value1`, `value2` ORDER BY `key` ASC, `value1` ASC, `value2` ASC LIMIT 10 -1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=0; -1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=1; +1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS enable_parallel_replicas = 1, enable_analyzer=0; +1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS enable_parallel_replicas = 1, enable_analyzer=1; =============== OUTER QUERY (NO PARALLEL) =============== >T%O ,z< 10 NQTpY# W\\Xx4 10 @@ -58,8 +58,8 @@ U c 10 UlI+1 10 bX?}ix [ Ny]2 G 10 t _CAST(1610517366120, \'UInt64\')) GROUP BY `__table3`.`key`, `__table3`.`value1`, `__table3`.`value2`) AS `__table2` USING (`key`) GROUP BY `__table1`.`key`, `__table2`.`value1`, `__table2`.`value2` +0 2 SELECT `__table2`.`value1` AS `value1`, `__table2`.`value2` AS `value2`, count() AS `count` FROM `default`.`join_outer_table` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`key` AS `key`, `__table3`.`value1` AS `value1`, `__table3`.`value2` AS `value2` FROM `default`.`join_inner_table` AS `__table3` PREWHERE (`__table3`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table3`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table3`.`key`, `__table3`.`value1`, `__table3`.`value2`) AS `__table2` USING (`key`) GROUP BY `__table1`.`key`, `__table2`.`value1`, `__table2`.`value2` 0 3 SELECT `key`, `value1`, `value2` FROM `default`.`join_inner_table` PREWHERE (`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`number` > toUInt64(\'1610517366120\')) GROUP BY `key`, `value1`, `value2` 0 3 SELECT `value1`, `value2`, count() AS `count` FROM `default`.`join_outer_table` ALL INNER JOIN `_data_` USING (`key`) GROUP BY `key`, `value1`, `value2` -1 1 -- Parallel full query\nSELECT\n value1,\n value2,\n avg(count) AS avg\nFROM\n (\n SELECT\n key,\n value1,\n value2,\n count() AS count\n FROM join_outer_table\n INNER JOIN\n (\n SELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\n FROM join_inner_table\n PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n GROUP BY key, value1, value2\n ) USING (key)\n GROUP BY key, value1, value2\n )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=0; -1 1 -- Parallel full query\nSELECT\n value1,\n value2,\n avg(count) AS avg\nFROM\n (\n SELECT\n key,\n value1,\n value2,\n count() AS count\n FROM join_outer_table\n INNER JOIN\n (\n SELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\n FROM join_inner_table\n PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n GROUP BY key, value1, value2\n ) USING (key)\n GROUP BY key, value1, value2\n )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=1; +1 1 -- Parallel full query\nSELECT\n value1,\n value2,\n avg(count) AS avg\nFROM\n (\n SELECT\n key,\n value1,\n value2,\n count() AS count\n FROM join_outer_table\n INNER JOIN\n (\n SELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\n FROM join_inner_table\n PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n GROUP BY key, value1, value2\n ) USING (key)\n GROUP BY key, value1, value2\n )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS enable_parallel_replicas = 1, enable_analyzer=0; +1 1 -- Parallel full query\nSELECT\n value1,\n value2,\n avg(count) AS avg\nFROM\n (\n SELECT\n key,\n value1,\n value2,\n count() AS count\n FROM join_outer_table\n INNER JOIN\n (\n SELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\n FROM join_inner_table\n PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n GROUP BY key, value1, value2\n ) USING (key)\n GROUP BY key, value1, value2\n )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS enable_parallel_replicas = 1, enable_analyzer=1; diff --git a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql index 8121d60a05be..2cf1ffa13e8d 100644 --- a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql +++ b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql @@ -22,8 +22,8 @@ SELECT LIMIT 100; SET max_parallel_replicas = 3; -SET prefer_localhost_replica = 1; SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; +SET parallel_replicas_local_plan = 1; SET joined_subquery_requires_alias = 0; SELECT '=============== INNER QUERY (NO PARALLEL) ==============='; @@ -64,7 +64,7 @@ PREWHERE (id = '833c9e22-c245-4eb5-8745-117a9a1f26b1') AND (number > toUInt64('1 GROUP BY key, value1, value2 ORDER BY key, value1, value2 LIMIT 10 -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=0; +SETTINGS enable_parallel_replicas = 1, enable_analyzer=0; -- Parallel inner query alone SELECT @@ -77,7 +77,7 @@ PREWHERE (id = '833c9e22-c245-4eb5-8745-117a9a1f26b1') AND (number > toUInt64('1 GROUP BY key, value1, value2 ORDER BY key, value1, value2 LIMIT 10 -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=1; +SETTINGS enable_parallel_replicas = 1, enable_analyzer=1; SELECT '=============== QUERIES EXECUTED BY PARALLEL INNER QUERY ALONE ==============='; @@ -184,7 +184,7 @@ FROM ) GROUP BY value1, value2 ORDER BY value1, value2 -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=0; +SETTINGS enable_parallel_replicas = 1, enable_analyzer=0; -- Parallel full query SELECT @@ -214,7 +214,7 @@ FROM ) GROUP BY value1, value2 ORDER BY value1, value2 -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=1; +SETTINGS enable_parallel_replicas = 1, enable_analyzer=1; SYSTEM FLUSH LOGS; diff --git a/tests/queries/0_stateless/02751_parallel_replicas_bug_chunkinfo_not_set.sql b/tests/queries/0_stateless/02751_parallel_replicas_bug_chunkinfo_not_set.sql index a7112e5484b1..ca22f324f2b5 100644 --- a/tests/queries/0_stateless/02751_parallel_replicas_bug_chunkinfo_not_set.sql +++ b/tests/queries/0_stateless/02751_parallel_replicas_bug_chunkinfo_not_set.sql @@ -18,7 +18,7 @@ INSERT INTO join_inner_table__fuzz_1 SELECT FROM generateRandom('number Int64, value1 String, value2 String, time Int64', 1, 10, 2) LIMIT 100; -SET max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree=1; +SET max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree=1; -- SELECT query will write a Warning to the logs SET send_logs_level='error'; diff --git a/tests/queries/0_stateless/02764_parallel_replicas_plain_merge_tree.sql b/tests/queries/0_stateless/02764_parallel_replicas_plain_merge_tree.sql index e166ce9b2849..b26cbc3cfad9 100644 --- a/tests/queries/0_stateless/02764_parallel_replicas_plain_merge_tree.sql +++ b/tests/queries/0_stateless/02764_parallel_replicas_plain_merge_tree.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS parallel_replicas_plain; CREATE TABLE parallel_replicas_plain (x String) ENGINE=MergeTree() ORDER BY x; INSERT INTO parallel_replicas_plain SELECT toString(number) FROM numbers(10); -SET max_parallel_replicas=3, allow_experimental_parallel_reading_from_replicas=1, cluster_for_parallel_replicas='parallel_replicas'; +SET max_parallel_replicas=3, enable_parallel_replicas=1, cluster_for_parallel_replicas='parallel_replicas'; SET send_logs_level='error'; SET parallel_replicas_for_non_replicated_merge_tree = 0; diff --git a/tests/queries/0_stateless/02765_parallel_replicas_final_modifier.sql b/tests/queries/0_stateless/02765_parallel_replicas_final_modifier.sql index 6c121802b063..6acfb2482d38 100644 --- a/tests/queries/0_stateless/02765_parallel_replicas_final_modifier.sql +++ b/tests/queries/0_stateless/02765_parallel_replicas_final_modifier.sql @@ -2,12 +2,12 @@ CREATE TABLE IF NOT EXISTS parallel_replicas_final (x String) ENGINE=ReplacingMe INSERT INTO parallel_replicas_final SELECT toString(number) FROM numbers(10); -SET max_parallel_replicas=3, allow_experimental_parallel_reading_from_replicas=1, cluster_for_parallel_replicas='parallel_replicas'; +SET max_parallel_replicas=3, enable_parallel_replicas=1, cluster_for_parallel_replicas='parallel_replicas'; SET parallel_replicas_for_non_replicated_merge_tree = 1; SELECT * FROM parallel_replicas_final FINAL FORMAT Null; -SET allow_experimental_parallel_reading_from_replicas=2; +SET enable_parallel_replicas=2; SELECT * FROM parallel_replicas_final FINAL FORMAT Null; -- { serverError SUPPORT_IS_DISABLED } diff --git a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql index 1a75e000349c..1b015ac495e6 100644 --- a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql +++ b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql @@ -2,13 +2,15 @@ DROP TABLE IF EXISTS test_parallel_replicas_unavailable_shards; CREATE TABLE test_parallel_replicas_unavailable_shards (n UInt64) ENGINE=MergeTree() ORDER BY tuple(); INSERT INTO test_parallel_replicas_unavailable_shards SELECT * FROM numbers(10); -SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=11, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1; +SET enable_parallel_replicas=2, max_parallel_replicas=11, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1; SET send_logs_level='error'; -SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*) SETTINGS log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79'; +-- with local plan for initiator, the query can be executed fast on initator, we can simply not come to the point where unavailable replica can be detected +-- therefore disable local plan for now +SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*) SETTINGS log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79', parallel_replicas_local_plan=0; SYSTEM FLUSH LOGS; -SET allow_experimental_parallel_reading_from_replicas=0; +SET enable_parallel_replicas=0; SELECT ProfileEvents['ParallelReplicasUnavailableCount'] FROM system.query_log WHERE yesterday() <= event_date AND query_id in (select query_id from system.query_log where log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79' and current_database = currentDatabase()) and type = 'QueryFinish' and query_id == initial_query_id; DROP TABLE test_parallel_replicas_unavailable_shards; diff --git a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference index f60f1e0a3768..b94774e6ca3a 100644 --- a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference +++ b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference @@ -8,5 +8,5 @@ 5935810273536892891 7885388429666205427 8124171311239967992 -1 1 -- Simple query with analyzer and pure parallel replicas\nSELECT number\nFROM join_inner_table__fuzz_146_replicated\n SETTINGS\n enable_analyzer = 1,\n max_parallel_replicas = 2,\n cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\',\n allow_experimental_parallel_reading_from_replicas = 1; -0 2 SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1` +1 1 -- Simple query with analyzer and pure parallel replicas\nSELECT number\nFROM join_inner_table__fuzz_146_replicated\n SETTINGS\n enable_analyzer = 1,\n max_parallel_replicas = 2,\n cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\',\n enable_parallel_replicas = 1; +0 1 SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1` diff --git a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql index e60049f2756a..081077ba4601 100644 --- a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql +++ b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql @@ -1,5 +1,5 @@ -- Tags: zookeeper - +DROP TABLE IF EXISTS join_inner_table__fuzz_146_replicated; CREATE TABLE join_inner_table__fuzz_146_replicated ( `id` UUID, @@ -17,6 +17,8 @@ INSERT INTO join_inner_table__fuzz_146_replicated SELECT CAST('833c9e22-c245-4eb5-8745-117a9a1f26b1', 'UUID') AS id, CAST(rowNumberInAllBlocks(), 'String') AS key, * FROM generateRandom('number Int64, value1 String, value2 String, time Int64', 1, 10, 2) LIMIT 10; +SET parallel_replicas_local_plan = 1; + -- Simple query with analyzer and pure parallel replicas SELECT number FROM join_inner_table__fuzz_146_replicated @@ -24,7 +26,7 @@ FROM join_inner_table__fuzz_146_replicated enable_analyzer = 1, max_parallel_replicas = 2, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', - allow_experimental_parallel_reading_from_replicas = 1; + enable_parallel_replicas = 1; SYSTEM FLUSH LOGS; -- There should be 2 different queries @@ -49,3 +51,5 @@ WHERE ) GROUP BY is_initial_query, query ORDER BY is_initial_query DESC, c, query; + +DROP TABLE join_inner_table__fuzz_146_replicated; diff --git a/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh b/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh index bf7170fd7fb4..47be7c2d372b 100755 --- a/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh +++ b/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh @@ -33,7 +33,7 @@ function run_query_with_pure_parallel_replicas () { --max_parallel_replicas 3 \ --prefer_localhost_replica 1 \ --cluster_for_parallel_replicas 'test_cluster_one_shard_three_replicas_localhost' \ - --allow_experimental_parallel_reading_from_replicas 1 \ + --enable_parallel_replicas 1 \ --enable_analyzer 0 $CLICKHOUSE_CLIENT \ @@ -42,7 +42,7 @@ function run_query_with_pure_parallel_replicas () { --max_parallel_replicas 3 \ --prefer_localhost_replica 1 \ --cluster_for_parallel_replicas 'test_cluster_one_shard_three_replicas_localhost' \ - --allow_experimental_parallel_reading_from_replicas 1 \ + --enable_parallel_replicas 1 \ --enable_analyzer 1 } @@ -56,7 +56,7 @@ function run_query_with_custom_key_parallel_replicas () { --query "$2" \ --query_id "${1}_custom_key" \ --max_parallel_replicas 3 \ - --parallel_replicas_custom_key_filter_type 'default' \ + --parallel_replicas_mode 'custom_key_sampling' \ --parallel_replicas_custom_key "$2" \ --enable_analyzer 0 @@ -64,7 +64,7 @@ function run_query_with_custom_key_parallel_replicas () { --query "$2" \ --query_id "${1}_custom_key_analyzer" \ --max_parallel_replicas 3 \ - --parallel_replicas_custom_key_filter_type 'default' \ + --parallel_replicas_mode 'custom_key_sampling' \ --parallel_replicas_custom_key "$2" \ --enable_analyzer 1 } diff --git a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh index 8a3b34e5cfa2..e0aeb39972be 100755 --- a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh +++ b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh @@ -6,16 +6,17 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_CLIENT_TRACE=${CLICKHOUSE_CLIENT/"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"/"--send_logs_level=trace"} function were_parallel_replicas_used () { - # Not using current_database = '$CLICKHOUSE_DATABASE' as nested parallel queries aren't run with it $CLICKHOUSE_CLIENT --query " SELECT initial_query_id, - concat('Used parallel replicas: ', (countIf(initial_query_id != query_id) != 0)::bool::String) as used + concat('Used parallel replicas: ', (ProfileEvents['ParallelReplicasUsedCount'] > 0)::bool::String) as used FROM system.query_log WHERE event_date >= yesterday() AND initial_query_id LIKE '$1%' - GROUP BY initial_query_id - ORDER BY min(event_time_microseconds) ASC + AND query_id = initial_query_id + AND type = 'QueryFinish' + AND current_database = '$CLICKHOUSE_DATABASE' + ORDER BY event_time_microseconds ASC FORMAT TSV" } @@ -48,11 +49,11 @@ function run_query_with_pure_parallel_replicas () { --query "$3" \ --query_id "${1}_pure" \ --max_parallel_replicas 3 \ - --prefer_localhost_replica 1 \ --cluster_for_parallel_replicas "parallel_replicas" \ - --allow_experimental_parallel_reading_from_replicas 1 \ + --enable_parallel_replicas 1 \ --parallel_replicas_for_non_replicated_merge_tree 1 \ --parallel_replicas_min_number_of_rows_per_replica "$2" \ + --max_threads 5 \ |& grep "It is enough work for" | awk '{ print substr($7, 2, length($7) - 2) "\t" $20 " estimated parallel replicas" }' } diff --git a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.reference b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.reference index 521e3e2edbc2..595b35db610c 100644 --- a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.reference +++ b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.reference @@ -4,8 +4,8 @@ 02784_automatic_parallel_replicas_join-default_simple_join_1M_pure 10 estimated parallel replicas 02784_automatic_parallel_replicas_join-default_simple_join_300k_pure 3 estimated parallel replicas 02784_automatic_parallel_replicas_join-default_simple_join_300k_pure 33 estimated parallel replicas -02784_automatic_parallel_replicas_join-default_simple_join_0_pure Distinct parallel subqueries: 2 Used parallel replicas: true -02784_automatic_parallel_replicas_join-default_simple_join_10M_pure Distinct parallel subqueries: 0 Used parallel replicas: false -02784_automatic_parallel_replicas_join-default_simple_join_5M_pure Distinct parallel subqueries: 1 Used parallel replicas: true -02784_automatic_parallel_replicas_join-default_simple_join_1M_pure Distinct parallel subqueries: 1 Used parallel replicas: true -02784_automatic_parallel_replicas_join-default_simple_join_300k_pure Distinct parallel subqueries: 2 Used parallel replicas: true +02784_automatic_parallel_replicas_join-default_simple_join_0_pure Used parallel replicas: true +02784_automatic_parallel_replicas_join-default_simple_join_10M_pure Used parallel replicas: false +02784_automatic_parallel_replicas_join-default_simple_join_5M_pure Used parallel replicas: true +02784_automatic_parallel_replicas_join-default_simple_join_1M_pure Used parallel replicas: true +02784_automatic_parallel_replicas_join-default_simple_join_300k_pure Used parallel replicas: true diff --git a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.sh b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.sh index 801cd22b79f6..61075de0f689 100755 --- a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.sh +++ b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.sh @@ -9,17 +9,17 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_CLIENT_TRACE=${CLICKHOUSE_CLIENT/"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"/"--send_logs_level=trace"} function were_parallel_replicas_used () { - # Not using current_database = '$CLICKHOUSE_DATABASE' as nested parallel queries aren't run with it $CLICKHOUSE_CLIENT --query " SELECT initial_query_id, - concat('Distinct parallel subqueries: ' , countDistinctIf(query, initial_query_id != query_id)::String) as subqueries_parallelized, - concat('Used parallel replicas: ', (countIf(initial_query_id != query_id) != 0)::bool::String) as used + concat('Used parallel replicas: ', (ProfileEvents['ParallelReplicasUsedCount'] > 0)::bool::String) as used FROM system.query_log WHERE event_date >= yesterday() AND initial_query_id LIKE '$1%' - GROUP BY initial_query_id - ORDER BY min(event_time_microseconds) ASC + AND query_id = initial_query_id + AND type = 'QueryFinish' + AND current_database = '$CLICKHOUSE_DATABASE' + ORDER BY event_time_microseconds ASC FORMAT TSV" } @@ -67,7 +67,7 @@ function run_query_with_pure_parallel_replicas () { --prefer_localhost_replica 1 \ --parallel_replicas_prefer_local_join 0 \ --cluster_for_parallel_replicas "parallel_replicas" \ - --allow_experimental_parallel_reading_from_replicas 1 \ + --enable_parallel_replicas 1 \ --parallel_replicas_for_non_replicated_merge_tree 1 \ --parallel_replicas_min_number_of_rows_per_replica "$2" \ |& grep "It is enough work for" | awk '{ print substr($7, 2, length($7) - 2) "\t" $20 " estimated parallel replicas" }' | sort -n -k2 -b | grep -Pv "\t0 estimated parallel replicas" diff --git a/tests/queries/0_stateless/02811_parallel_replicas_prewhere_count.sql b/tests/queries/0_stateless/02811_parallel_replicas_prewhere_count.sql index 294c1325ba63..b68faa8500cb 100644 --- a/tests/queries/0_stateless/02811_parallel_replicas_prewhere_count.sql +++ b/tests/queries/0_stateless/02811_parallel_replicas_prewhere_count.sql @@ -10,7 +10,8 @@ SELECT count() FROM users PREWHERE uid > 2000; -- enable parallel replicas but with high rows threshold SET -allow_experimental_parallel_reading_from_replicas=1, +skip_unavailable_shards=1, +enable_parallel_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1, diff --git a/tests/queries/0_stateless/02835_parallel_replicas_over_distributed.sql b/tests/queries/0_stateless/02835_parallel_replicas_over_distributed.sql index 1e6f9304c0cb..d5489ce657d1 100644 --- a/tests/queries/0_stateless/02835_parallel_replicas_over_distributed.sql +++ b/tests/queries/0_stateless/02835_parallel_replicas_over_distributed.sql @@ -14,13 +14,13 @@ insert into test select *, today() from numbers(100); SELECT count(), min(id), max(id), avg(id) FROM test_d -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1; insert into test select *, today() from numbers(100); SELECT count(), min(id), max(id), avg(id) FROM test_d -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1; -- 2 shards @@ -38,10 +38,10 @@ insert into test2 select *, today() from numbers(100); SELECT count(), min(id), max(id), avg(id) FROM test2_d -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1; insert into test2 select *, today() from numbers(100); SELECT count(), min(id), max(id), avg(id) FROM test2_d -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1; diff --git a/tests/queries/0_stateless/02841_parallel_replicas_summary.sh b/tests/queries/0_stateless/02841_parallel_replicas_summary.sh index c82d2c8b0c05..928d73a4a70b 100755 --- a/tests/queries/0_stateless/02841_parallel_replicas_summary.sh +++ b/tests/queries/0_stateless/02841_parallel_replicas_summary.sh @@ -27,6 +27,7 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE replicas_summary (n Int64) ENGINE = Mer query_id_base="02841_summary_$CLICKHOUSE_DATABASE" +# TODO: rethink the test, for now temporary disable parallel_replicas_local_plan echo " SELECT * FROM replicas_summary @@ -34,9 +35,10 @@ echo " SETTINGS max_parallel_replicas = 2, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', - allow_experimental_parallel_reading_from_replicas = 2, + enable_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, - interactive_delay=0 + interactive_delay=0, + parallel_replicas_local_plan=0 "\ | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&query_id=${query_id_base}_interactive_0" --data-binary @- -vvv 2>&1 \ | grep "Summary" | grep -cv '"read_rows":"0"' @@ -48,9 +50,10 @@ echo " SETTINGS max_parallel_replicas = 2, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', - allow_experimental_parallel_reading_from_replicas = 2, + enable_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, - interactive_delay=99999999999 + interactive_delay=99999999999, + parallel_replicas_local_plan=0 "\ | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&query_id=${query_id_base}_interactive_high" --data-binary @- -vvv 2>&1 \ | grep "Summary" | grep -cv '"read_rows":"0"' diff --git a/tests/queries/0_stateless/02861_index_set_incorrect_args.sql b/tests/queries/0_stateless/02861_index_set_incorrect_args.sql index 17b505cd0516..985e2a17ff0b 100644 --- a/tests/queries/0_stateless/02861_index_set_incorrect_args.sql +++ b/tests/queries/0_stateless/02861_index_set_incorrect_args.sql @@ -2,5 +2,5 @@ DROP TABLE IF EXISTS set_index__fuzz_41; CREATE TABLE set_index__fuzz_41 (`a` Date, `b` Nullable(DateTime64(3)), INDEX b_set b TYPE set(0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO set_index__fuzz_41 (a) VALUES (today()); -SELECT b FROM set_index__fuzz_41 WHERE and(b = 256) SETTINGS force_data_skipping_indices = 'b_set', optimize_move_to_prewhere = 0, max_parallel_replicas=2, parallel_replicas_for_non_replicated_merge_tree=1, allow_experimental_parallel_reading_from_replicas=2; -- { serverError TOO_FEW_ARGUMENTS_FOR_FUNCTION } +SELECT b FROM set_index__fuzz_41 WHERE and(b = 256) SETTINGS force_data_skipping_indices = 'b_set', optimize_move_to_prewhere = 0, max_parallel_replicas=2, parallel_replicas_for_non_replicated_merge_tree=1, enable_parallel_replicas=2; -- { serverError TOO_FEW_ARGUMENTS_FOR_FUNCTION } DROP TABLE set_index__fuzz_41; diff --git a/tests/queries/0_stateless/02869_parallel_replicas_read_from_several.sql b/tests/queries/0_stateless/02869_parallel_replicas_read_from_several.sql index e040fae1fa6b..c4359f059ba3 100644 --- a/tests/queries/0_stateless/02869_parallel_replicas_read_from_several.sql +++ b/tests/queries/0_stateless/02869_parallel_replicas_read_from_several.sql @@ -24,5 +24,5 @@ system sync replica t3; SELECT count(), min(k), max(k), avg(k) FROM t1 -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, prefer_localhost_replica = 0, +SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, prefer_localhost_replica = 0, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost', parallel_replicas_single_task_marks_count_multiplier = 0.001; diff --git a/tests/queries/0_stateless/02875_parallel_replicas_cluster_all_replicas.sql b/tests/queries/0_stateless/02875_parallel_replicas_cluster_all_replicas.sql index f59d38ceb046..51301fe4ea65 100644 --- a/tests/queries/0_stateless/02875_parallel_replicas_cluster_all_replicas.sql +++ b/tests/queries/0_stateless/02875_parallel_replicas_cluster_all_replicas.sql @@ -2,12 +2,12 @@ DROP TABLE IF EXISTS tt; CREATE TABLE tt (n UInt64) ENGINE=MergeTree() ORDER BY tuple(); INSERT INTO tt SELECT * FROM numbers(10); -SET allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1; +SET enable_parallel_replicas=1, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1; SELECT count() FROM clusterAllReplicas('test_cluster_two_shard_three_replicas_localhost', currentDatabase(), tt) settings log_comment='02875_190aed82-2423-413b-ad4c-24dcca50f65b'; SYSTEM FLUSH LOGS; -SET allow_experimental_parallel_reading_from_replicas=0; +SET enable_parallel_replicas=0; SELECT count() > 0 FROM system.text_log WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment = '02875_190aed82-2423-413b-ad4c-24dcca50f65b') AND message LIKE '%Parallel reading from replicas is disabled for cluster%'; diff --git a/tests/queries/0_stateless/02875_parallel_replicas_remote.sql b/tests/queries/0_stateless/02875_parallel_replicas_remote.sql index 5fbaf34b621e..259ed02b2a3a 100644 --- a/tests/queries/0_stateless/02875_parallel_replicas_remote.sql +++ b/tests/queries/0_stateless/02875_parallel_replicas_remote.sql @@ -2,12 +2,12 @@ DROP TABLE IF EXISTS tt; CREATE TABLE tt (n UInt64) ENGINE=MergeTree() ORDER BY tuple(); INSERT INTO tt SELECT * FROM numbers(10); -SET allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1; +SET enable_parallel_replicas=1, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1; SELECT count() FROM remote('127.0.0.{1..6}', currentDatabase(), tt) settings log_comment='02875_89f3c39b-1919-48cb-b66e-ef9904e73146'; SYSTEM FLUSH LOGS; -SET allow_experimental_parallel_reading_from_replicas=0; +SET enable_parallel_replicas=0; SELECT count() > 0 FROM system.text_log WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment = '02875_89f3c39b-1919-48cb-b66e-ef9904e73146') AND message LIKE '%Parallel reading from replicas is disabled for cluster%'; diff --git a/tests/queries/0_stateless/02898_parallel_replicas_custom_key_final.sql b/tests/queries/0_stateless/02898_parallel_replicas_custom_key_final.sql index 7df88988bab1..32e2592430d3 100644 --- a/tests/queries/0_stateless/02898_parallel_replicas_custom_key_final.sql +++ b/tests/queries/0_stateless/02898_parallel_replicas_custom_key_final.sql @@ -8,12 +8,12 @@ SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02898_parallel_replicas_final) FINAL GROUP BY y ORDER BY y -SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default'; +SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_sampling'; SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02898_parallel_replicas_final) FINAL GROUP BY y ORDER BY y -SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='range'; +SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_range'; -DROP TABLE 02898_parallel_replicas_final; \ No newline at end of file +DROP TABLE 02898_parallel_replicas_final; diff --git a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql index d8bfec12b3a8..cbb4ff5334c8 100644 --- a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql +++ b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql @@ -14,7 +14,7 @@ system sync replica t1; system sync replica t2; system sync replica t3; -SET allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; +SET enable_parallel_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; -- default coordinator SELECT count(), min(k), max(k), avg(k) FROM t1 SETTINGS log_comment='02898_default_190aed82-2423-413b-ad4c-24dcca50f65b'; @@ -23,15 +23,16 @@ SELECT count(), min(k), max(k), avg(k) FROM t1 SETTINGS log_comment='02898_defau SYSTEM FLUSH LOGS; SELECT count() > 0 FROM system.text_log WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment='02898_default_190aed82-2423-413b-ad4c-24dcca50f65b') - AND message LIKE '%Total rows to read: 3000%' SETTINGS allow_experimental_parallel_reading_from_replicas=0; + AND message LIKE '%Total rows to read: 3000%' SETTINGS enable_parallel_replicas=0; -- reading in order coordinator -SELECT k, sipHash64(v) FROM t1 order by k limit 5 offset 998 SETTINGS optimize_read_in_order=1, log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b'; +-- disable parallel_replicas_local_plan since the test relay on traces which only present in case of no local plan +SELECT k, sipHash64(v) FROM t1 order by k limit 5 offset 998 SETTINGS optimize_read_in_order=1, parallel_replicas_local_plan=0, log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b'; SYSTEM FLUSH LOGS; SELECT count() > 0 FROM system.text_log WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b') - AND message LIKE '%Updated total rows to read: added % rows, total 3000 rows%' SETTINGS allow_experimental_parallel_reading_from_replicas=0; + AND message LIKE '%Updated total rows to read: added % rows, total 3000 rows%' SETTINGS enable_parallel_replicas=0; DROP TABLE t1 SYNC; DROP TABLE t2 SYNC; diff --git a/tests/queries/0_stateless/02901_parallel_replicas_rollup.sh b/tests/queries/0_stateless/02901_parallel_replicas_rollup.sh index def813c17b47..2fc4502c5314 100755 --- a/tests/queries/0_stateless/02901_parallel_replicas_rollup.sh +++ b/tests/queries/0_stateless/02901_parallel_replicas_rollup.sh @@ -30,7 +30,7 @@ $CLICKHOUSE_CLIENT \ --max_parallel_replicas 3 \ --prefer_localhost_replica 1 \ --cluster_for_parallel_replicas "test_cluster_one_shard_three_replicas_localhost" \ - --allow_experimental_parallel_reading_from_replicas 1 \ + --enable_parallel_replicas 1 \ --parallel_replicas_for_non_replicated_merge_tree 1 \ --parallel_replicas_min_number_of_rows_per_replica 0 \ --query " @@ -68,7 +68,7 @@ $CLICKHOUSE_CLIENT \ --max_parallel_replicas 3 \ --prefer_localhost_replica 1 \ --cluster_for_parallel_replicas "test_cluster_one_shard_three_replicas_localhost" \ - --allow_experimental_parallel_reading_from_replicas 1 \ + --enable_parallel_replicas 1 \ --parallel_replicas_for_non_replicated_merge_tree 1 \ --parallel_replicas_min_number_of_rows_per_replica 0 \ --query "SELECT * FROM (SELECT year, month, day, count(*) FROM days GROUP BY year, month, day WITH ROLLUP) ORDER BY 1, 2, 3"; diff --git a/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.reference b/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.reference index 2d97dd0e12e7..490d36aafdf8 100644 --- a/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.reference +++ b/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.reference @@ -3,7 +3,7 @@ SELECT y, count() FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas) GROUP BY y ORDER BY y -SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default'; +SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_sampling'; 0 250 1 250 2 250 @@ -12,7 +12,7 @@ SELECT y, count() FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas) GROUP BY y ORDER BY y -SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='range'; +SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_range'; 0 250 1 250 2 250 @@ -22,7 +22,7 @@ SELECT y, count() FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas) GROUP BY y ORDER BY y -SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default'; +SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_sampling'; 0 250 1 250 2 250 diff --git a/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.sql b/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.sql index c36b5bebd582..763a4530b944 100644 --- a/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.sql +++ b/tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.sql @@ -14,20 +14,20 @@ SELECT y, count() FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas) GROUP BY y ORDER BY y -SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default'; +SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_sampling'; SELECT y, count() FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas) GROUP BY y ORDER BY y -SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='range'; +SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_range'; SET use_hedged_requests=0; SELECT y, count() FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas) GROUP BY y ORDER BY y -SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default'; +SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_sampling'; -- { echoOff } SET send_logs_level='warning'; diff --git a/tests/queries/0_stateless/02922_analyzer_aggregate_nothing_type.sql b/tests/queries/0_stateless/02922_analyzer_aggregate_nothing_type.sql index 5b9343c6e13b..c855e01a2b3a 100644 --- a/tests/queries/0_stateless/02922_analyzer_aggregate_nothing_type.sql +++ b/tests/queries/0_stateless/02922_analyzer_aggregate_nothing_type.sql @@ -88,7 +88,7 @@ CREATE TABLE t1 (`n` UInt64) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO t1 SELECT * FROM numbers(10); SET - allow_experimental_parallel_reading_from_replicas=1, +enable_parallel_replicas=1, max_parallel_replicas=2, use_hedged_requests=0, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost', diff --git a/tests/queries/0_stateless/02932_parallel_replicas_fuzzer.sql b/tests/queries/0_stateless/02932_parallel_replicas_fuzzer.sql index e5cbe100a58b..038f5c1c98a3 100644 --- a/tests/queries/0_stateless/02932_parallel_replicas_fuzzer.sql +++ b/tests/queries/0_stateless/02932_parallel_replicas_fuzzer.sql @@ -9,7 +9,7 @@ SELECT key, value1, value2, toUInt64(min(time)) AS start_ts FROM join_inner_tabl max_parallel_replicas = 3, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', - allow_experimental_parallel_reading_from_replicas = 1, + enable_parallel_replicas = 1, use_hedged_requests = 0; @@ -22,7 +22,7 @@ ORDER BY nan DESC, [0, NULL, NULL, NULL, NULL] DESC FORMAT Null SETTINGS max_parallel_replicas = 3, - allow_experimental_parallel_reading_from_replicas = 1, + enable_parallel_replicas = 1, use_hedged_requests = 0, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; @@ -35,4 +35,4 @@ ORDER BY nan DESC, _CAST([0, NULL, NULL, NULL, NULL], 'Array(Nullable(UInt8))') DESC FORMAT Null -SETTINGS receive_timeout = 10., receive_data_timeout_ms = 10000, use_hedged_requests = 0, allow_suspicious_low_cardinality_types = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, log_queries = 1, table_function_remote_max_addresses = 200, enable_analyzer = 1; +SETTINGS receive_timeout = 10., receive_data_timeout_ms = 10000, use_hedged_requests = 0, allow_suspicious_low_cardinality_types = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, log_queries = 1, table_function_remote_max_addresses = 200, enable_analyzer = 1; diff --git a/tests/queries/0_stateless/02935_parallel_replicas_settings.sql b/tests/queries/0_stateless/02935_parallel_replicas_settings.sql index be6f1c2958cc..5e3b1a13f1ac 100644 --- a/tests/queries/0_stateless/02935_parallel_replicas_settings.sql +++ b/tests/queries/0_stateless/02935_parallel_replicas_settings.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS test_parallel_replicas_settings; CREATE TABLE test_parallel_replicas_settings (n UInt64) ENGINE=MergeTree() ORDER BY tuple(); INSERT INTO test_parallel_replicas_settings SELECT * FROM numbers(10); -SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1; +SET enable_parallel_replicas=2, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1; SET cluster_for_parallel_replicas=''; SELECT count() FROM test_parallel_replicas_settings WHERE NOT ignore(*); -- { serverError CLUSTER_DOESNT_EXIST } @@ -16,20 +16,20 @@ SELECT count() > 0 FROM system.text_log WHERE yesterday() <= event_date AND query_id in (select query_id from system.query_log where current_database=currentDatabase() AND log_comment='0_f621c4f2-4da7-4a7c-bb6d-052c442d0f7f') AND level = 'Information' - AND message ILIKE '%Disabling ''use_hedged_requests'' in favor of ''allow_experimental_parallel_reading_from_replicas''%' -SETTINGS allow_experimental_parallel_reading_from_replicas=0; + AND message ILIKE '%Disabling ''use_hedged_requests'' in favor of ''enable_parallel_replicas''%' +SETTINGS enable_parallel_replicas=0; SET use_hedged_requests=1; SELECT count() FROM test_parallel_replicas_settings WHERE NOT ignore(*) settings log_comment='1_f621c4f2-4da7-4a7c-bb6d-052c442d0f7f'; SYSTEM FLUSH LOGS; -SET allow_experimental_parallel_reading_from_replicas=0; +SET enable_parallel_replicas=0; SELECT count() > 0 FROM system.text_log WHERE yesterday() <= event_date AND query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment = '1_f621c4f2-4da7-4a7c-bb6d-052c442d0f7f') AND level = 'Warning' - AND message ILIKE '%Setting ''use_hedged_requests'' explicitly with enabled ''allow_experimental_parallel_reading_from_replicas'' has no effect%' -SETTINGS allow_experimental_parallel_reading_from_replicas=0; + AND message ILIKE '%Setting ''use_hedged_requests'' explicitly with enabled ''enable_parallel_replicas'' has no effect%' +SETTINGS enable_parallel_replicas=0; DROP TABLE test_parallel_replicas_settings; diff --git a/tests/queries/0_stateless/02946_parallel_replicas_distributed.sql b/tests/queries/0_stateless/02946_parallel_replicas_distributed.sql index 1afd4ff01929..e6de0e8cee64 100644 --- a/tests/queries/0_stateless/02946_parallel_replicas_distributed.sql +++ b/tests/queries/0_stateless/02946_parallel_replicas_distributed.sql @@ -11,7 +11,7 @@ ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDat SELECT count(), sum(id) FROM test_d -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1; DROP TABLE test_d; DROP TABLE test; diff --git a/tests/queries/0_stateless/02946_parallel_replicas_force_primary_key.sql b/tests/queries/0_stateless/02946_parallel_replicas_force_primary_key.sql index d33c8cdbc933..b307932412a0 100644 --- a/tests/queries/0_stateless/02946_parallel_replicas_force_primary_key.sql +++ b/tests/queries/0_stateless/02946_parallel_replicas_force_primary_key.sql @@ -22,10 +22,10 @@ FROM t1 WHERE k > 0 GROUP BY k ORDER BY k -SETTINGS force_primary_key = 1, allow_experimental_parallel_reading_from_replicas = 0; +SETTINGS force_primary_key = 1, enable_parallel_replicas = 0; -- parallel replicas, primary key is used -SET allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; +SET enable_parallel_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; SELECT k, count() diff --git a/tests/queries/0_stateless/02947_parallel_replicas_remote.sql b/tests/queries/0_stateless/02947_parallel_replicas_remote.sql index 345d9f9cb03f..2f8d777ab368 100644 --- a/tests/queries/0_stateless/02947_parallel_replicas_remote.sql +++ b/tests/queries/0_stateless/02947_parallel_replicas_remote.sql @@ -7,6 +7,6 @@ AS select *, '2023-12-25' from numbers(100); SELECT count(), sum(id) FROM remote('127.0.0.1|127.0.0.2|127.0.0.3|127.0.0.4', currentDatabase(), test) -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 4, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree = 1; -- { serverError CLUSTER_DOESNT_EXIST } +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 4, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree = 1; -- { serverError CLUSTER_DOESNT_EXIST } DROP TABLE test; diff --git a/tests/queries/0_stateless/02949_parallel_replicas_in_subquery.sql b/tests/queries/0_stateless/02949_parallel_replicas_in_subquery.sql index ab6e1532299c..1e71d2feba04 100644 --- a/tests/queries/0_stateless/02949_parallel_replicas_in_subquery.sql +++ b/tests/queries/0_stateless/02949_parallel_replicas_in_subquery.sql @@ -8,23 +8,23 @@ INSERT INTO merge_tree_in_subqueries VALUES(5, 'test5', 0); SET max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost', parallel_replicas_for_non_replicated_merge_tree=1; -SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS allow_experimental_parallel_reading_from_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED } -SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS allow_experimental_parallel_reading_from_replicas=2, parallel_replicas_allow_in_with_subquery=1; -SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS allow_experimental_parallel_reading_from_replicas=1; +SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED } +SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1; +SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS enable_parallel_replicas=1; SELECT '---'; -SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS allow_experimental_parallel_reading_from_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED }; -SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS allow_experimental_parallel_reading_from_replicas=2, parallel_replicas_allow_in_with_subquery=1; -SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS allow_experimental_parallel_reading_from_replicas=1; +SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED }; +SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1; +SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS enable_parallel_replicas=1; SELECT '---'; -SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS allow_experimental_parallel_reading_from_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED }; -SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS allow_experimental_parallel_reading_from_replicas=2, parallel_replicas_allow_in_with_subquery=1; -SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS allow_experimental_parallel_reading_from_replicas=1; +SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED }; +SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1; +SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS enable_parallel_replicas=1; -- IN with tuples is allowed SELECT '---'; -SELECT id, name FROM merge_tree_in_subqueries WHERE (id, name) IN (3, 'test3') SETTINGS allow_experimental_parallel_reading_from_replicas=2, parallel_replicas_allow_in_with_subquery=0; -SELECT id, name FROM merge_tree_in_subqueries WHERE (id, name) IN (3, 'test3') SETTINGS allow_experimental_parallel_reading_from_replicas=2, parallel_replicas_allow_in_with_subquery=1; +SELECT id, name FROM merge_tree_in_subqueries WHERE (id, name) IN (3, 'test3') SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0; +SELECT id, name FROM merge_tree_in_subqueries WHERE (id, name) IN (3, 'test3') SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1; DROP TABLE IF EXISTS merge_tree_in_subqueries; diff --git a/tests/queries/0_stateless/02949_parallel_replicas_scalar_subquery_big_integer.sql b/tests/queries/0_stateless/02949_parallel_replicas_scalar_subquery_big_integer.sql index 26f87180ab21..3b5f6277df7c 100644 --- a/tests/queries/0_stateless/02949_parallel_replicas_scalar_subquery_big_integer.sql +++ b/tests/queries/0_stateless/02949_parallel_replicas_scalar_subquery_big_integer.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS test; CREATE TABLE test (x UInt8) ENGINE = MergeTree ORDER BY x; INSERT INTO test VALUES (1), (2), (3); -SET allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree = 1; +SET enable_parallel_replicas = 1, max_parallel_replicas = 2, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree = 1; WITH (SELECT '111111111111111111111111111111111111111'::UInt128) AS v SELECT sum(x), max(v) FROM test; diff --git a/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference b/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference index 21b7b527b7a5..c1265b7ca145 100644 --- a/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference +++ b/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference @@ -1,4 +1,4 @@ -100 4950 +10000 49995000 1 89 90 @@ -6,3 +6,9 @@ 92 93 1 +93 +92 +91 +90 +89 +1 diff --git a/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql b/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql index 22f55acd3652..b9295dbc2317 100644 --- a/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql +++ b/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql @@ -2,24 +2,30 @@ DROP TABLE IF EXISTS test; CREATE TABLE test (k UInt64, v String) ENGINE = MergeTree -ORDER BY k; +ORDER BY k +SETTINGS index_granularity=1; -INSERT INTO test SELECT number, toString(number) FROM numbers(100); - -SET allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; +INSERT INTO test SELECT number, toString(number) FROM numbers(10_000); +SET enable_parallel_replicas = 2, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; -- default coordinator SELECT count(), sum(k) FROM test SETTINGS log_comment = '02950_parallel_replicas_used_replicas_count'; SYSTEM FLUSH LOGS; -SELECT ProfileEvents['ParallelReplicasUsedCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0; +SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS enable_parallel_replicas=0; -- In order coordinator -SELECT k FROM test order by k limit 5 offset 89 SETTINGS optimize_read_in_order=1, log_comment='02950_parallel_replicas_used_replicas_count_2'; +SELECT k FROM test order by k limit 5 offset 89 SETTINGS optimize_read_in_order=1, log_comment='02950_parallel_replicas_used_replicas_count_2', merge_tree_min_rows_for_concurrent_read=1, max_threads=1; + +SYSTEM FLUSH LOGS; +SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_2' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS enable_parallel_replicas=0; + +-- In reverse order coordinator +SELECT k FROM test order by k desc limit 5 offset 9906 SETTINGS optimize_read_in_order=1, log_comment='02950_parallel_replicas_used_replicas_count_3', merge_tree_min_rows_for_concurrent_read=1, max_threads=1; SYSTEM FLUSH LOGS; -SELECT ProfileEvents['ParallelReplicasUsedCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_2' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0; +SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_3' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS enable_parallel_replicas=0; DROP TABLE test; diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.reference index c0485b817c4e..87e78d7d1169 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.reference @@ -11,10 +11,8 @@ nested join with analyzer and parallel replicas, both global 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done nested join with analyzer and parallel replicas, global + local @@ -29,10 +27,8 @@ nested join with analyzer and parallel replicas, global + local 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting merge join @@ -47,10 +43,8 @@ nested join with analyzer and parallel replicas, both local, both full sorting m 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) WithOrderCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting and hash join @@ -65,10 +59,8 @@ nested join with analyzer and parallel replicas, both local, both full sorting a 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting and hash join @@ -83,8 +75,6 @@ nested join with analyzer and parallel replicas, both local, both full sorting a 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) WithOrderCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh index e954cb0e78e0..6c296257c538 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh @@ -17,6 +17,8 @@ insert into num_1 select number * 2, toString(number * 2) from numbers(1e7); insert into num_2 select number * 3, -number from numbers(1.5e6); " +PARALLEL_REPLICAS_SETTINGS="enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_local_plan=1" + ############## echo echo "nested join with analyzer and parallel replicas, both global" @@ -26,18 +28,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=0) r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS enable_analyzer=1, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=0) r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS enable_analyzer=1, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 | +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, send_logs_level='trace', parallel_replicas_prefer_local_join=0" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -51,18 +49,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS enable_analyzer=1, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS enable_analyzer=1, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 | +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, send_logs_level='trace', parallel_replicas_prefer_local_join=0" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -77,18 +71,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" 2>&1 | +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, send_logs_level='trace', join_algorithm='full_sorting_merge'" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -102,18 +92,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='hash') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='hash') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" 2>&1 | +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, send_logs_level='trace', join_algorithm='full_sorting_merge'" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -127,18 +113,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='hash'" +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='hash'" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='hash'" 2>&1 | +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, send_logs_level='trace', join_algorithm='hash'" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' diff --git a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference index 36f02b2f764c..93003b6cf6d8 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference @@ -2,7 +2,7 @@ set parallel_replicas_prefer_local_join = 0; -- A query with only INNER/LEFT joins is fully send to replicas. JOIN is executed in GLOBAL mode. -select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -18,9 +18,21 @@ select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x 13 13 0 0 0 0 14 14 14 14 0 0 15 15 0 0 0 0 -explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -Expression - ReadFromRemoteParallelReplicas +explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +Union + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + ReadFromMemoryStorage + Expression + ReadFromMemoryStorage + Expression + ReadFromRemoteParallelReplicas -- -- The same query with cte; with sub1 as (select x, y from tab1 where x != 2), @@ -29,7 +41,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -52,11 +64,25 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Sorting + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + ReadFromMemoryStorage + Expression + ReadFromMemoryStorage + Expression + ReadFromRemoteParallelReplicas -- -- GROUP BY should work up to WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -65,7 +91,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key) select * from sub5 order by key -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 54 54 50 50 12 12 0 64 64 0 0 0 0 1 explain description=0 @@ -75,13 +101,27 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key) select * from sub5 order by key -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting Expression MergingAggregated - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Aggregating + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + ReadFromMemoryStorage + Expression + ReadFromMemoryStorage + Expression + ReadFromRemoteParallelReplicas -- -- ORDER BY in sub3 : sub1 is fully pushed, sub3 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -90,7 +130,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -113,15 +153,27 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting Expression Join - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + ReadFromMemoryStorage + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- ORDER BY in sub1 : sub1 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2 order by y), @@ -130,7 +182,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -153,19 +205,31 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting Expression Join Expression Join + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- RIGHT JOIN in sub3: sub3 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -174,7 +238,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 6 6 6 6 0 0 8 8 8 8 0 0 @@ -197,17 +261,29 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Join Expression Join + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- RIGHT JOIN in sub5: sub5 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -215,7 +291,7 @@ sub2 as (select y, z from tab2 where y != 4), sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y), sub4 as (select z, a from tab3 where z != 8), sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z) -select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 @@ -237,19 +313,31 @@ sub2 as (select y, z from tab2 where y != 4), sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y), sub4 as (select z, a from tab3 where z != 8), sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z) -select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting Expression Join - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas Expression Join - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- Subqueries for IN allowed with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)), @@ -258,7 +346,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -281,11 +369,31 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Sorting + Expression + Join + Expression + Join + Expression + CreatingSets + Expression + Expression + ReadFromMergeTree + CreatingSet + Expression + Filter + ReadFromSystemNumbers + Expression + ReadFromMemoryStorage + Expression + ReadFromMemoryStorage + Expression + ReadFromRemoteParallelReplicas -- -- Subqueries for IN are not allowed with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)), @@ -294,7 +402,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0; +SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -317,7 +425,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;-- { echoOn } +SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;-- { echoOn } Expression Sorting Expression @@ -333,13 +441,21 @@ Expression Expression Filter ReadFromSystemNumbers + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas set parallel_replicas_prefer_local_join = 1; -- A query with only INNER/LEFT joins is fully send to replicas. JOIN is executed in GLOBAL mode. -select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -355,9 +471,23 @@ select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x 13 13 0 0 0 0 14 14 14 14 0 0 15 15 0 0 0 0 -explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -Expression - ReadFromRemoteParallelReplicas +explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +Union + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- The same query with cte; with sub1 as (select x, y from tab1 where x != 2), @@ -366,7 +496,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -389,11 +519,27 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Sorting + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- GROUP BY should work up to WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -402,7 +548,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key) select * from sub5 order by key -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 54 54 50 50 12 12 0 64 64 0 0 0 0 1 explain description=0 @@ -412,13 +558,29 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key) select * from sub5 order by key -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting Expression MergingAggregated - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Aggregating + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- ORDER BY in sub3 : sub1 is fully pushed, sub3 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -427,7 +589,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -450,15 +612,28 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting Expression Join - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- ORDER BY in sub1 : sub1 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2 order by y), @@ -467,7 +642,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -490,19 +665,31 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting Expression Join Expression Join + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- RIGHT JOIN in sub3: sub3 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -511,7 +698,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 6 6 6 6 0 0 8 8 8 8 0 0 @@ -534,17 +721,29 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Join Expression Join + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- RIGHT JOIN in sub5: sub5 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -552,7 +751,7 @@ sub2 as (select y, z from tab2 where y != 4), sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y), sub4 as (select z, a from tab3 where z != 8), sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z) -select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 @@ -574,19 +773,31 @@ sub2 as (select y, z from tab2 where y != 4), sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y), sub4 as (select z, a from tab3 where z != 8), sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z) -select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting Expression Join - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas Expression Join - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- Subqueries for IN allowed with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)), @@ -595,7 +806,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -618,11 +829,33 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Expression Sorting - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Sorting + Expression + Join + Expression + Join + Expression + CreatingSets + Expression + Expression + ReadFromMergeTree + CreatingSet + Expression + Filter + ReadFromSystemNumbers + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- Subqueries for IN are not allowed with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)), @@ -631,7 +864,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0; +SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0; 0 0 0 0 0 0 1 1 0 0 0 0 3 3 0 0 0 0 @@ -654,7 +887,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0; +SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0; Expression Sorting Expression @@ -670,7 +903,15 @@ Expression Expression Filter ReadFromSystemNumbers + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas diff --git a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2 b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2 index 775663768fef..23291881eb4b 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2 +++ b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2 @@ -10,6 +10,8 @@ insert into tab1 select number, number, number from numbers(16); insert into tab2 select number * 2, number * 2 from numbers(8); insert into tab3 select number * 4, number * 4 from numbers(4); +set parallel_replicas_local_plan=1; + {% for use_global_in in [0, 1] -%} -- { echoOn } @@ -17,8 +19,8 @@ insert into tab3 select number * 4, number * 4 from numbers(4); set parallel_replicas_prefer_local_join = {{use_global_in}}; -- A query with only INNER/LEFT joins is fully send to replicas. JOIN is executed in GLOBAL mode. -select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -- -- The same query with cte; with sub1 as (select x, y from tab1 where x != 2), @@ -27,7 +29,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; explain description=0 with sub1 as (select x, y from tab1 where x != 2), @@ -36,7 +38,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -- -- GROUP BY should work up to WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -45,7 +47,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key) select * from sub5 order by key -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; explain description=0 with sub1 as (select x, y from tab1 where x != 2), @@ -54,7 +56,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key) select * from sub5 order by key -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -- -- ORDER BY in sub3 : sub1 is fully pushed, sub3 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -63,7 +65,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; explain description=0 with sub1 as (select x, y from tab1 where x != 2), @@ -72,7 +74,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -- -- ORDER BY in sub1 : sub1 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2 order by y), @@ -81,7 +83,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; explain description=0 with sub1 as (select x, y from tab1 where x != 2 order by y), @@ -90,7 +92,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -- -- RIGHT JOIN in sub3: sub3 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -99,7 +101,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; explain description=0 with sub1 as (select x, y from tab1 where x != 2), @@ -108,7 +110,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -- -- RIGHT JOIN in sub5: sub5 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -116,7 +118,7 @@ sub2 as (select y, z from tab2 where y != 4), sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y), sub4 as (select z, a from tab3 where z != 8), sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z) -select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; explain description=0 with sub1 as (select x, y from tab1 where x != 2), @@ -124,7 +126,7 @@ sub2 as (select y, z from tab2 where y != 4), sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y), sub4 as (select z, a from tab3 where z != 8), sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z) -select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -- -- Subqueries for IN allowed @@ -134,7 +136,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; explain description=0 with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)), @@ -143,7 +145,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -- -- Subqueries for IN are not allowed @@ -153,7 +155,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0; +SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0; explain description=0 with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)), @@ -162,6 +164,6 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y sub4 as (select z, a from tab3 where z != 8), sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z) select * from sub5 order by x -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0; +SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0; {%- endfor %} diff --git a/tests/queries/0_stateless/02972_parallel_replicas_cte.sql b/tests/queries/0_stateless/02972_parallel_replicas_cte.sql index 767cd61216b7..10eb1eb1d718 100644 --- a/tests/queries/0_stateless/02972_parallel_replicas_cte.sql +++ b/tests/queries/0_stateless/02972_parallel_replicas_cte.sql @@ -13,21 +13,21 @@ SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups. WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100) SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +SETTINGS enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- Testing that it is disabled for enable_analyzer=0. With analyzer it will be supported (with correct result) WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100) SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a -SETTINGS enable_analyzer = 0, allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED } +SETTINGS enable_analyzer = 0, enable_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED } --- Disabled for any value of allow_experimental_parallel_reading_from_replicas != 1, not just 2 +-- Disabled for any value of enable_parallel_replicas != 1, not just 2 WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100) SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a -SETTINGS enable_analyzer = 0, allow_experimental_parallel_reading_from_replicas = 512, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED } +SETTINGS enable_analyzer = 0, enable_parallel_replicas = 512, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED } -- Sanitizer SELECT count() FROM pr_2 JOIN numbers(10) as pr_1 ON pr_2.a = pr_1.number -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +SETTINGS enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- Parallel replicas detection should work inside subqueries SELECT * @@ -36,7 +36,7 @@ FROM WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100) SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a ) -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +SETTINGS enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- Subquery + subquery SELECT count() @@ -49,7 +49,7 @@ FROM SELECT count() as c FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a ) ) -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +SETTINGS enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; CREATE TABLE numbers_1e3 ( @@ -74,7 +74,7 @@ WITH ) SELECT count() FROM cte2 -SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; +SETTINGS enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; DROP TABLE IF EXISTS numbers_1e6; DROP TABLE IF EXISTS pr_1; diff --git a/tests/queries/0_stateless/02981_insert_select_resize_to_max_insert_threads.sh b/tests/queries/0_stateless/02981_insert_select_resize_to_max_insert_threads.sh index 7ad5a2179f9a..77eddbcf63fc 100755 --- a/tests/queries/0_stateless/02981_insert_select_resize_to_max_insert_threads.sh +++ b/tests/queries/0_stateless/02981_insert_select_resize_to_max_insert_threads.sh @@ -49,7 +49,7 @@ ${CLICKHOUSE_CLIENT} --max_insert_threads "$max_insert_threads" --parallel_distr """ | grep -v EmptySink | grep -c Sink echo "inserting into a remote table from remote (reading with parallel replicas) with concurrency max_insert_threads" -${CLICKHOUSE_CLIENT} --max_insert_threads "$max_insert_threads" --allow_experimental_parallel_reading_from_replicas 2 --cluster_for_parallel_replicas 'parallel_replicas' --max_parallel_replicas 3 -q """ +${CLICKHOUSE_CLIENT} --max_insert_threads "$max_insert_threads" --enable_parallel_replicas 2 --cluster_for_parallel_replicas 'parallel_replicas' --max_parallel_replicas 3 -q """ EXPLAIN PIPELINE INSERT INTO t3_dist SELECT * FROM t4_pr; diff --git a/tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.sql b/tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.sql index 210b7d2a18a5..d70da5b5ae7f 100644 --- a/tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.sql +++ b/tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS test_unexpected_cluster; CREATE TABLE test_unexpected_cluster (n UInt64) ENGINE=MergeTree() ORDER BY tuple(); INSERT INTO test_unexpected_cluster SELECT * FROM numbers(10); -SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=2, cluster_for_parallel_replicas='test_cluster_two_shards', parallel_replicas_for_non_replicated_merge_tree=1; +SET enable_parallel_replicas=2, max_parallel_replicas=2, cluster_for_parallel_replicas='test_cluster_two_shards', parallel_replicas_for_non_replicated_merge_tree=1; SELECT count() FROM test_unexpected_cluster WHERE NOT ignore(*); -- { serverError UNEXPECTED_CLUSTER } DROP TABLE test_unexpected_cluster; diff --git a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql index 7c8d6dd9affe..ff56540b3916 100644 --- a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql +++ b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql @@ -21,7 +21,7 @@ AS SELECT * FROM numbers(10); SET enable_analyzer = 1; -SET allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=0; +SET enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=0; EXPLAIN SYNTAX WITH diff --git a/tests/queries/0_stateless/03006_parallel_replicas_prewhere.reference b/tests/queries/0_stateless/03006_parallel_replicas_prewhere.reference index e69de29bb2d1..8b8d0e5d565c 100644 --- a/tests/queries/0_stateless/03006_parallel_replicas_prewhere.reference +++ b/tests/queries/0_stateless/03006_parallel_replicas_prewhere.reference @@ -0,0 +1 @@ +default url_na_log 1 130000 130 diff --git a/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql b/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql index 4b84646c0345..a2198086b107 100644 --- a/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql +++ b/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql @@ -21,11 +21,21 @@ SELECT FROM numbers(130000) SETTINGS max_insert_block_size = 200000; +SET max_block_size = 1048576, max_threads = 1, enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=10000; + +EXPLAIN ESTIMATE +SELECT count() +FROM url_na_log +PREWHERE (DateVisit >= toFixedString('2022-08-10', 10)) AND (DateVisit <= '2022-08-20') +SETTINGS parallel_replicas_local_plan=0; + +-- here parallel replicas uses local snapshot as working set +-- so, the estimation can be done EXPLAIN ESTIMATE SELECT count() FROM url_na_log PREWHERE (DateVisit >= toFixedString('2022-08-10', 10)) AND (DateVisit <= '2022-08-20') -SETTINGS max_block_size = 1048576, max_threads = 1, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=10000; +SETTINGS allow_experimental_analyzer=1, parallel_replicas_local_plan=1; DROP POLICY url_na_log_policy0 ON url_na_log; DROP TABLE url_na_log; diff --git a/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.reference b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.reference new file mode 100644 index 000000000000..99ede711976d --- /dev/null +++ b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.reference @@ -0,0 +1,347 @@ +1000 +bloom filter is off, all row groups should be read +expect rows_read = select count() +{ + "data": [ + { + "string": "AZSR", + "flba": "WNMM" + }, + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 2, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +bloom filter is on, some row groups should be skipped +expect rows_read much less than select count() +{ + "data": [ + { + "string": "AZSR", + "flba": "WNMM" + }, + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 2, + "statistics": { + "rows_read": 464, + "bytes_read": 21703 + } +} +bloom filter is on, but where predicate contains data from 2 row groups out of 3. +Rows read should be less than select count, but greater than previous selects +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + }, + { + "string": "ZHZK", + "flba": "HRWD" + } + ], + "rows": 2, + "statistics": { + "rows_read": 536, + "bytes_read": 25708 + } +} +bloom filter is on, but where predicate contains data from all row groups +expect rows_read = select count() +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + }, + { + "string": "OKAI", + "flba": "UXGT" + }, + { + "string": "ZHZK", + "flba": "HRWD" + } + ], + "rows": 3, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +IN check +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + }, + { + "string": "ZHZK", + "flba": "HRWD" + } + ], + "rows": 2, + "statistics": { + "rows_read": 536, + "bytes_read": 25708 + } +} +tuple in case, bf is off. +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +tuple in case, bf is on. +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 1, + "statistics": { + "rows_read": 464, + "bytes_read": 21703 + } +} +complex tuple in case, bf is off +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +complex tuple in case, bf is on +{ + "data": [ + { + "string": "PFJH", + "flba": "GKJC" + } + ], + "rows": 1, + "statistics": { + "rows_read": 464, + "bytes_read": 21703 + } +} +complex tuple in case, bf is on. Non existent +{ + "data": [], + "rows": 0, + "statistics": { + "rows_read": 0, + "bytes_read": 0 + } +} +Bloom filter for json column. BF is off +{ + "data": [ + { + "json": "{\"key\":38, \"value\":\"NXONM\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +Bloom filter for json column. BF is on +{ + "data": [ + { + "json": "{\"key\":38, \"value\":\"NXONM\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 72, + "bytes_read": 4005 + } +} +Bloom filter for ipv4 column. BF is off +{ + "data": [ + { + "json": "{\"key\":38, \"value\":\"NXONM\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +Bloom filter for ipv4 column. BF is on +{ + "data": [ + { + "json": "{\"key\":38, \"value\":\"NXONM\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 72, + "bytes_read": 4005 + } +} +Bloom filter for ipv4 column. BF is on. Specified in the schema +{ + "data": [ + { + "ipv4": "0.0.1.143" + } + ], + "rows": 1, + "statistics": { + "rows_read": 72, + "bytes_read": 4005 + } +} +Bloom filter on 64 bit column read as ipv4. We explicitly deny it, should read all rg +{ + "data": [ + { + "uint64_logical": "22.230.220.164" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +BF off for parquet uint64 logical type. Should read everything +{ + "data": [ + { + "json": "{\"key\":683, \"value\":\"YKCPD\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +BF on for parquet uint64 logical type. Uint64 is stored as a signed int 64, but with logical annotation. Make sure a value greater than int64 can be queried +{ + "data": [ + { + "json": "{\"key\":683, \"value\":\"YKCPD\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 464, + "bytes_read": 21711 + } +} +Uint16 is stored as physical type int32 with bidwidth = 16 and sign = false. Make sure a value greater than int16 can be queried. BF is on. +{ + "data": [ + { + "json": "{\"key\":874, \"value\":\"JENHW\"}" + } + ], + "rows": 1, + "statistics": { + "rows_read": 464, + "bytes_read": 21703 + } +} +BF off for parquet int8 logical type. Should read everything +{ + "data": [ + { + "json": "{\"key\":89, \"value\":\"MFIYP\"}" + }, + { + "json": "{\"key\":321, \"value\":\"JNOIA\"}" + }, + { + "json": "{\"key\":938, \"value\":\"UBMLO\"}" + }, + { + "json": "{\"key\":252, \"value\":\"ZVLKF\"}" + } + ], + "rows": 4, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +BF on for parquet int8 logical type. Should skip row groups +{ + "data": [ + { + "json": "{\"key\":89, \"value\":\"MFIYP\"}" + }, + { + "json": "{\"key\":321, \"value\":\"JNOIA\"}" + }, + { + "json": "{\"key\":938, \"value\":\"UBMLO\"}" + }, + { + "json": "{\"key\":252, \"value\":\"ZVLKF\"}" + } + ], + "rows": 4, + "statistics": { + "rows_read": 536, + "bytes_read": 25716 + } +} +Invalid column conversion with in operation. String type can not be hashed against parquet int64 physical type. Should read everything +{ + "data": [], + "rows": 0, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} +Transformations on key column shall not be allowed. Should read everything +{ + "data": [ + { + "uint64_logical": "7711695863945021976" + } + ], + "rows": 1, + "statistics": { + "rows_read": 1000, + "bytes_read": 47419 + } +} diff --git a/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.sh b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.sh new file mode 100755 index 000000000000..b7d40a1be63a --- /dev/null +++ b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +USER_FILES_PATH=$($CLICKHOUSE_CLIENT_BINARY --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +WORKING_DIR="${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +mkdir -p "${WORKING_DIR}" + +DATA_FILE="${CUR_DIR}/data_parquet/multi_column_bf.gz.parquet" + +DATA_FILE_USER_PATH="${WORKING_DIR}/multi_column_bf.gz.parquet" + +cp ${DATA_FILE} ${DATA_FILE_USER_PATH} + +${CLICKHOUSE_CLIENT} --query="select count(*) from file('${DATA_FILE_USER_PATH}', Parquet) SETTINGS use_cache_for_count_from_files=false;" + +echo "bloom filter is off, all row groups should be read" +echo "expect rows_read = select count()" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where string='PFJH' or flba='WNMM' order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false" | jq 'del(.meta,.statistics.elapsed)' + +echo "bloom filter is on, some row groups should be skipped" +echo "expect rows_read much less than select count()" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where string='PFJH' or flba='WNMM' order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false" | jq 'del(.meta,.statistics.elapsed)' + +echo "bloom filter is on, but where predicate contains data from 2 row groups out of 3." +echo "Rows read should be less than select count, but greater than previous selects" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where string='PFJH' or string='ZHZK' order by uint16_logical asc Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "bloom filter is on, but where predicate contains data from all row groups" +echo "expect rows_read = select count()" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where string='PFJH' or string='ZHZK' or uint64_logical=18441251162536403933 order by uint16_logical asc Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "IN check" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where string in ('PFJH', 'ZHZK') order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "tuple in case, bf is off." +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where (string, flba) in ('PFJH', 'GKJC') order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "tuple in case, bf is on." +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where (string, flba) in ('PFJH', 'GKJC') order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "complex tuple in case, bf is off" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where (string, flba) in (('NON1', 'NON1'), ('PFJH', 'GKJC'), ('NON2', 'NON2')) order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "complex tuple in case, bf is on" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where (string, flba) in (('NON1', 'NON1'), ('PFJH', 'GKJC'), ('NON2', 'NON2')) order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "complex tuple in case, bf is on. Non existent" +${CLICKHOUSE_CLIENT} --query="select string, flba from file('${DATA_FILE_USER_PATH}', Parquet) where (string, flba) in (('NON1', 'NON1'), ('NON2', 'NON2'), ('NON3', 'NON3')) order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter for json column. BF is off" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where json = '{\"key\":38, \"value\":\"NXONM\"}' order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter for json column. BF is on" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where json = '{\"key\":38, \"value\":\"NXONM\"}' order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter for ipv4 column. BF is off" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where ipv4 = IPv4StringToNum('0.0.1.143') order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter for ipv4 column. BF is on" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where ipv4 = IPv4StringToNum('0.0.1.143') order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter for ipv4 column. BF is on. Specified in the schema" +${CLICKHOUSE_CLIENT} --query="select ipv4 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv4 IPv4') where ipv4 = toIPv4('0.0.1.143') order by ipv4 asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Bloom filter on 64 bit column read as ipv4. We explicitly deny it, should read all rg" +${CLICKHOUSE_CLIENT} --query="select uint64_logical from file ('${DATA_FILE_USER_PATH}', Parquet, 'uint64_logical IPv4') where uint64_logical = toIPv4(5552715629697883300) order by uint64_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "BF off for parquet uint64 logical type. Should read everything" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where uint64_logical=18441251162536403933 order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "BF on for parquet uint64 logical type. Uint64 is stored as a signed int 64, but with logical annotation. Make sure a value greater than int64 can be queried" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where uint64_logical=18441251162536403933 order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Uint16 is stored as physical type int32 with bidwidth = 16 and sign = false. Make sure a value greater than int16 can be queried. BF is on." +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where uint16_logical=65528 order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "BF off for parquet int8 logical type. Should read everything" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where int8_logical=-126 order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "BF on for parquet int8 logical type. Should skip row groups" +${CLICKHOUSE_CLIENT} --query="select json from file('${DATA_FILE_USER_PATH}', Parquet) where int8_logical=-126 order by uint16_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Invalid column conversion with in operation. String type can not be hashed against parquet int64 physical type. Should read everything" +${CLICKHOUSE_CLIENT} --query="select uint64_logical from file('${DATA_FILE_USER_PATH}', Parquet, 'uint64_logical String') where uint64_logical in ('5') order by uint64_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "Transformations on key column shall not be allowed. Should read everything" +${CLICKHOUSE_CLIENT} --query="select uint64_logical from file('${DATA_FILE_USER_PATH}', Parquet) where negate(uint64_logical) = -7711695863945021976 order by uint64_logical asc FORMAT Json SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +rm -rf ${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME:?}/* diff --git a/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.reference b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.reference new file mode 100644 index 000000000000..acb66d986e5f --- /dev/null +++ b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.reference @@ -0,0 +1,76 @@ +bloom filter is off, row groups should be read +expect rows_read = select count() +{ + "data": [ + { + "ipv6": "7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995" + } + ], + "rows": 1, + "statistics": { + "rows_read": 5, + "bytes_read": 128 + } +} +bloom filter is on for ipv6, row groups should also be read since there is only one. Below queries just make sure the data is properly returned +{ + "data": [ + { + "ipv6": "7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995" + } + ], + "rows": 1, + "statistics": { + "rows_read": 5, + "bytes_read": 128 + } +} +{ + "data": [ + { + "ipv6": "7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995" + } + ], + "rows": 1, + "statistics": { + "rows_read": 5, + "bytes_read": 128 + } +} +{ + "data": [ + { + "toIPv6(ipv6)": "7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995" + } + ], + "rows": 1, + "statistics": { + "rows_read": 5, + "bytes_read": 128 + } +} +non existent ipv6, row group should be skipped +{ + "data": [], + "rows": 0, + "statistics": { + "rows_read": 0, + "bytes_read": 0 + } +} +{ + "data": [], + "rows": 0, + "statistics": { + "rows_read": 0, + "bytes_read": 0 + } +} +{ + "data": [], + "rows": 0, + "statistics": { + "rows_read": 5, + "bytes_read": 128 + } +} diff --git a/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.sh b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.sh new file mode 100755 index 000000000000..752e7ed38a58 --- /dev/null +++ b/tests/queries/0_stateless/03036_test_parquet_bloom_filter_push_down_ipv6.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +USER_FILES_PATH=$($CLICKHOUSE_CLIENT_BINARY --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +WORKING_DIR="${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +mkdir -p "${WORKING_DIR}" + +DATA_FILE="${CUR_DIR}/data_parquet/ipv6_bloom_filter.gz.parquet" + +DATA_FILE_USER_PATH="${WORKING_DIR}/ipv6_bloom_filter.gz.parquet" + +cp ${DATA_FILE} ${DATA_FILE_USER_PATH} + +echo "bloom filter is off, row groups should be read" +echo "expect rows_read = select count()" +${CLICKHOUSE_CLIENT} --query="select ipv6 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv6 IPv6') where ipv6 = '7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995' Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=false, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "bloom filter is on for ipv6, row groups should also be read since there is only one. Below queries just make sure the data is properly returned" +${CLICKHOUSE_CLIENT} --query="select ipv6 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv6 IPv6') where ipv6 = '7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995' Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' +${CLICKHOUSE_CLIENT} --query="select ipv6 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv6 IPv6') where ipv6 = toIPv6('7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995') Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' +${CLICKHOUSE_CLIENT} --query="select toIPv6(ipv6) from file('${DATA_FILE_USER_PATH}', Parquet) where ipv6 = toIPv6('7afe:b9d4:e754:4e78:8783:37f5:b2ea:9995') Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' + +echo "non existent ipv6, row group should be skipped" +${CLICKHOUSE_CLIENT} --query="select ipv6 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv6 IPv6') where ipv6 = 'fafe:b9d4:e754:4e78:8783:37f5:b2ea:9995' Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' +${CLICKHOUSE_CLIENT} --query="select ipv6 from file('${DATA_FILE_USER_PATH}', Parquet, 'ipv6 IPv6') where ipv6 = toIPv6('fafe:b9d4:e754:4e78:8783:37f5:b2ea:9995') Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' +${CLICKHOUSE_CLIENT} --query="select toIPv6(ipv6) from file('${DATA_FILE_USER_PATH}', Parquet) where ipv6 = toIPv6('fafe:b9d4:e754:4e78:8783:37f5:b2ea:9995') Format JSON SETTINGS input_format_parquet_bloom_filter_push_down=true, input_format_parquet_filter_push_down=false;" | jq 'del(.meta,.statistics.elapsed)' diff --git a/tests/queries/0_stateless/03093_reading_bug_with_parallel_replicas.sql b/tests/queries/0_stateless/03093_reading_bug_with_parallel_replicas.sql index 9592764baa60..00dd17c4904e 100644 --- a/tests/queries/0_stateless/03093_reading_bug_with_parallel_replicas.sql +++ b/tests/queries/0_stateless/03093_reading_bug_with_parallel_replicas.sql @@ -3,7 +3,7 @@ set max_threads = 16; set use_hedged_requests = 0; set max_parallel_replicas = 3; set cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; -set allow_experimental_parallel_reading_from_replicas = 1; +set enable_parallel_replicas = 1; set parallel_replicas_for_non_replicated_merge_tree = 1; set allow_aggregate_partitions_independently = 1; diff --git a/tests/queries/0_stateless/03143_parallel_replicas_mat_view_bug.sql b/tests/queries/0_stateless/03143_parallel_replicas_mat_view_bug.sql index 02a8a2f3ce07..10e1de815344 100644 --- a/tests/queries/0_stateless/03143_parallel_replicas_mat_view_bug.sql +++ b/tests/queries/0_stateless/03143_parallel_replicas_mat_view_bug.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS mv_table; DROP TABLE IF EXISTS null_table; -SET cluster_for_parallel_replicas='parallel_replicas', max_parallel_replicas=4, allow_experimental_parallel_reading_from_replicas=1; +SET cluster_for_parallel_replicas='parallel_replicas', max_parallel_replicas=4, enable_parallel_replicas=1; SET enable_analyzer=1; CREATE TABLE null_table (str String) ENGINE = Null; diff --git a/tests/queries/0_stateless/03164_early_constant_folding_analyzer.sql b/tests/queries/0_stateless/03164_early_constant_folding_analyzer.sql index b1018d000829..7fca1c556f86 100644 --- a/tests/queries/0_stateless/03164_early_constant_folding_analyzer.sql +++ b/tests/queries/0_stateless/03164_early_constant_folding_analyzer.sql @@ -26,5 +26,4 @@ ENGINE = MergeTree ORDER BY (date, pull_request_number, commit_sha, check_name, insert into checks select * from generateRandom() limit 1; - -select trimLeft(explain) from (explain SELECT count(1) FROM checks WHERE test_name IS NOT NULL) where explain like '%ReadFromPreparedSource%' SETTINGS enable_analyzer = 1, allow_experimental_parallel_reading_from_replicas = 0; +select trimLeft(explain) from (explain SELECT count(1) FROM checks WHERE test_name IS NOT NULL) where explain like '%ReadFromPreparedSource%' SETTINGS enable_analyzer = 1, enable_parallel_replicas = 0; diff --git a/tests/queries/0_stateless/03164_parallel_replicas_range_filter_min_max.sql b/tests/queries/0_stateless/03164_parallel_replicas_range_filter_min_max.sql index 58143395e444..5ed492def4d2 100644 --- a/tests/queries/0_stateless/03164_parallel_replicas_range_filter_min_max.sql +++ b/tests/queries/0_stateless/03164_parallel_replicas_range_filter_min_max.sql @@ -4,86 +4,126 @@ CREATE TABLE range_filter_custom_range_test (k UInt64) ENGINE=MergeTree ORDER BY INSERT INTO range_filter_custom_range_test SELECT number + 5 from numbers(10); -SELECT count() -FROM +SELECT count() +FROM ( SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test) - SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0, - parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range', - parallel_replicas_custom_key_range_lower = 5, parallel_replicas_custom_key_range_upper = 15 + SETTINGS prefer_localhost_replica = 0, + max_parallel_replicas = 3, + distributed_group_by_no_merge = 0, + enable_parallel_replicas = 1, + parallel_replicas_custom_key = 'k', + parallel_replicas_mode = 'custom_key_range', + parallel_replicas_custom_key_range_lower = 5, + parallel_replicas_custom_key_range_upper = 15 ); -SELECT count() -FROM +SELECT count() +FROM ( SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test) - SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0, - parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range', - parallel_replicas_custom_key_range_lower = 4, parallel_replicas_custom_key_range_upper = 14 + SETTINGS prefer_localhost_replica = 0, + max_parallel_replicas = 3, + distributed_group_by_no_merge = 0, + enable_parallel_replicas = 1, + parallel_replicas_custom_key = 'k', + parallel_replicas_mode = 'custom_key_range', + parallel_replicas_custom_key_range_lower = 4, + parallel_replicas_custom_key_range_upper = 14 ); -SELECT count() -FROM +SELECT count() +FROM ( SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test) - SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0, - parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range', - parallel_replicas_custom_key_range_lower = 6, parallel_replicas_custom_key_range_upper = 17 + SETTINGS prefer_localhost_replica = 0, + max_parallel_replicas = 3, + distributed_group_by_no_merge = 0, + enable_parallel_replicas = 1, + parallel_replicas_custom_key = 'k', + parallel_replicas_mode = 'custom_key_range', + parallel_replicas_custom_key_range_lower = 6, + parallel_replicas_custom_key_range_upper = 17 ); -SELECT count() -FROM +SELECT count() +FROM ( SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test) - SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0, - parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range', - parallel_replicas_custom_key_range_lower = 0, parallel_replicas_custom_key_range_upper = 15 + SETTINGS prefer_localhost_replica = 0, + max_parallel_replicas = 3, + distributed_group_by_no_merge = 0, + enable_parallel_replicas = 1, + parallel_replicas_custom_key = 'k', + parallel_replicas_mode = 'custom_key_range', + parallel_replicas_custom_key_range_lower = 0, + parallel_replicas_custom_key_range_upper = 15 ); -SELECT count() -FROM +SELECT count() +FROM ( SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test) - SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0, - parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range', - parallel_replicas_custom_key_range_lower = 15, parallel_replicas_custom_key_range_upper = 25 + SETTINGS prefer_localhost_replica = 0, + max_parallel_replicas = 3, + distributed_group_by_no_merge = 0, + enable_parallel_replicas = 1, + parallel_replicas_custom_key = 'k', + parallel_replicas_mode = 'custom_key_range', + parallel_replicas_custom_key_range_lower = 15, + parallel_replicas_custom_key_range_upper = 25 ); -SELECT count() -FROM +SELECT count() +FROM ( SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test) - SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0, - parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range', - parallel_replicas_custom_key_range_lower = 0, parallel_replicas_custom_key_range_upper = 5 + SETTINGS prefer_localhost_replica = 0, + max_parallel_replicas = 3, + distributed_group_by_no_merge = 0, + enable_parallel_replicas = 1, + parallel_replicas_custom_key = 'k', + parallel_replicas_mode = 'custom_key_range', + parallel_replicas_custom_key_range_lower = 0, + parallel_replicas_custom_key_range_upper = 5 ); -SELECT count() -FROM +SELECT count() +FROM ( SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test) - SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0, - parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range', - parallel_replicas_custom_key_range_lower = 500, parallel_replicas_custom_key_range_upper = 10000 + SETTINGS prefer_localhost_replica = 0, + max_parallel_replicas = 3, + distributed_group_by_no_merge = 0, + enable_parallel_replicas = 1, + parallel_replicas_custom_key = 'k', + parallel_replicas_mode = 'custom_key_range', + parallel_replicas_custom_key_range_lower = 500, + parallel_replicas_custom_key_range_upper = 10000 ); -SELECT count() -FROM +SELECT count() +FROM ( SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test) - SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0, - parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range', - parallel_replicas_custom_key_range_lower = 10, parallel_replicas_custom_key_range_upper = 13 + SETTINGS prefer_localhost_replica = 0, + max_parallel_replicas = 3, + distributed_group_by_no_merge = 0, + enable_parallel_replicas = 1, + parallel_replicas_custom_key = 'k', + parallel_replicas_mode = 'custom_key_range', + parallel_replicas_custom_key_range_lower = 10, + parallel_replicas_custom_key_range_upper = 13 ); DROP TABLE range_filter_custom_range_test; @@ -94,14 +134,19 @@ CREATE TABLE range_filter_custom_range_test_2 (k UInt64) ENGINE=MergeTree ORDER INSERT INTO range_filter_custom_range_test_2 SELECT number from numbers(13); -SELECT count() -FROM +SELECT count() +FROM ( SELECT * FROM cluster(parallel_replicas, currentDatabase(), range_filter_custom_range_test_2) - SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 12, distributed_group_by_no_merge = 0, - parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range', - parallel_replicas_custom_key_range_lower = 0, parallel_replicas_custom_key_range_upper = 13 + SETTINGS prefer_localhost_replica = 0, + max_parallel_replicas = 12, + distributed_group_by_no_merge = 0, + enable_parallel_replicas = 1, + parallel_replicas_custom_key = 'k', + parallel_replicas_mode = 'custom_key_range', + parallel_replicas_custom_key_range_lower = 0, + parallel_replicas_custom_key_range_upper = 13 ); DROP TABLE range_filter_custom_range_test_2; @@ -112,14 +157,19 @@ CREATE TABLE range_filter_custom_range_test_3 (k UInt64) ENGINE=MergeTree ORDER INSERT INTO range_filter_custom_range_test_3 SELECT number from numbers(4); -SELECT count() -FROM +SELECT count() +FROM ( SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test_3) - SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 12, distributed_group_by_no_merge = 0, - parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range', - parallel_replicas_custom_key_range_lower = 0, parallel_replicas_custom_key_range_upper = 4 + SETTINGS prefer_localhost_replica = 0, + max_parallel_replicas = 12, + distributed_group_by_no_merge = 0, + enable_parallel_replicas = 1, + parallel_replicas_custom_key = 'k', + parallel_replicas_mode = 'custom_key_range', + parallel_replicas_custom_key_range_lower = 0, + parallel_replicas_custom_key_range_upper = 4 ); -DROP TABLE range_filter_custom_range_test_3; \ No newline at end of file +DROP TABLE range_filter_custom_range_test_3; diff --git a/tests/queries/0_stateless/03168_loop_engine_with_parallel_replicas.sql b/tests/queries/0_stateless/03168_loop_engine_with_parallel_replicas.sql index da4626ad8970..d86334a236b0 100644 --- a/tests/queries/0_stateless/03168_loop_engine_with_parallel_replicas.sql +++ b/tests/queries/0_stateless/03168_loop_engine_with_parallel_replicas.sql @@ -6,6 +6,7 @@ CREATE TABLE 03147_db.t (n Int8) ENGINE=MergeTree ORDER BY n; INSERT INTO 03147_db.t SELECT * FROM numbers(10); USE 03147_db; +-- We use the old setting here just to make sure we preserve it as an alias. SET allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 100; SELECT * FROM loop(03147_db.t) LIMIT 15 FORMAT Null; diff --git a/tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh b/tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh index af7025697949..289a49c72f49 100755 --- a/tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh +++ b/tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh @@ -16,8 +16,8 @@ $CLICKHOUSE_CLIENT -q " INSERT INTO data2 VALUES ('a1451105-722e-4fe7-bfaa-65ad2ae249c2', '2000-01-02', 'CREATED'); " -$CLICKHOUSE_CLIENT -nq " -SET enable_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 10, allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1; +$CLICKHOUSE_CLIENT -q " +SET enable_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 10, enable_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1; SELECT id, diff --git a/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.reference b/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql b/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql new file mode 100644 index 000000000000..240203b2a7c6 --- /dev/null +++ b/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql @@ -0,0 +1,34 @@ +-- Tags: no-s3-storage +-- no-s3-storage: the test checks that there is no crash when we set parallel_replicas_single_task_marks_count_multiplier to 0 and we have some custom heuristic for +-- finding its optimal value in case if the table is stored on remote disk. + +DROP TABLE IF EXISTS 03215_parallel_replicas; + +CREATE TABLE 03215_parallel_replicas +( + `k` Int16, + `v` Nullable(UInt8) +) +ENGINE = MergeTree +ORDER BY k +SETTINGS index_granularity = 10; + +INSERT INTO 03215_parallel_replicas SELECT + number, + number +FROM numbers(1000); + +INSERT INTO 03215_parallel_replicas SELECT + number, + number +FROM numbers(1000, 1000); + +INSERT INTO 03215_parallel_replicas SELECT + number, + number +FROM numbers(2000, 1000); + +SET parallel_distributed_insert_select = 2, prefer_localhost_replica = false, enable_parallel_replicas = 1, max_parallel_replicas = 65535, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_single_task_marks_count_multiplier = -0., parallel_replicas_for_non_replicated_merge_tree = true; +SELECT max(k) IGNORE NULLS FROM 03215_parallel_replicas WITH TOTALS SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 65535, prefer_localhost_replica = 0, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_single_task_marks_count_multiplier = -0; -- { serverError 36 } + +DROP TABLE IF EXISTS 03215_parallel_replicas; diff --git a/tests/queries/0_stateless/03222_parallel_replicas_final_in_subquery.reference b/tests/queries/0_stateless/03222_parallel_replicas_final_in_subquery.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/03222_parallel_replicas_final_in_subquery.sql b/tests/queries/0_stateless/03222_parallel_replicas_final_in_subquery.sql new file mode 100644 index 000000000000..6c2e05c12a90 --- /dev/null +++ b/tests/queries/0_stateless/03222_parallel_replicas_final_in_subquery.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS test_00808; + +CREATE TABLE test_00808 +( + `date` Date, + `id` Int8, + `name` String, + `value` Int64, + `sign` Int8 +) +ENGINE = CollapsingMergeTree(sign) +ORDER BY (id, date); + +INSERT INTO test_00808 VALUES('2000-01-01', 1, 'test string 1', 1, 1); +INSERT INTO test_00808 VALUES('2000-01-01', 2, 'test string 2', 2, 1); + +SET allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; + +SELECT * FROM (SELECT * FROM test_00808 FINAL) WHERE id = 1; -- { serverError SUPPORT_IS_DISABLED } + +DROP TABLE test_00808; diff --git a/tests/queries/0_stateless/03222_pr_asan_index_granularity.reference b/tests/queries/0_stateless/03222_pr_asan_index_granularity.reference new file mode 100644 index 000000000000..662880055bdb --- /dev/null +++ b/tests/queries/0_stateless/03222_pr_asan_index_granularity.reference @@ -0,0 +1,100 @@ +0 18 9899 +0 18 9898 +0 18 9897 +0 18 9896 +0 18 9895 +0 18 9894 +0 18 9893 +0 18 9892 +0 18 9891 +0 18 9890 +0 18 9889 +0 18 9888 +0 18 9887 +0 18 9886 +0 18 9885 +0 18 9884 +0 18 9883 +0 18 9882 +0 18 9881 +0 18 9880 +0 18 9879 +0 18 9878 +0 18 9877 +0 18 9876 +0 18 9875 +0 18 9874 +0 18 9873 +0 18 9872 +0 18 9871 +0 18 9870 +0 18 9869 +0 18 9868 +0 18 9867 +0 18 9866 +0 18 9865 +0 18 9864 +0 18 9863 +0 18 9862 +0 18 9861 +0 18 9860 +0 18 9859 +0 18 9858 +0 18 9857 +0 18 9856 +0 18 9855 +0 18 9854 +0 18 9853 +0 18 9852 +0 18 9851 +0 18 9850 +0 18 9849 +0 18 9848 +0 18 9847 +0 18 9846 +0 18 9845 +0 18 9844 +0 18 9843 +0 18 9842 +0 18 9841 +0 18 9840 +0 18 9839 +0 18 9838 +0 18 9837 +0 18 9836 +0 18 9835 +0 18 9834 +0 18 9833 +0 18 9832 +0 18 9831 +0 18 9830 +0 18 9829 +0 18 9828 +0 18 9827 +0 18 9826 +0 18 9825 +0 18 9824 +0 18 9823 +0 18 9822 +0 18 9821 +0 18 9820 +0 18 9819 +0 18 9818 +0 18 9817 +0 18 9816 +0 18 9815 +0 18 9814 +0 18 9813 +0 18 9812 +0 18 9811 +0 18 9810 +0 18 9809 +0 18 9808 +0 18 9807 +0 18 9806 +0 18 9805 +0 18 9804 +0 18 9803 +0 18 9802 +0 18 9801 +0 18 9800 diff --git a/tests/queries/0_stateless/03222_pr_asan_index_granularity.sql b/tests/queries/0_stateless/03222_pr_asan_index_granularity.sql new file mode 100644 index 000000000000..b7f37dd28568 --- /dev/null +++ b/tests/queries/0_stateless/03222_pr_asan_index_granularity.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS test; + +CREATE TABLE test (k UInt64, v String) +ENGINE = MergeTree +ORDER BY k +SETTINGS index_granularity=1; + +INSERT INTO test SELECT number, toString(number) FROM numbers(10_000); + +SET allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; + +SELECT 0, materialize(18), k FROM test PREWHERE toNullable(toNullable(11)) WHERE toNullable(11) ORDER BY k DESC NULLS LAST LIMIT 100, 100 SETTINGS optimize_read_in_order = 1, merge_tree_min_rows_for_concurrent_read = 9223372036854775806, max_threads = 1; + +-- DROP TABLE test; diff --git a/tests/queries/0_stateless/03228_pr_subquery_view_order_by.reference b/tests/queries/0_stateless/03228_pr_subquery_view_order_by.reference new file mode 100644 index 000000000000..aad720e27cfa --- /dev/null +++ b/tests/queries/0_stateless/03228_pr_subquery_view_order_by.reference @@ -0,0 +1,20 @@ +300 +299 +298 +297 +296 +295 +294 +293 +292 +291 +290 +289 +288 +287 +286 +285 +284 +283 +282 +281 diff --git a/tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql b/tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql new file mode 100644 index 000000000000..804a97f737f2 --- /dev/null +++ b/tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS view1; +DROP TABLE IF EXISTS table1; +CREATE TABLE table1 (number UInt64) ENGINE=MergeTree ORDER BY number SETTINGS index_granularity=1; +INSERT INTO table1 SELECT number FROM numbers(1, 300); +CREATE VIEW view1 AS SELECT number FROM table1; + +SELECT * +FROM +( + SELECT * + FROM view1 +) +ORDER BY number DESC +LIMIT 20 +SETTINGS cluster_for_parallel_replicas = 'parallel_replicas', allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_local_plan = 1; + +DROP TABLE view1; +DROP TABLE table1; diff --git a/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.reference b/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.reference new file mode 100644 index 000000000000..3ae545099d6a --- /dev/null +++ b/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.reference @@ -0,0 +1,10 @@ +abc 2 +abc 2 +abc 3 +abc 4 +\N 5 +abc 2 +abc 2 +abc 3 +abc 4 +\N 5 diff --git a/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.sh b/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.sh new file mode 100755 index 000000000000..217491534e99 --- /dev/null +++ b/tests/queries/0_stateless/03251_parquet_page_v2_native_reader.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER_FILES_PATH=$($CLICKHOUSE_CLIENT_BINARY --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +WORKING_DIR="${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +mkdir -p "${WORKING_DIR}" + +DATA_FILE="${CUR_DIR}/data_parquet/datapage_v2.snappy.parquet" + +DATA_FILE_USER_PATH="${WORKING_DIR}/datapage_v2.snappy.parquet" + +cp ${DATA_FILE} ${DATA_FILE_USER_PATH} + +# Not reading all columns because some data types and encodings are not supported by native reader yet +# TODO read all columns once implemented +${CLICKHOUSE_CLIENT} --query="select a, c from file('${DATA_FILE_USER_PATH}', Parquet) order by c SETTINGS input_format_parquet_use_native_reader=false;" +${CLICKHOUSE_CLIENT} --query="select a, c from file('${DATA_FILE_USER_PATH}', Parquet) order by c SETTINGS input_format_parquet_use_native_reader=true;" diff --git a/tests/queries/0_stateless/03254_parquet_bool_native_reader.reference b/tests/queries/0_stateless/03254_parquet_bool_native_reader.reference new file mode 100644 index 000000000000..0c7e55ad2344 --- /dev/null +++ b/tests/queries/0_stateless/03254_parquet_bool_native_reader.reference @@ -0,0 +1,20 @@ +0 false +1 \N +2 false +3 \N +4 false +5 \N +6 false +7 \N +8 true +9 \N +0 false +1 \N +2 false +3 \N +4 false +5 \N +6 false +7 \N +8 true +9 \N diff --git a/tests/queries/0_stateless/03254_parquet_bool_native_reader.sh b/tests/queries/0_stateless/03254_parquet_bool_native_reader.sh new file mode 100755 index 000000000000..c28523b3c549 --- /dev/null +++ b/tests/queries/0_stateless/03254_parquet_bool_native_reader.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: no-ubsan, no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER_FILES_PATH=$($CLICKHOUSE_CLIENT_BINARY --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +WORKING_DIR="${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +mkdir -p "${WORKING_DIR}" + +DATA_FILE="${CUR_DIR}/data_parquet/nullbool.parquet" + +DATA_FILE_USER_PATH="${WORKING_DIR}/nullbool.parquet" + +cp ${DATA_FILE} ${DATA_FILE_USER_PATH} + +${CLICKHOUSE_CLIENT} --query="select id, bool from file('${DATA_FILE_USER_PATH}', Parquet) order by id SETTINGS input_format_parquet_use_native_reader=false;" +${CLICKHOUSE_CLIENT} --query="select id, bool from file('${DATA_FILE_USER_PATH}', Parquet) order by id SETTINGS input_format_parquet_use_native_reader=true;" diff --git a/tests/queries/0_stateless/03276_parquet_output_compression_level.reference b/tests/queries/0_stateless/03276_parquet_output_compression_level.reference new file mode 100644 index 000000000000..6ed281c757a9 --- /dev/null +++ b/tests/queries/0_stateless/03276_parquet_output_compression_level.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/03276_parquet_output_compression_level.sql b/tests/queries/0_stateless/03276_parquet_output_compression_level.sql new file mode 100644 index 000000000000..27e096a632a2 --- /dev/null +++ b/tests/queries/0_stateless/03276_parquet_output_compression_level.sql @@ -0,0 +1,19 @@ +-- Tags: no-parallel, no-fasttest + +insert into function file(03276_parquet_custom_encoder_compression_level_1.parquet) SETTINGS output_format_parquet_compression_method = 'zstd', output_format_compression_level=1, output_format_parquet_use_custom_encoder=1, engine_file_truncate_on_insert=1 SELECT number AS id, toString(number) AS name, now() + number AS timestamp FROM numbers(100000); +insert into function file(03276_parquet_custom_encoder_compression_level_22.parquet) SETTINGS output_format_parquet_compression_method = 'zstd', output_format_compression_level=22, output_format_parquet_use_custom_encoder=1, engine_file_truncate_on_insert=1 SELECT number AS id, toString(number) AS name, now() + number AS timestamp FROM numbers(100000); + +WITH + (SELECT total_compressed_size FROM file(03276_parquet_custom_encoder_compression_level_1.parquet, ParquetMetadata)) AS size_level_1, + (SELECT total_compressed_size FROM file(03276_parquet_custom_encoder_compression_level_22.parquet, ParquetMetadata)) AS size_level_22 +SELECT + size_level_22 < size_level_1 AS compression_higher_level_produces_smaller_file; + +insert into function file(03276_parquet_arrow_encoder_compression_level_1.parquet) SETTINGS output_format_parquet_compression_method = 'zstd', output_format_compression_level=1, output_format_parquet_use_custom_encoder=0, engine_file_truncate_on_insert=1 SELECT number AS id, toString(number) AS name, now() + number AS timestamp FROM numbers(100000); +insert into function file(03276_parquet_arrow_encoder_compression_level_22.parquet) SETTINGS output_format_parquet_compression_method = 'zstd', output_format_compression_level=22, output_format_parquet_use_custom_encoder=0, engine_file_truncate_on_insert=1 SELECT number AS id, toString(number) AS name, now() + number AS timestamp FROM numbers(100000); + +WITH + (SELECT total_compressed_size FROM file(03276_parquet_arrow_encoder_compression_level_1.parquet, ParquetMetadata)) AS size_level_1, + (SELECT total_compressed_size FROM file(03276_parquet_arrow_encoder_compression_level_22.parquet, ParquetMetadata)) AS size_level_22 +SELECT + size_level_22 < size_level_1 AS compression_higher_level_produces_smaller_file; diff --git a/tests/queries/0_stateless/data_parquet/ipv6_bloom_filter.gz.parquet b/tests/queries/0_stateless/data_parquet/ipv6_bloom_filter.gz.parquet new file mode 100644 index 000000000000..335fddc1f0ad Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/ipv6_bloom_filter.gz.parquet differ diff --git a/tests/queries/0_stateless/data_parquet/multi_column_bf.gz.parquet b/tests/queries/0_stateless/data_parquet/multi_column_bf.gz.parquet new file mode 100644 index 000000000000..bfa0a39062a0 Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/multi_column_bf.gz.parquet differ diff --git a/tests/queries/0_stateless/data_parquet/nullbool.parquet b/tests/queries/0_stateless/data_parquet/nullbool.parquet new file mode 100644 index 000000000000..d9b365bbe75b Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/nullbool.parquet differ diff --git a/tests/queries/1_stateful/00177_memory_bound_merging.reference b/tests/queries/1_stateful/00177_memory_bound_merging.reference index 5689152d60a1..4c7505d11232 100644 --- a/tests/queries/1_stateful/00177_memory_bound_merging.reference +++ b/tests/queries/1_stateful/00177_memory_bound_merging.reference @@ -10,6 +10,8 @@ http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 2014-03-18 http:/ http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 2014-03-19 http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 2014-03-20 http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 1 -SortingAggregatedTransform MergingAggregatedBucketTransform -GroupingAggregatedTransform +FinishAggregatingInOrderTransform +FinalizeAggregatedTransform +AggregatingInOrderTransform +MergeTreeSelect diff --git a/tests/queries/1_stateful/00177_memory_bound_merging.sh b/tests/queries/1_stateful/00177_memory_bound_merging.sh index d5cd1a05cd8d..d256f814f15e 100755 --- a/tests/queries/1_stateful/00177_memory_bound_merging.sh +++ b/tests/queries/1_stateful/00177_memory_bound_merging.sh @@ -31,7 +31,7 @@ test1() { GROUP BY CounterID, URL, EventDate ORDER BY URL, EventDate LIMIT 5 OFFSET 10 - SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3" + SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3" check_replicas_read_in_order $query_id } @@ -48,14 +48,14 @@ test2() { GROUP BY URL, EventDate ORDER BY URL, EventDate LIMIT 5 OFFSET 10 - SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3, query_plan_aggregation_in_order = 1" + SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3, query_plan_aggregation_in_order = 1" check_replicas_read_in_order $query_id } test3() { $CLICKHOUSE_CLIENT -nq " SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; - SET max_threads = 16, prefer_localhost_replica = 1, read_in_order_two_level_merge_threshold = 1000, query_plan_aggregation_in_order = 1, distributed_aggregation_memory_efficient = 1; + SET max_threads = 16, read_in_order_two_level_merge_threshold = 1000, query_plan_aggregation_in_order = 1, distributed_aggregation_memory_efficient = 1; SELECT replaceRegexpOne(explain, '^ *(\w+).*', '\\1') FROM ( @@ -64,7 +64,7 @@ test3() { FROM test.hits WHERE CounterID = 1704509 AND UserID = 4322253409885123546 GROUP BY URL, EventDate - SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3 + SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3, parallel_replicas_local_plan=1 ) WHERE explain LIKE '%Aggr%Transform%' OR explain LIKE '%InOrder%'" } diff --git a/utils/clickhouse-docker b/utils/clickhouse-docker index cfe515f1de54..62d980f2f407 100755 --- a/utils/clickhouse-docker +++ b/utils/clickhouse-docker @@ -24,13 +24,13 @@ param="$1" if [ "${param}" = "list" ] then # https://stackoverflow.com/a/39454426/1555175 - wget -nv https://registry.hub.docker.com/v1/repositories/clickhouse/clickhouse-server/tags -O - | sed -e 's/[][]//g' -e 's/"//g' -e 's/ //g' | tr '}' '\n' | awk -F: '{print $3}' + wget -nv https://registry.hub.docker.com/v1/repositories/altinity/clickhouse-server/tags -O - | sed -e 's/[][]//g' -e 's/"//g' -e 's/ //g' | tr '}' '\n' | awk -F: '{print $3}' else - docker pull clickhouse/clickhouse-server:${param} + docker pull altinity/clickhouse-server:${param} tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX) # older version require /nonexistent folder to exist to run clickhouse client :D chmod 777 ${tmp_dir} set -e - containerid=`docker run -v${tmp_dir}:/nonexistent -d clickhouse/clickhouse-server:${param}` + containerid=`docker run -v${tmp_dir}:/nonexistent -d altinity/clickhouse-server:${param}` set +e while : do diff --git a/utils/test_history/requirements.txt b/utils/test_history/requirements.txt index 07970e9fab5f..220519696891 100644 --- a/utils/test_history/requirements.txt +++ b/utils/test_history/requirements.txt @@ -1,3 +1,3 @@ -pygithub==1.43.5 -tabulate==0.8.6 -termcolor==1.1.0 +pygithub~=1.43.5 +tabulate~=0.8.6 +termcolor~=1.1.0