Merge pull request #582 from Altinity/customizations/24.8.11

24.8.11 Altinity Stable Pre-release

Author: Enmk

.github/actionlint.yml

@@ -4,6 +4,10 @@ self-hosted-runner:
     - func-tester
     - func-tester-aarch64
     - fuzzer-unit-tester
+    - altinity-on-demand
+    - altinity-type-cpx51
+    - altinity-in-ash
+    - altinity-image-x86-system-ubuntu-22.04
     - style-checker
     - style-checker-aarch64
     - release-maker

.github/actions/common_setup/action.yml

@@ -28,6 +28,16 @@ runs:
       run: |
           # to remove every leftovers
           sudo rm -fr "$TEMP_PATH" && mkdir -p "$TEMP_PATH"
+    - name: Setup zram
+      shell: bash
+      run: |
+          sudo modprobe zram
+          MemTotal=$(grep -Po "(?<=MemTotal:)\s+\d+" /proc/meminfo) # KiB
+          Percent=200
+          ZRAM_SIZE=$(($MemTotal / 1024 / 1024 * $Percent / 100)) # Convert to GiB
+          .github/retry.sh 30 2 sudo zramctl --size ${ZRAM_SIZE}GiB --algorithm zstd /dev/zram0
+          sudo mkswap /dev/zram0 && sudo swapon -p 100 /dev/zram0
+          sudo sysctl vm.swappiness=200
     - name: Tune vm.mmap_rnd_bits for sanitizers
       shell: bash
       run: |

.github/actions/docker_setup/action.yml

@@ -0,0 +1,31 @@
+name: Docker setup
+description: Setup docker
+inputs:
+  test_name:
+    description: name of the test, used in determining ipv6 configs.
+    default: None
+    type: string
+runs:
+  using: "composite"
+  steps:
+    - name: Docker IPv6 configuration
+      shell: bash
+      env:
+        ipv6_subnet: ${{ contains(inputs.test_name, 'Integration') && '2001:db8:1::/64' || '2001:3984:3989::/64' }}
+      run: |
+        # make sure docker uses proper IPv6 config
+        sudo touch /etc/docker/daemon.json
+        sudo chown ubuntu:ubuntu /etc/docker/daemon.json
+        sudo cat <<EOT > /etc/docker/daemon.json
+        {
+          "ipv6": true,
+          "fixed-cidr-v6": "${{ env.ipv6_subnet }}"
+        }
+        EOT
+        sudo chown root:root /etc/docker/daemon.json
+        sudo systemctl restart docker
+        sudo systemctl status docker
+    - name: Docker info
+      shell: bash
+      run: |
+        docker info

.github/retry.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Execute command until exitcode is 0 or
+# maximum number of retries is reached
+# Example:
+#    ./retry <retries> <delay> <command>
+retries=$1
+delay=$2
+command="${@:3}"
+exitcode=0
+try=0
+until [ "$try" -ge $retries ]
+do
+  echo "$command"
+  eval "$command"
+  exitcode=$?
+  if [ $exitcode -eq 0 ]; then
+    break
+  fi
+  try=$((try+1))
+  sleep $2
+done
+exit $exitcode

.github/workflows/backport_branches.yml

@@ -22,7 +22,7 @@ jobs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           clear-repository: true # to ensure correct digests
           fetch-depth: 0 # to get version
@@ -164,7 +164,7 @@ jobs:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
       - name: Download reports
         run: |
           python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds
@@ -194,7 +194,7 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Install packages (aarch64)
-      runner_type: style-checker-aarch64
+      runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
         python3 install_check.py "$CHECK_NAME"
@@ -229,7 +229,7 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Stress test (tsan)
-      runner_type: func-tester
+      runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
 #############################################################################################
 ############################# INTEGRATION TESTS #############################################
@@ -240,15 +240,15 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Integration tests (asan, old analyzer)
-      runner_type: func-tester
+      runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04 
       data: ${{ needs.RunConfig.outputs.data }}
   IntegrationTestsTsan:
     needs: [RunConfig, BuilderDebTsan]
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Integration tests (tsan)
-      runner_type: func-tester
+      runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-system-ubuntu-22.04
       data: ${{ needs.RunConfig.outputs.data }}
   FinishCheck:
     if: ${{ !cancelled() }}
@@ -265,7 +265,7 @@ jobs:
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           clear-repository: true
       - name: Finish label

.github/workflows/cherry_pick.yml

@@ -28,7 +28,7 @@ jobs:
           REPO_TEAM=core
           EOF
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           clear-repository: true
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}

.github/workflows/create_release.yml

@@ -34,7 +34,7 @@ jobs:
     runs-on: [self-hosted, release-maker]
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
           fetch-depth: 0

.github/workflows/docker_test_images.yml

@@ -10,16 +10,31 @@ name: Build docker images
         description: set latest tag for resulting multiarch manifest
         required: false
         type: boolean
-        default: false
+        default: false    
+    secrets:
+      secret_envs:
+        description: if given, it's passed to the environments
+        required: false
+      AWS_SECRET_ACCESS_KEY:
+        description: the access key to the aws param store.
+        required: true
+      AWS_ACCESS_KEY_ID:
+        description: the access key id to the aws param store.
+        required: true
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
 jobs:
   DockerBuildAarch64:
-    runs-on: [self-hosted, style-checker-aarch64]
+    runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none]
     if: |
       !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_aarch64) != '[]'
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
         with:
           ref: ${{ fromJson(inputs.data).git_ref }}
       - name: Build images
@@ -29,12 +44,12 @@ jobs:
             --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \
             --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_aarch64) }}'
   DockerBuildAmd64:
-    runs-on: [self-hosted, style-checker]
+    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
     if: |
       !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_amd64) != '[]'
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
         with:
           ref: ${{ fromJson(inputs.data).git_ref }}
       - name: Build images
@@ -45,12 +60,12 @@ jobs:
             --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_amd64) }}'
   DockerMultiArchManifest:
     needs: [DockerBuildAmd64, DockerBuildAarch64]
-    runs-on: [self-hosted, style-checker]
+    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
     if: |
       !failure() && !cancelled() && (toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]' || inputs.set_latest)
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6
         with:
           ref: ${{ fromJson(inputs.data).git_ref }}
       - name: Build images

.github/workflows/jepsen.yml

@@ -16,7 +16,7 @@ jobs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           clear-repository: true # to ensure correct digests
           fetch-depth: 0 # to get version
@@ -43,7 +43,7 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse Keeper Jepsen
-      runner_type: style-checker-aarch64
+      runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
         python3 jepsen_check.py keeper
@@ -53,7 +53,7 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse Server Jepsen
-      runner_type: style-checker-aarch64
+      runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
         python3 jepsen_check.py server
@@ -63,7 +63,7 @@ jobs:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
       - name: Check Workflow results
         if: ${{ !cancelled() }}
         run: |

.github/workflows/master.yml

@@ -16,7 +16,7 @@ jobs:
       data: ${{ steps.runconfig.outputs.CI_DATA }}
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
         with:
           clear-repository: true # to ensure correct digests
           fetch-depth: 0 # to get version
@@ -118,7 +118,7 @@ jobs:
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Builds
-      runner_type: style-checker-aarch64
+      runner_type: altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-app-docker-ce
       data: ${{ needs.RunConfig.outputs.data }}
 
   FinishCheck:
@@ -127,7 +127,7 @@ jobs:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
-        uses: ClickHouse/checkout@v1
+        uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f
       - name: Finish label
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"