diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 899697eb4898..b7d322e54949 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -88,9 +88,9 @@ name: Regression test workflow env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }} @@ -174,7 +174,7 @@ jobs: run: python3 -u ${{ env.SUITE }}/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Set Commit Status if: always() @@ -231,7 +231,7 @@ jobs: -u alter/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} --only "/alter/${{ matrix.ONLY }} partition/*" - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Set Commit Status if: always() @@ -296,7 +296,7 @@ jobs: --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Set Commit Status if: always() @@ -350,7 +350,7 @@ jobs: -u ${{ env.SUITE }}/regression.py --ssl --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Set Commit Status if: always() @@ -406,7 +406,7 @@ jobs: run: python3 -u ${{ env.SUITE }}/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Set Commit Status if: always() @@ -458,7 +458,7 @@ jobs: run: python3 -u ${{ env.SUITE }}/regression.py --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Set Commit Status if: always() @@ -521,7 +521,7 @@ jobs: --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Set Commit Status if: always() @@ -586,7 +586,7 @@ jobs: --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Set Commit Status if: always() @@ -650,7 +650,7 @@ jobs: --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} --with-${{ matrix.STORAGE }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" + --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" ${{ env.args }} - name: Set Commit Status if: always() diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 08fee7dddb76..71061b4f7dfd 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -9,6 +9,7 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }} on: # yamllint disable-line rule:truthy pull_request: @@ -32,7 +33,7 @@ on: # yamllint disable-line rule:truthy jobs: DockerHubPushAarch64: - runs-on: [self-hosted, altinity-on-demand, altinity-type-cax41, altinity-in-hel1, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none] + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -54,7 +55,7 @@ jobs: path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: - runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -77,7 +78,7 @@ jobs: DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64] - runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] steps: - name: Check out repository code uses: Altinity/checkout@19599efdf36c4f3f30eb55d5bb388896faea69f6 @@ -121,7 +122,7 @@ jobs: secrets: inherit with: test_name: Compatibility check X86 - runner_type: altinity-on-demand, altinity-type-cpx51, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: self-hosted, altinity-on-demand, altinity-func-tester timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" @@ -133,7 +134,7 @@ jobs: secrets: inherit with: test_name: Compatibility check Aarch64 - runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none + runner_type: self-hosted, altinity-on-demand, altinity-func-tester-aarch64 timeout_minutes: 180 run_command: | cd "$REPO_COPY/tests/ci" @@ -150,7 +151,7 @@ jobs: build_name: package_release checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-builder additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -162,7 +163,7 @@ jobs: build_name: package_aarch64 checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-builder additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -174,7 +175,7 @@ jobs: build_name: package_asan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-builder additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -186,7 +187,7 @@ jobs: build_name: package_ubsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-builder additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -198,7 +199,7 @@ jobs: build_name: package_tsan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-builder additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -210,7 +211,7 @@ jobs: build_name: package_msan checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-builder additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -222,7 +223,7 @@ jobs: build_name: package_debug checkout_depth: 0 timeout_minutes: 180 - runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-hil, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none + runner_type: altinity-on-demand, altinity-builder additional_envs: | CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable @@ -233,7 +234,7 @@ jobs: needs: - BuilderDebRelease - BuilderDebAarch64 - runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx41, altinity-in-ash, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none] + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] timeout-minutes: 180 steps: - name: Check out repository code @@ -269,7 +270,7 @@ jobs: secrets: inherit with: test_name: ClickHouse build check - runner_type: altinity-on-demand, altinity-type-cax11, altinity-in-hel1, altinity-image-arm-snapshot-22.04-arm, altinity-startup-snapshot, altinity-setup-none + runner_type: self-hosted, altinity-on-demand, altinity-style-checker-aarch64 timeout_minutes: 180 additional_envs: | NEEDS_DATA< None: params = { "query": query, @@ -53,7 +55,7 @@ def insert_file( with open(file, "rb") as data_fd: ClickHouseHelper._insert_post( - url, params=params, data=data_fd, headers=auth + url, params=params, data=data_fd, headers=auth, **kwargs ) @staticmethod diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index adab6be672c8..2bb75c739723 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -13,8 +13,8 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status -from env_helper import REPO_COPY, RUNNER_TEMP, GITHUB_RUN_URL -from get_robot_token import get_best_robot_token, get_parameter_from_ssm +from env_helper import DOCKER_PASSWORD, REPO_COPY, RUNNER_TEMP, GITHUB_RUN_URL +from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult from s3_helper import S3Helper @@ -426,7 +426,7 @@ def main(): logging.info('Doing docker login') subprocess.check_output( # pylint: disable=unexpected-keyword-arg "docker login --username 'altinityinfra' --password-stdin", - input=get_parameter_from_ssm("dockerhub-password"), + input=DOCKER_PASSWORD, encoding="utf-8", shell=True, ) diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index 2692a0da2cb9..f7fee72e7c0d 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -18,8 +18,8 @@ ) from commit_status_helper import format_description, get_commit, post_commit_status from docker_images_helper import IMAGES_FILE_PATH, get_image_names -from env_helper import RUNNER_TEMP, REPO_COPY -from get_robot_token import get_best_robot_token, get_parameter_from_ssm +from env_helper import DOCKER_PASSWORD, RUNNER_TEMP, REPO_COPY +from get_robot_token import get_best_robot_token from git_helper import Runner from pr_info import PRInfo from report import TestResults, TestResult @@ -251,7 +251,7 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg "docker login --username 'altinityinfra' --password-stdin", - input=get_parameter_from_ssm("dockerhub-password"), + input=DOCKER_PASSWORD, encoding="utf-8", shell=True, ) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 0cec4a8968b8..6fd4ce61184f 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -17,8 +17,8 @@ from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse from commit_status_helper import format_description, get_commit, post_commit_status from docker_images_check import DockerImage -from env_helper import CI, GITHUB_RUN_URL, RUNNER_TEMP, S3_BUILDS_BUCKET, S3_DOWNLOAD -from get_robot_token import get_best_robot_token, get_parameter_from_ssm +from env_helper import DOCKER_PASSWORD, CI, GITHUB_RUN_URL, RUNNER_TEMP, S3_BUILDS_BUCKET, S3_DOWNLOAD +from get_robot_token import get_best_robot_token from git_helper import Git from pr_info import PRInfo from report import TestResults, TestResult @@ -342,7 +342,7 @@ def main(): if args.push: subprocess.check_output( # pylint: disable=unexpected-keyword-arg "docker login --username 'altinityinfra' --password-stdin", - input=get_parameter_from_ssm("dockerhub-password"), + input=DOCKER_PASSWORD, encoding="utf-8", shell=True, ) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 16dd5495d023..85a4b8ca7863 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -38,6 +38,11 @@ f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/" "{pr_or_release}/{commit}/{build_name}/{artifact}" ) +CLICKHOUSE_TEST_STAT_LOGIN = os.getenv("CLICKHOUSE_TEST_STAT_LOGIN") +CLICKHOUSE_TEST_STAT_PASSWORD = os.getenv("CLICKHOUSE_TEST_STAT_PASSWORD") +CLICKHOUSE_TEST_STAT_URL = os.getenv("CLICKHOUSE_TEST_STAT_URL") +DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD") +ROBOT_TOKEN = os.getenv("ROBOT_TOKEN") # These parameters are set only on demand, and only once _GITHUB_JOB_ID = "" diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index b93eea52f700..abadb30adce1 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -1,20 +1,28 @@ #!/usr/bin/env python3 import logging +import os +import random from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union import boto3 # type: ignore from github import Github from github.AuthenticatedUser import AuthenticatedUser +from github.GithubException import BadCredentialsException +from github.NamedUser import NamedUser +ROBOT_TOKEN = os.getenv("ROBOT_TOKEN") @dataclass class Token: - user: AuthenticatedUser + user: Union[AuthenticatedUser, NamedUser] value: str rest: int +SAFE_REQUESTS_LIMIT = 1000 + + def get_parameter_from_ssm( name: str, decrypt: bool = True, client: Optional[Any] = None ) -> str: @@ -50,23 +58,14 @@ def get_parameters_from_ssm( return results - -ROBOT_TOKEN = None # type: Optional[Token] - # NOTE(Arthur Passos): Original CI code uses the "_original" version of this method. Each robot token is rate limited # and the original implementation selects the "best one". To make it simpler and iterate faster, # we are using only one robot and keeping the method signature. In the future we might reconsider # having multiple robot tokens -def get_best_robot_token(token_prefix_env_name="github_robot_token"): - # Re-use already fetched token (same as in get_best_robot_token_original) - # except here we assume it is always a string (since we use only one token and don't do token rotation) - global ROBOT_TOKEN - if ROBOT_TOKEN is not None: - return ROBOT_TOKEN - ROBOT_TOKEN = get_parameter_from_ssm(token_prefix_env_name) +def get_best_robot_token(): return ROBOT_TOKEN -def get_best_robot_token_original(token_prefix_env_name: str="github_robot_token_") -> str: +def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: global ROBOT_TOKEN if ROBOT_TOKEN is not None: return ROBOT_TOKEN.value @@ -79,20 +78,34 @@ def get_best_robot_token_original(token_prefix_env_name: str="github_robot_token } assert tokens - for value in tokens.values(): + token_items = list(tokens.items()) + random.shuffle(token_items) + + best_token: Optional[Token] = None + + for name, value in token_items: gh = Github(value, per_page=100) - # Do not spend additional request to API by accessin user.login unless - # the token is chosen by the remaining requests number - user = gh.get_user() - rest, _ = gh.rate_limiting - logging.info("Get token with %s remaining requests", rest) - if ROBOT_TOKEN is None: - ROBOT_TOKEN = Token(user, value, rest) + try: + # Do not spend additional request to API by accessing user.login unless + # the token is chosen by the remaining requests number + user = gh.get_user() + rest, _ = gh.rate_limiting + except BadCredentialsException: + logging.error( + "The token %(name)s has expired, please update it\n" + "::error::Token %(name)s has expired, it must be updated", + {"name": name}, + ) continue - if ROBOT_TOKEN.rest < rest: - ROBOT_TOKEN.user, ROBOT_TOKEN.value, ROBOT_TOKEN.rest = user, value, rest - - assert ROBOT_TOKEN + logging.info("Get token with %s remaining requests", rest) + if best_token is None: + best_token = Token(user, value, rest) + elif best_token.rest < rest: + best_token = Token(user, value, rest) + if best_token.rest > SAFE_REQUESTS_LIMIT: + break + assert best_token + ROBOT_TOKEN = best_token logging.info( "User %s with %s remaining requests is used", ROBOT_TOKEN.user.login, diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index bb047b4f4efd..6886db38a8df 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -7,6 +7,7 @@ from multiprocessing.dummy import Pool from pathlib import Path from typing import List, Union +import os import boto3 # type: ignore import botocore # type: ignore @@ -21,6 +22,41 @@ ) from compress_files import compress_file_fast +sensitive_var_pattern = re.compile( + r"\b[A-Z_]*(? str: + logging.debug("Checking %s for sensitive values", file_path) + try: + file_content = file_path.read_text(encoding="utf-8") + except UnicodeDecodeError: + logging.warning("Failed to read file %s, unknown encoding", file_path) + else: + scan_file_for_sensitive_data(file_content, file_path.name) logging.debug( "Start uploading %s to bucket=%s path=%s", file_path, bucket_name, s3_path ) diff --git a/utils/tests-visualizer/index.html b/utils/tests-visualizer/index.html index b2db5dbed338..09f7ced3c444 100644 --- a/utils/tests-visualizer/index.html +++ b/utils/tests-visualizer/index.html @@ -53,7 +53,7 @@

 

Data not load

- +

Loading (~10 seconds, ~20 MB)