From c47d1b3ba837fdcc97c3e3e1782bc3a72d9dd36b Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 31 Jan 2025 12:01:10 +0200 Subject: [PATCH] Revert "Use centralized version qualifier (#16995)" This reverts commit ac56d99553476b6bf30012642dd1ab42a93dd0b6. --- .buildkite/dra_pipeline.yml | 2 +- .buildkite/scripts/dra/generatesteps.py | 66 +++++-------------------- 2 files changed, 14 insertions(+), 54 deletions(-) diff --git a/.buildkite/dra_pipeline.yml b/.buildkite/dra_pipeline.yml index fae27b3c738..10007715ccd 100644 --- a/.buildkite/dra_pipeline.yml +++ b/.buildkite/dra_pipeline.yml @@ -6,7 +6,7 @@ steps: set -euo pipefail echo "--- Building [$${WORKFLOW_TYPE}] artifacts" - python3 -m pip install pyyaml urllib3 + python3 -m pip install pyyaml echo "--- Building dynamic pipeline steps" python3 .buildkite/scripts/dra/generatesteps.py > steps.yml echo "--- Printing dynamic pipeline steps" diff --git a/.buildkite/scripts/dra/generatesteps.py b/.buildkite/scripts/dra/generatesteps.py index 8274e4bba69..d26e34009e3 100644 --- a/.buildkite/scripts/dra/generatesteps.py +++ b/.buildkite/scripts/dra/generatesteps.py @@ -1,38 +1,10 @@ import os -import urllib3 -from urllib3.util.retry import Retry -from urllib3.exceptions import HTTPError +import sys import yaml YAML_HEADER = '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json\n' -def fetch_version_qualifier_url_with_retries(branch, max_retries=3, backoff_factor=2): - """ - Return the version qualifier from a centralized URL based on branch. For any failure or response apart from 200, - assume no qualifier (return empty string). - """ - url = f"https://storage.googleapis.com/dra-qualifier/{branch}" - - http = urllib3.PoolManager() - - # Configure retries: Exponential backoff with 3 retries - retries = Retry( - total=max_retries, - backoff_factor=backoff_factor, # Wait time increases exponentially (2, 4, 8s) - status_forcelist=[500, 502, 503, 504], # Retry only on server errors - raise_on_status=False # Do not raise exception on failed status codes - ) - - try: - response = http.request("GET", url, retries=retries, timeout=5) - if response.status == 200: - return response.data.decode("utf-8").strip() - except HTTPError as e: - pass - - return "" - def to_bk_key_friendly_string(key): """ Convert and return key to an acceptable format for Buildkite's key: field @@ -43,7 +15,7 @@ def to_bk_key_friendly_string(key): return key.translate(mapping_table) -def package_x86_step(branch, workflow_type, version_qualifier): +def package_x86_step(branch, workflow_type): step = f''' - label: ":package: Build packages / {branch}-{workflow_type.upper()} DRA artifacts" key: "logstash_build_packages_dra" @@ -55,7 +27,6 @@ def package_x86_step(branch, workflow_type, version_qualifier): diskSizeGb: 200 command: | export WORKFLOW_TYPE="{workflow_type}" - export VERSION_QUALIFIER="{version_qualifier}" export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH" eval "$(rbenv init -)" .buildkite/scripts/dra/build_packages.sh @@ -63,7 +34,7 @@ def package_x86_step(branch, workflow_type, version_qualifier): return step -def package_x86_docker_step(branch, workflow_type, version_qualifier): +def package_x86_docker_step(branch, workflow_type): step = f''' - label: ":package: Build x86_64 Docker / {branch}-{workflow_type.upper()} DRA artifacts" key: "logstash_build_x86_64_docker_dra" @@ -75,7 +46,6 @@ def package_x86_docker_step(branch, workflow_type, version_qualifier): diskSizeGb: 200 command: | export WORKFLOW_TYPE="{workflow_type}" - export VERSION_QUALIFIER="{version_qualifier}" export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH" export ARCH="x86_64" eval "$(rbenv init -)" @@ -84,7 +54,7 @@ def package_x86_docker_step(branch, workflow_type, version_qualifier): return step -def package_aarch64_docker_step(branch, workflow_type, version_qualifier): +def package_aarch64_docker_step(branch, workflow_type): step = f''' - label: ":package: Build aarch64 Docker / {branch}-{workflow_type.upper()} DRA artifacts" key: "logstash_build_aarch64_docker_dra" @@ -95,7 +65,6 @@ def package_aarch64_docker_step(branch, workflow_type, version_qualifier): diskSizeGb: 200 command: | export WORKFLOW_TYPE="{workflow_type}" - export VERSION_QUALIFIER="{version_qualifier}" export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH" export ARCH="aarch64" eval "$(rbenv init -)" @@ -104,7 +73,7 @@ def package_aarch64_docker_step(branch, workflow_type, version_qualifier): return step -def publish_dra_step(branch, workflow_type, depends_on, version_qualifier): +def publish_dra_step(branch, workflow_type, depends_on): step = f''' - label: ":elastic-stack: Publish / {branch}-{workflow_type.upper()} DRA artifacts" key: "logstash_publish_dra" @@ -123,59 +92,50 @@ def publish_dra_step(branch, workflow_type, depends_on, version_qualifier): sudo chown -R :1000 build echo "+++ Running DRA publish step" export WORKFLOW_TYPE="{workflow_type}" - export VERSION_QUALIFIER="{version_qualifier}" .buildkite/scripts/dra/publish.sh ''' return step -def build_steps_to_yaml(branch, workflow_type, version_qualifier): +def build_steps_to_yaml(branch, workflow_type): steps = [] - steps.extend(yaml.safe_load(package_x86_step(branch, workflow_type, version_qualifier))) - steps.extend(yaml.safe_load(package_x86_docker_step(branch, workflow_type, version_qualifier))) - steps.extend(yaml.safe_load(package_aarch64_docker_step(branch, workflow_type, version_qualifier))) + steps.extend(yaml.safe_load(package_x86_step(branch, workflow_type))) + steps.extend(yaml.safe_load(package_x86_docker_step(branch, workflow_type))) + steps.extend(yaml.safe_load(package_aarch64_docker_step(branch, workflow_type))) return steps if __name__ == "__main__": - # DRA_BRANCH can be used for manually testing packaging with PRs - # e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build - branch = os.environ.get("DRA_BRANCH", os.environ["BUILDKITE_BRANCH"]) - try: workflow_type = os.environ["WORKFLOW_TYPE"] + version_qualifier = os.environ.get("VERSION_QUALIFIER", "") except ImportError: print(f"Missing env variable WORKFLOW_TYPE. Use export WORKFLOW_TYPE=\n.Exiting.") exit(1) - # allow manually set version qualifier via BK env vars (should be rarely used, only for testing) - version_qualifier = os.environ.get("VERSION_QUALIFIER", "") + branch = os.environ["BUILDKITE_BRANCH"] structure = {"steps": []} if workflow_type.upper() == "SNAPSHOT" and len(version_qualifier)>0: - # externally set VERSION_QUALIFIER is NOT allowed with SNAPSHOT DRA. Skip. structure["steps"].append({ "label": f"no-op pipeline because prerelease builds (VERSION_QUALIFIER is set to [{version_qualifier}]) don't support the [{workflow_type}] workflow", "command": ":", "skip": "VERSION_QUALIFIER (prerelease builds) not supported with SNAPSHOT DRA", }) else: - if workflow_type.upper() == "STAGING" and len(version_qualifier)==0: - version_qualifier = fetch_version_qualifier_url_with_retries(branch) - # Group defining parallel steps that build and save artifacts group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}") structure["steps"].append({ "group": f":Build Artifacts - {workflow_type.upper()}", "key": group_key, - "steps": build_steps_to_yaml(branch, workflow_type, version_qualifier), + "steps": build_steps_to_yaml(branch, workflow_type), }) # Final step: pull artifacts built above and publish them via the release-manager structure["steps"].extend( - yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key, version_qualifier=version_qualifier)), + yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)), ) print(YAML_HEADER + yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))