diff --git a/.buildkite/dra_pipeline.yml b/.buildkite/dra_pipeline.yml index 10007715ccd..fae27b3c738 100644 --- a/.buildkite/dra_pipeline.yml +++ b/.buildkite/dra_pipeline.yml @@ -6,7 +6,7 @@ steps: set -euo pipefail echo "--- Building [$${WORKFLOW_TYPE}] artifacts" - python3 -m pip install pyyaml + python3 -m pip install pyyaml urllib3 echo "--- Building dynamic pipeline steps" python3 .buildkite/scripts/dra/generatesteps.py > steps.yml echo "--- Printing dynamic pipeline steps" diff --git a/.buildkite/scripts/dra/generatesteps.py b/.buildkite/scripts/dra/generatesteps.py index d26e34009e3..8274e4bba69 100644 --- a/.buildkite/scripts/dra/generatesteps.py +++ b/.buildkite/scripts/dra/generatesteps.py @@ -1,10 +1,38 @@ import os -import sys +import urllib3 +from urllib3.util.retry import Retry +from urllib3.exceptions import HTTPError import yaml YAML_HEADER = '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json\n' +def fetch_version_qualifier_url_with_retries(branch, max_retries=3, backoff_factor=2): + """ + Return the version qualifier from a centralized URL based on branch. For any failure or response apart from 200, + assume no qualifier (return empty string). + """ + url = f"https://storage.googleapis.com/dra-qualifier/{branch}" + + http = urllib3.PoolManager() + + # Configure retries: Exponential backoff with 3 retries + retries = Retry( + total=max_retries, + backoff_factor=backoff_factor, # Wait time increases exponentially (2, 4, 8s) + status_forcelist=[500, 502, 503, 504], # Retry only on server errors + raise_on_status=False # Do not raise exception on failed status codes + ) + + try: + response = http.request("GET", url, retries=retries, timeout=5) + if response.status == 200: + return response.data.decode("utf-8").strip() + except HTTPError as e: + pass + + return "" + def to_bk_key_friendly_string(key): """ Convert and return key to an acceptable format for Buildkite's key: field @@ -15,7 +43,7 @@ def to_bk_key_friendly_string(key): return key.translate(mapping_table) -def package_x86_step(branch, workflow_type): +def package_x86_step(branch, workflow_type, version_qualifier): step = f''' - label: ":package: Build packages / {branch}-{workflow_type.upper()} DRA artifacts" key: "logstash_build_packages_dra" @@ -27,6 +55,7 @@ def package_x86_step(branch, workflow_type): diskSizeGb: 200 command: | export WORKFLOW_TYPE="{workflow_type}" + export VERSION_QUALIFIER="{version_qualifier}" export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH" eval "$(rbenv init -)" .buildkite/scripts/dra/build_packages.sh @@ -34,7 +63,7 @@ def package_x86_step(branch, workflow_type): return step -def package_x86_docker_step(branch, workflow_type): +def package_x86_docker_step(branch, workflow_type, version_qualifier): step = f''' - label: ":package: Build x86_64 Docker / {branch}-{workflow_type.upper()} DRA artifacts" key: "logstash_build_x86_64_docker_dra" @@ -46,6 +75,7 @@ def package_x86_docker_step(branch, workflow_type): diskSizeGb: 200 command: | export WORKFLOW_TYPE="{workflow_type}" + export VERSION_QUALIFIER="{version_qualifier}" export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH" export ARCH="x86_64" eval "$(rbenv init -)" @@ -54,7 +84,7 @@ def package_x86_docker_step(branch, workflow_type): return step -def package_aarch64_docker_step(branch, workflow_type): +def package_aarch64_docker_step(branch, workflow_type, version_qualifier): step = f''' - label: ":package: Build aarch64 Docker / {branch}-{workflow_type.upper()} DRA artifacts" key: "logstash_build_aarch64_docker_dra" @@ -65,6 +95,7 @@ def package_aarch64_docker_step(branch, workflow_type): diskSizeGb: 200 command: | export WORKFLOW_TYPE="{workflow_type}" + export VERSION_QUALIFIER="{version_qualifier}" export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH" export ARCH="aarch64" eval "$(rbenv init -)" @@ -73,7 +104,7 @@ def package_aarch64_docker_step(branch, workflow_type): return step -def publish_dra_step(branch, workflow_type, depends_on): +def publish_dra_step(branch, workflow_type, depends_on, version_qualifier): step = f''' - label: ":elastic-stack: Publish / {branch}-{workflow_type.upper()} DRA artifacts" key: "logstash_publish_dra" @@ -92,50 +123,59 @@ def publish_dra_step(branch, workflow_type, depends_on): sudo chown -R :1000 build echo "+++ Running DRA publish step" export WORKFLOW_TYPE="{workflow_type}" + export VERSION_QUALIFIER="{version_qualifier}" .buildkite/scripts/dra/publish.sh ''' return step -def build_steps_to_yaml(branch, workflow_type): +def build_steps_to_yaml(branch, workflow_type, version_qualifier): steps = [] - steps.extend(yaml.safe_load(package_x86_step(branch, workflow_type))) - steps.extend(yaml.safe_load(package_x86_docker_step(branch, workflow_type))) - steps.extend(yaml.safe_load(package_aarch64_docker_step(branch, workflow_type))) + steps.extend(yaml.safe_load(package_x86_step(branch, workflow_type, version_qualifier))) + steps.extend(yaml.safe_load(package_x86_docker_step(branch, workflow_type, version_qualifier))) + steps.extend(yaml.safe_load(package_aarch64_docker_step(branch, workflow_type, version_qualifier))) return steps if __name__ == "__main__": + # DRA_BRANCH can be used for manually testing packaging with PRs + # e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build + branch = os.environ.get("DRA_BRANCH", os.environ["BUILDKITE_BRANCH"]) + try: workflow_type = os.environ["WORKFLOW_TYPE"] - version_qualifier = os.environ.get("VERSION_QUALIFIER", "") except ImportError: print(f"Missing env variable WORKFLOW_TYPE. Use export WORKFLOW_TYPE=\n.Exiting.") exit(1) - branch = os.environ["BUILDKITE_BRANCH"] + # allow manually set version qualifier via BK env vars (should be rarely used, only for testing) + version_qualifier = os.environ.get("VERSION_QUALIFIER", "") structure = {"steps": []} if workflow_type.upper() == "SNAPSHOT" and len(version_qualifier)>0: + # externally set VERSION_QUALIFIER is NOT allowed with SNAPSHOT DRA. Skip. structure["steps"].append({ "label": f"no-op pipeline because prerelease builds (VERSION_QUALIFIER is set to [{version_qualifier}]) don't support the [{workflow_type}] workflow", "command": ":", "skip": "VERSION_QUALIFIER (prerelease builds) not supported with SNAPSHOT DRA", }) else: + if workflow_type.upper() == "STAGING" and len(version_qualifier)==0: + version_qualifier = fetch_version_qualifier_url_with_retries(branch) + # Group defining parallel steps that build and save artifacts group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}") structure["steps"].append({ "group": f":Build Artifacts - {workflow_type.upper()}", "key": group_key, - "steps": build_steps_to_yaml(branch, workflow_type), + "steps": build_steps_to_yaml(branch, workflow_type, version_qualifier), }) # Final step: pull artifacts built above and publish them via the release-manager structure["steps"].extend( - yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)), + yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key, version_qualifier=version_qualifier)), ) print(YAML_HEADER + yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))