From 852a80354fbf1af8ff95765963c4ae78685209a3 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Fri, 26 Sep 2025 04:06:53 -0700 Subject: [PATCH 1/4] Push multiarch manifests as nightly builds Signed-off-by: Sahithi Chigurupati --- .buildkite/release-pipeline.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index 8c6ef7817aaf..caca60531634 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -150,11 +150,10 @@ steps: queue: cpu_queue_postmerge commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT vllm/vllm-openai:nightly" - - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT vllm/vllm-openai:nightly-$BUILDKITE_COMMIT" - - "docker push vllm/vllm-openai:nightly" - - "docker push vllm/vllm-openai:nightly-$BUILDKITE_COMMIT" + - "docker manifest create vllm/vllm-openai:nightly public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend" + - "docker manifest create vllm/vllm-openai:nightly-$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend" + - "docker manifest push vllm/vllm-openai:nightly" + - "docker manifest push vllm/vllm-openai:nightly-$BUILDKITE_COMMIT" # Clean up old nightly builds (keep only last 14) - "bash .buildkite/scripts/cleanup-nightly-builds.sh" plugins: From b8fbd1febf078b157d62a19f509b2be5c693d531 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Fri, 26 Sep 2025 05:12:14 -0700 Subject: [PATCH 2/4] Fix nightly cleanup script Signed-off-by: Sahithi Chigurupati --- .buildkite/release-pipeline.yaml | 1 + .buildkite/scripts/cleanup-nightly-builds.sh | 23 +++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index caca60531634..9d2fc5baf070 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -162,3 +162,4 @@ steps: password-env: DOCKERHUB_TOKEN env: DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" diff --git a/.buildkite/scripts/cleanup-nightly-builds.sh b/.buildkite/scripts/cleanup-nightly-builds.sh index 1a82f7d08523..c3845f8441af 100755 --- a/.buildkite/scripts/cleanup-nightly-builds.sh +++ b/.buildkite/scripts/cleanup-nightly-builds.sh @@ -8,19 +8,36 @@ set -ex # DockerHub API endpoint for vllm/vllm-openai repository REPO_API_URL="https://hub.docker.com/v2/repositories/vllm/vllm-openai/tags" -# Get DockerHub token from environment +# Get DockerHub credentials from environment if [ -z "$DOCKERHUB_TOKEN" ]; then echo "Error: DOCKERHUB_TOKEN environment variable is not set" exit 1 fi +if [ -z "$DOCKERHUB_USERNAME" ]; then + echo "Error: DOCKERHUB_USERNAME environment variable is not set" + exit 1 +fi + +# Get DockerHub bearer token +echo "Getting DockerHub bearer token..." +BEARER_TOKEN=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_TOKEN\"}" \ + "https://hub.docker.com/v2/users/login" | jq -r '.token') + +if [ -z "$BEARER_TOKEN" ] || [ "$BEARER_TOKEN" = "null" ]; then + echo "Error: Failed to get DockerHub bearer token" + exit 1 +fi + # Function to get all tags from DockerHub get_all_tags() { local page=1 local all_tags="" while true; do - local response=$(curl -s -H "Authorization: Bearer $DOCKERHUB_TOKEN" \ + local response=$(curl -s -H "Authorization: Bearer $BEARER_TOKEN" \ "$REPO_API_URL?page=$page&page_size=100") # Get both last_updated timestamp and tag name, separated by | @@ -43,7 +60,7 @@ delete_tag() { echo "Deleting tag: $tag_name" local delete_url="https://hub.docker.com/v2/repositories/vllm/vllm-openai/tags/$tag_name" - local response=$(curl -s -X DELETE -H "Authorization: Bearer $DOCKERHUB_TOKEN" "$delete_url") + local response=$(curl -s -X DELETE -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url") if echo "$response" | jq -e '.detail' > /dev/null 2>&1; then echo "Warning: Failed to delete tag $tag_name: $(echo "$response" | jq -r '.detail')" From 1ec3636714962a70b7e6f4f6cb23042df7d13641 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Wed, 1 Oct 2025 09:51:19 -0700 Subject: [PATCH 3/4] push docker containers for nightly builds Signed-off-by: Sahithi Chigurupati --- .buildkite/release-pipeline.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index 9d2fc5baf070..7294cd3b03bd 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -150,8 +150,14 @@ steps: queue: cpu_queue_postmerge commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker manifest create vllm/vllm-openai:nightly public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend" - - "docker manifest create vllm/vllm-openai:nightly-$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend" + - "docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64" + - "docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64" + - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 vllm/vllm-openai:nightly-x86_64" + - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 vllm/vllm-openai:nightly-aarch64" + - "docker push vllm/vllm-openai:nightly-x86_64" + - "docker push vllm/vllm-openai:nightly-aarch64" + - "docker manifest create vllm/vllm-openai:nightly vllm/vllm-openai:nightly-x86_64 vllm/vllm-openai:nightly-aarch64 --amend" + - "docker manifest create vllm/vllm-openai:nightly-$BUILDKITE_COMMIT vllm/vllm-openai:nightly-x86_64 vllm/vllm-openai:nightly-aarch64 --amend" - "docker manifest push vllm/vllm-openai:nightly" - "docker manifest push vllm/vllm-openai:nightly-$BUILDKITE_COMMIT" # Clean up old nightly builds (keep only last 14) From 62e8ae3494fed8cb5792a3dbe13e336989bec9c4 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Thu, 2 Oct 2025 05:40:59 -0700 Subject: [PATCH 4/4] fix vulnerabilities in cleanup script Signed-off-by: Sahithi Chigurupati --- .buildkite/scripts/cleanup-nightly-builds.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.buildkite/scripts/cleanup-nightly-builds.sh b/.buildkite/scripts/cleanup-nightly-builds.sh index c3845f8441af..f02a128c6772 100755 --- a/.buildkite/scripts/cleanup-nightly-builds.sh +++ b/.buildkite/scripts/cleanup-nightly-builds.sh @@ -21,10 +21,12 @@ fi # Get DockerHub bearer token echo "Getting DockerHub bearer token..." +set +x BEARER_TOKEN=$(curl -s -X POST \ -H "Content-Type: application/json" \ -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_TOKEN\"}" \ "https://hub.docker.com/v2/users/login" | jq -r '.token') +set -x if [ -z "$BEARER_TOKEN" ] || [ "$BEARER_TOKEN" = "null" ]; then echo "Error: Failed to get DockerHub bearer token" @@ -37,8 +39,10 @@ get_all_tags() { local all_tags="" while true; do + set +x local response=$(curl -s -H "Authorization: Bearer $BEARER_TOKEN" \ "$REPO_API_URL?page=$page&page_size=100") + set -x # Get both last_updated timestamp and tag name, separated by | local tags=$(echo "$response" | jq -r '.results[] | select(.name | startswith("nightly-")) | "\(.last_updated)|\(.name)"') @@ -60,7 +64,9 @@ delete_tag() { echo "Deleting tag: $tag_name" local delete_url="https://hub.docker.com/v2/repositories/vllm/vllm-openai/tags/$tag_name" + set +x local response=$(curl -s -X DELETE -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url") + set -x if echo "$response" | jq -e '.detail' > /dev/null 2>&1; then echo "Warning: Failed to delete tag $tag_name: $(echo "$response" | jq -r '.detail')"