From cb169a27fd9093ccbf3b043a5ec946857116b441 Mon Sep 17 00:00:00 2001 From: kevin Date: Tue, 20 Aug 2024 22:23:46 +0000 Subject: [PATCH 1/9] p Signed-off-by: kevin --- Dockerfile | 80 ++++++++++++++++++------------------------------------ 1 file changed, 26 insertions(+), 54 deletions(-) diff --git a/Dockerfile b/Dockerfile index c13cb5c7e7a9..53c1e14842b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,34 +9,23 @@ ARG CUDA_VERSION=12.4.1 #################### BASE BUILD IMAGE #################### # prepare basic build environment FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04 AS base - ARG CUDA_VERSION=12.4.1 ARG PYTHON_VERSION=3.10 - ENV DEBIAN_FRONTEND=noninteractive +# Install Python and other dependencies RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ && apt-get update -y \ - && apt-get install -y ccache software-properties-common \ + && apt-get install -y ccache software-properties-common git curl sudo \ && add-apt-repository ppa:deadsnakes/ppa \ && apt-get update -y \ - && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \ - && if [ "${PYTHON_VERSION}" != "3" ]; then update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1; fi \ - && python3 --version - -RUN apt-get update -y \ - && apt-get install -y git curl sudo - -# Install pip s.t. it will be compatible with our PYTHON_VERSION -RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} -RUN python3 -m pip --version - -# Workaround for https://github.com/openai/triton/issues/2507 and -# https://github.com/pytorch/pytorch/issues/107960 -- hopefully -# this won't be needed for future versions of this docker image -# or future versions of triton. -RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/ + && apt-get install -y python${PYTHON_VERSION_BUILD} python${PYTHON_VERSION_BUILD}-dev python${PYTHON_VERSION_BUILD}-venv \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION_BUILD} 1 \ + && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION_BUILD} \ + && ln -sf /usr/bin/python${PYTHON_VERSION_BUILD}-config /usr/bin/python3-config \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION_BUILD} \ + && python3 --version && python3 -m pip --version WORKDIR /workspace @@ -50,29 +39,17 @@ RUN --mount=type=cache,target=/root/.cache/pip \ COPY requirements-mamba.txt requirements-mamba.txt RUN python3 -m pip install packaging RUN python3 -m pip install -r requirements-mamba.txt - -# cuda arch list used by torch -# can be useful for both `dev` and `test` -# explicitly set the list to avoid issues with torch 2.2 -# see https://github.com/pytorch/pytorch/pull/123243 -ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX' -ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} #################### BASE BUILD IMAGE #################### #################### WHEEL BUILD IMAGE #################### FROM base AS build -ARG PYTHON_VERSION=3.10 - # install build dependencies COPY requirements-build.txt requirements-build.txt RUN --mount=type=cache,target=/root/.cache/pip \ python3 -m pip install -r requirements-build.txt -# install compiler cache to speed up compilation leveraging local or remote caching -RUN apt-get update -y && apt-get install -y ccache - # files and directories related to build wheels COPY csrc csrc COPY setup.py setup.py @@ -95,6 +72,8 @@ ARG buildkite_commit ENV BUILDKITE_COMMIT=${buildkite_commit} ARG USE_SCCACHE +ARG SCCACHE_BUCKET_NAME=vllm-build-sccache +ARG SCCACHE_REGION=us-west-2 # if USE_SCCACHE is set, use sccache to speed up compilation RUN --mount=type=cache,target=/root/.cache/pip \ if [ "$USE_SCCACHE" = "1" ]; then \ @@ -103,12 +82,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \ && tar -xzf sccache.tar.gz \ && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \ && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \ - && if [ "$CUDA_VERSION" = "11.8.0" ]; then \ - export SCCACHE_BUCKET=vllm-build-sccache-2; \ - else \ - export SCCACHE_BUCKET=vllm-build-sccache; \ - fi \ - && export SCCACHE_REGION=us-west-2 \ + && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \ + && export SCCACHE_REGION=${SCCACHE_REGION} \ && export CMAKE_BUILD_TYPE=Release \ && sccache --show-stats \ && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \ @@ -160,29 +135,26 @@ FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu20.04 AS vllm-base ARG CUDA_VERSION=12.4.1 ARG PYTHON_VERSION=3.10 WORKDIR /vllm-workspace +ENV DEBIAN_FRONTEND=noninteractive + +RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \ + echo "export PYTHON_VERSION_STR=${PYTHON_VERSION_STR}" >> /etc/environment +# Install Python and other dependencies RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ && apt-get update -y \ - && apt-get install -y ccache software-properties-common \ + && apt-get install -y ccache software-properties-common git curl sudo vim gcc \ && add-apt-repository ppa:deadsnakes/ppa \ && apt-get update -y \ && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \ - && if [ "${PYTHON_VERSION}" != "3" ]; then update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1; fi \ - && python3 --version - -RUN apt-get update -y \ - && apt-get install -y python3-pip git vim curl libibverbs-dev - -# Install pip s.t. it will be compatible with our PYTHON_VERSION -RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} -RUN python3 -m pip --version - -# Workaround for https://github.com/openai/triton/issues/2507 and -# https://github.com/pytorch/pytorch/issues/107960 -- hopefully -# this won't be needed for future versions of this docker image -# or future versions of triton. -RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ + && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \ + && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ + && python3 --version && python3 -m pip --version \ + && apt-get update -y \ + && apt-get install -y libiverbs-dev # install vllm wheel first, so that torch etc will be installed RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \ @@ -194,7 +166,7 @@ RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamb python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir RUN --mount=type=cache,target=/root/.cache/pip \ - python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.4/flashinfer-0.1.4+cu121torch2.4-cp310-cp310-linux_x86_64.whl + python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.4/flashinfer-0.1.4+cu121torch2.4-cp${PYTHON_VERSION_STR}-cp${PYTHON_VERSION_STR}-linux_x86_64.whl #################### vLLM installation IMAGE #################### From c58746af88f13e20c5b09ccb08ff2e087c537c5c Mon Sep 17 00:00:00 2001 From: kevin Date: Tue, 20 Aug 2024 22:31:06 +0000 Subject: [PATCH 2/9] p Signed-off-by: kevin --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 53c1e14842b3..00ad32700768 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,11 +20,11 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ && apt-get install -y ccache software-properties-common git curl sudo \ && add-apt-repository ppa:deadsnakes/ppa \ && apt-get update -y \ - && apt-get install -y python${PYTHON_VERSION_BUILD} python${PYTHON_VERSION_BUILD}-dev python${PYTHON_VERSION_BUILD}-venv \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION_BUILD} 1 \ - && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION_BUILD} \ - && ln -sf /usr/bin/python${PYTHON_VERSION_BUILD}-config /usr/bin/python3-config \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION_BUILD} \ + && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ + && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \ + && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ && python3 --version && python3 -m pip --version WORKDIR /workspace From 8185362e6ad161d42bd72c8881eb30ec0db785c7 Mon Sep 17 00:00:00 2001 From: kevin Date: Tue, 20 Aug 2024 22:31:57 +0000 Subject: [PATCH 3/9] p Signed-off-by: kevin --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 00ad32700768..7329fcf27286 100644 --- a/Dockerfile +++ b/Dockerfile @@ -166,6 +166,7 @@ RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamb python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir RUN --mount=type=cache,target=/root/.cache/pip \ + . /etc/environment && \ python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.4/flashinfer-0.1.4+cu121torch2.4-cp${PYTHON_VERSION_STR}-cp${PYTHON_VERSION_STR}-linux_x86_64.whl #################### vLLM installation IMAGE #################### From da2c420e63f0166545db121f39a43a944db2e362 Mon Sep 17 00:00:00 2001 From: kevin Date: Tue, 20 Aug 2024 22:58:50 +0000 Subject: [PATCH 4/9] p Signed-off-by: kevin --- Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7329fcf27286..e20e97e39547 100644 --- a/Dockerfile +++ b/Dockerfile @@ -147,14 +147,12 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ && apt-get install -y ccache software-properties-common git curl sudo vim gcc \ && add-apt-repository ppa:deadsnakes/ppa \ && apt-get update -y \ - && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \ + && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libibverbs-dev \ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \ && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \ && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ - && python3 --version && python3 -m pip --version \ - && apt-get update -y \ - && apt-get install -y libiverbs-dev + && python3 --version && python3 -m pip --version # install vllm wheel first, so that torch etc will be installed RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \ From cb6a9a006df95c35ee37f0c2cccbf5560af0ff27 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 21 Aug 2024 04:14:36 +0000 Subject: [PATCH 5/9] p Signed-off-by: kevin --- .buildkite/test-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 59d7241bd452..c76fe4ef6946 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -34,7 +34,7 @@ steps: fast_check: true no_gpu: True commands: - - pip install -r requirements-docs.txt + - python3 -m pip install -r requirements-docs.txt - SPHINXOPTS=\"-W\" make html # Check API reference (if it fails, you may have missing mock imports) - grep \"sig sig-object py\" build/html/dev/sampling_params.html From 494195fe3df5b7921d544ce4a807230a2ffdcad4 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 21 Aug 2024 07:47:08 +0000 Subject: [PATCH 6/9] p Signed-off-by: kevin --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e20e97e39547..a4f485c6ff96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -144,7 +144,7 @@ RUN PYTHON_VERSION_STR=$(echo ${PYTHON_VERSION} | sed 's/\.//g') && \ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ && apt-get update -y \ - && apt-get install -y ccache software-properties-common git curl sudo vim gcc \ + && apt-get install -y ccache software-properties-common git curl sudo vim python3-pip \ && add-apt-repository ppa:deadsnakes/ppa \ && apt-get update -y \ && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libibverbs-dev \ From 4d99c3d0870aab88f168db0a47831531239a58bf Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 21 Aug 2024 07:47:42 +0000 Subject: [PATCH 7/9] p Signed-off-by: kevin --- .buildkite/test-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index c76fe4ef6946..59d7241bd452 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -34,7 +34,7 @@ steps: fast_check: true no_gpu: True commands: - - python3 -m pip install -r requirements-docs.txt + - pip install -r requirements-docs.txt - SPHINXOPTS=\"-W\" make html # Check API reference (if it fails, you may have missing mock imports) - grep \"sig sig-object py\" build/html/dev/sampling_params.html From 8a8d9dec90245034c781d415bd77f297c9f9641d Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 21 Aug 2024 09:00:08 +0000 Subject: [PATCH 8/9] p Signed-off-by: kevin --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a4f485c6ff96..e2e0e22c94b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,7 +73,7 @@ ENV BUILDKITE_COMMIT=${buildkite_commit} ARG USE_SCCACHE ARG SCCACHE_BUCKET_NAME=vllm-build-sccache -ARG SCCACHE_REGION=us-west-2 +ARG SCCACHE_REGION_NAME=us-west-2 # if USE_SCCACHE is set, use sccache to speed up compilation RUN --mount=type=cache,target=/root/.cache/pip \ if [ "$USE_SCCACHE" = "1" ]; then \ @@ -83,7 +83,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \ && sudo mv sccache-v0.8.1-x86_64-unknown-linux-musl/sccache /usr/bin/sccache \ && rm -rf sccache.tar.gz sccache-v0.8.1-x86_64-unknown-linux-musl \ && export SCCACHE_BUCKET=${SCCACHE_BUCKET_NAME} \ - && export SCCACHE_REGION=${SCCACHE_REGION} \ + && export SCCACHE_REGION=${SCCACHE_REGION_NAME} \ + && export SCCACHE_IDLE_TIMEOUT=0 \ && export CMAKE_BUILD_TYPE=Release \ && sccache --show-stats \ && python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \ From 6e59cdf31a35141bdde45dc054447a80b59f3f9f Mon Sep 17 00:00:00 2001 From: kevin Date: Thu, 22 Aug 2024 18:21:29 +0000 Subject: [PATCH 9/9] p Signed-off-by: kevin --- Dockerfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Dockerfile b/Dockerfile index e2e0e22c94b8..36fcc2f83e9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,12 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ && python3 --version && python3 -m pip --version +# Workaround for https://github.com/openai/triton/issues/2507 and +# https://github.com/pytorch/pytorch/issues/107960 -- hopefully +# this won't be needed for future versions of this docker image +# or future versions of triton. +RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/ + WORKDIR /workspace # install build and runtime dependencies @@ -39,6 +45,13 @@ RUN --mount=type=cache,target=/root/.cache/pip \ COPY requirements-mamba.txt requirements-mamba.txt RUN python3 -m pip install packaging RUN python3 -m pip install -r requirements-mamba.txt + +# cuda arch list used by torch +# can be useful for both `dev` and `test` +# explicitly set the list to avoid issues with torch 2.2 +# see https://github.com/pytorch/pytorch/pull/123243 +ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX' +ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} #################### BASE BUILD IMAGE #################### #################### WHEEL BUILD IMAGE #################### @@ -155,6 +168,12 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ && python3 --version && python3 -m pip --version +# Workaround for https://github.com/openai/triton/issues/2507 and +# https://github.com/pytorch/pytorch/issues/107960 -- hopefully +# this won't be needed for future versions of this docker image +# or future versions of triton. +RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/ + # install vllm wheel first, so that torch etc will be installed RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \ --mount=type=cache,target=/root/.cache/pip \