Skip to content

Commit c6e9fe8

Browse files
narendasannarendasan
authored
Merge pull request #2073 from pytorch/gha-ci-infra
infra: testing out GHA CI
2 parents 0e5a497 + a3edfdc commit c6e9fe8

24 files changed

+912
-78
lines changed

‎.github/scripts/install-torch-tensorrt.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
set -eou pipefail
3+
# Source conda so it's available to the script environment
4+
source ${BUILD_ENV_FILE}
5+
${CONDA_RUN} ${PIP_INSTALL_TORCH} torchvision pyyaml
6+
export TRT_VERSION=$(${CONDA_RUN} python -c "import versions; versions.tensorrt_version()")
7+
${CONDA_RUN} python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*+${CU_VERSION}*.whl tensorrt~=${TRT_VERSION} tensorrt-bindings~=${TRT_VERSION} --extra-index-url=https://pypi.ngc.nvidia.com
8+
9+
echo -e "Running test script";

‎.github/scripts/setup-env.sh‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/usr/bin/env bash
2+
3+
set -euxo pipefail
4+
5+
# Prepare conda
6+
set +x && eval "$($(which conda) shell.bash hook)" && set -x
7+
8+
# Setup the OS_TYPE environment variable that should be used for conditions involving the OS below.
9+
case $(uname) in
10+
Linux)
11+
OS_TYPE=linux
12+
;;
13+
Darwin)
14+
OS_TYPE=macos
15+
;;
16+
MSYS*)
17+
OS_TYPE=windows
18+
;;
19+
*)
20+
echo "Unknown OS type:" $(uname)
21+
exit 1
22+
;;
23+
esac
24+
25+
if [[ "${OS_TYPE}" == "macos" && $(uname -m) == x86_64 ]]; then
26+
echo '::group::Uninstall system JPEG libraries on macOS'
27+
# The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG and PNG libraries
28+
# installed by default that interfere with our build. We uninstall them here and use the one from conda below.
29+
IMAGE_LIBS=$(brew list | grep -E "jpeg|png")
30+
for lib in $IMAGE_LIBS; do
31+
brew uninstall --ignore-dependencies --force "${lib}"
32+
done
33+
echo '::endgroup::'
34+
fi
35+
36+
echo '::group::Create build environment'
37+
# See https://github.com/pytorch/vision/issues/7296 for ffmpeg
38+
conda create \
39+
--name ci \
40+
--quiet --yes \
41+
python="${PYTHON_VERSION}" pip \
42+
ninja cmake \
43+
libpng jpeg \
44+
'ffmpeg<4.3'
45+
conda activate ci
46+
pip install --progress-bar=off --upgrade setuptools
47+
48+
# See https://github.com/pytorch/vision/issues/6790
49+
if [[ "${PYTHON_VERSION}" != "3.11" ]]; then
50+
pip install --progress-bar=off av!=10.0.0
51+
fi
52+
53+
echo '::endgroup::'
54+
55+
if [[ "${OS_TYPE}" == windows && "${GPU_ARCH_TYPE}" == cuda ]]; then
56+
echo '::group::Install VisualStudio CUDA extensions on Windows'
57+
if [[ "${VC_YEAR:-}" == "2022" ]]; then
58+
TARGET_DIR="/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/MSBuild/Microsoft/VC/v170/BuildCustomizations"
59+
else
60+
TARGET_DIR="/c/Program Files (x86)/Microsoft Visual Studio/2019/BuildTools/MSBuild/Microsoft/VC/v160/BuildCustomizations"
61+
fi
62+
mkdir -p "${TARGET_DIR}"
63+
cp -r "${CUDA_HOME}/MSBuildExtensions/"* "${TARGET_DIR}"
64+
echo '::endgroup::'
65+
fi
66+
67+
echo '::group::Install PyTorch'
68+
# TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`.
69+
if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
70+
CHANNEL=test
71+
else
72+
CHANNEL=nightly
73+
fi
74+
75+
pip install --progress-bar=off light-the-torch
76+
ltt install --progress-bar=off \
77+
--pytorch-computation-backend="${GPU_ARCH_TYPE}${GPU_ARCH_VERSION}" \
78+
--pytorch-channel="${CHANNEL}" \
79+
torch
80+
81+
if [[ $GPU_ARCH_TYPE == 'cuda' ]]; then
82+
python -c "import torch; exit(not torch.cuda.is_available())"
83+
fi
84+
echo '::endgroup::'
85+
86+
echo '::group::Install third party dependencies prior to TorchVision install'
87+
# Installing with `easy_install`, e.g. `python setup.py install` or `python setup.py develop`, has some quirks when
88+
# when pulling in third-party dependencies. For example:
89+
# - On Windows, we often hit an SSL error although `pip` can install just fine.
90+
# - It happily pulls in pre-releases, which can lead to more problems down the line.
91+
# `pip` does not unless explicitly told to do so.
92+
# Thus, we use `easy_install` to extract the third-party dependencies here and install them upfront with `pip`.
93+
python setup.py egg_info
94+
# The requires.txt cannot be used with `pip install -r` directly. The requirements are listed at the top and the
95+
# optional dependencies come in non-standard syntax after a blank line. Thus, we just extract the header.
96+
sed -e '/^$/,$d' *.egg-info/requires.txt | tee requirements.txt
97+
pip install --progress-bar=off -r requirements.txt
98+
echo '::endgroup::'
99+
100+
echo '::group::Install TorchVision'
101+
python setup.py develop
102+
echo '::endgroup::'
103+
104+
echo '::group::Collect environment information'
105+
conda list
106+
python -m torch.utils.collect_env
107+
echo '::endgroup::'

‎.github/scripts/tests-torchscript-cpp.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
./.github/scripts/setup-env.sh
6+
7+
# Activate conda environment
8+
eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci
9+
10+
echo '::group::Install testing utilities'
11+
pip install --progress-bar=off pytest pytest-mock pytest-cov
12+
echo '::endgroup::'
13+
14+
pytest --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v --durations=25

‎.github/workflows/build-test.yml‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
name: Build and test linux wheels
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- nightly
8+
- main
9+
- release/*
10+
tags:
11+
# NOTE: Binary build pipelines should only get triggered on release candidate builds
12+
# Release candidate tags look like: v1.11.0-rc1
13+
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
14+
workflow_dispatch:
15+
16+
jobs:
17+
generate-matrix:
18+
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
19+
with:
20+
package-type: wheel
21+
os: linux
22+
test-infra-repository: pytorch/test-infra
23+
test-infra-ref: main
24+
with-rocm: false
25+
with-cpu: false
26+
27+
build:
28+
needs: generate-matrix
29+
strategy:
30+
fail-fast: false
31+
matrix:
32+
include:
33+
- repository: pytorch/tensorrt
34+
pre-script: packaging/pre_build_script.sh
35+
env-var-script: packaging/env_vars.txt
36+
post-script: ""
37+
smoke-test-script: ""
38+
package-name: torch_tensorrt
39+
name: Build torch-tensorrt whl package
40+
uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
41+
with:
42+
repository: ${{ matrix.repository }}
43+
ref: ""
44+
test-infra-repository: pytorch/test-infra
45+
test-infra-ref: main
46+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
47+
pre-script: ${{ matrix.pre-script }}
48+
env-var-script: ${{ matrix.env-var-script }}
49+
post-script: ${{ matrix.post-script }}
50+
package-name: ${{ matrix.package-name }}
51+
smoke-test-script: ${{ matrix.smoke-test-script }}
52+
trigger-event: ${{ github.event_name }}
53+
secrets:
54+
AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }}
55+
AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }}
56+
57+
# tests-py-torchscript-fe:
58+
# name: Test torchscript frontend [Python]
59+
# needs: [generate-matrix, build]
60+
# strategy:
61+
# fail-fast: false
62+
# matrix:
63+
# include:
64+
# - repository: pytorch/tensorrt
65+
# package-name: torch_tensorrt
66+
# pre-script: packaging/pre_build_script.sh
67+
# uses: pytorch/tensorrt/.github/workflows/linux-test.yml@gha-ci-infra
68+
# with:
69+
# job-name: tests-py-torchscript-fe
70+
# repository: "pytorch/tensorrt"
71+
# ref: ""
72+
# test-infra-repository: pytorch/test-infra
73+
# test-infra-ref: main
74+
# build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
75+
# pre-script: ${{ matrix.pre-script }}
76+
# script: |
77+
# export USE_HOST_DEPS=1
78+
# pushd .
79+
# cd tests/modules
80+
# ${CONDA_RUN} python -m pip install -r requirements.txt
81+
# ${CONDA_RUN} python hub.py
82+
# popd
83+
# pushd .
84+
# cd tests/py/ts
85+
# ${CONDA_RUN} python -m pip install --pre pytest timm transformers parameterized expecttest --use-deprecated=legacy-resolver
86+
# ${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_api_test_results.xml api/
87+
# ${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_models_test_results.xml models/
88+
# ${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_integrations_test_results.xml integrations/
89+
# popd
90+
91+
tests-py-dynamo-converters:
92+
name: Test dynamo converters [Python]
93+
needs: [generate-matrix, build]
94+
strategy:
95+
fail-fast: false
96+
matrix:
97+
include:
98+
- repository: pytorch/tensorrt
99+
package-name: torch_tensorrt
100+
pre-script: packaging/pre_build_script.sh
101+
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@gha-ci-infra
102+
with:
103+
job-name: tests-py-dynamo-converters
104+
repository: "pytorch/tensorrt"
105+
ref: ""
106+
test-infra-repository: pytorch/test-infra
107+
test-infra-ref: main
108+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
109+
pre-script: ${{ matrix.pre-script }}
110+
script: |
111+
export USE_HOST_DEPS=1
112+
pushd .
113+
cd tests/modules
114+
${CONDA_RUN} python -m pip install -r requirements.txt
115+
${CONDA_RUN} python hub.py
116+
popd
117+
pushd .
118+
cd tests/py/dynamo
119+
${CONDA_RUN} python -m pip install --pre pytest-xdist timm transformers parameterized expecttest --use-deprecated=legacy-resolver
120+
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 10 conversion/
121+
popd
122+
123+
tests-py-dynamo-fe:
124+
name: Test dynamo frontend [Python]
125+
needs: [generate-matrix, build]
126+
strategy:
127+
fail-fast: false
128+
matrix:
129+
include:
130+
- repository: pytorch/tensorrt
131+
package-name: torch_tensorrt
132+
pre-script: packaging/pre_build_script.sh
133+
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@gha-ci-infra
134+
with:
135+
job-name: tests-py-dynamo-fe
136+
repository: "pytorch/tensorrt"
137+
ref: ""
138+
test-infra-repository: pytorch/test-infra
139+
test-infra-ref: main
140+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
141+
pre-script: ${{ matrix.pre-script }}
142+
script: |
143+
export USE_HOST_DEPS=1
144+
pushd .
145+
cd tests/py/dynamo
146+
${CONDA_RUN} python -m pip install --pre pytest timm transformers parameterized expecttest --use-deprecated=legacy-resolver
147+
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_fe_test_results.xml --ir dynamo models/test_models_export.py
148+
popd
149+
150+
tests-py-torch-compile-be:
151+
name: Test torch compile backend [Python]
152+
needs: [generate-matrix, build]
153+
strategy:
154+
fail-fast: false
155+
matrix:
156+
include:
157+
- repository: pytorch/tensorrt
158+
package-name: torch_tensorrt
159+
pre-script: packaging/pre_build_script.sh
160+
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@gha-ci-infra
161+
with:
162+
job-name: tests-py-torch-compile-be
163+
repository: "pytorch/tensorrt"
164+
ref: ""
165+
test-infra-repository: pytorch/test-infra
166+
test-infra-ref: main
167+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
168+
pre-script: ${{ matrix.pre-script }}
169+
script: |
170+
export USE_HOST_DEPS=1
171+
pushd .
172+
cd tests/py/dynamo
173+
${CONDA_RUN} python -m pip install --pre pytest-xdist timm transformers parameterized expecttest --use-deprecated=legacy-resolver
174+
${CONDA_RUN} python -m pytest -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/
175+
${CONDA_RUN} python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_comple_be_e2e_test_results.xml --ir torch_compile models/test_models.py
176+
popd
177+
178+
tests-py-dynamo-core:
179+
name: Test dynamo core [Python]
180+
needs: [generate-matrix, build]
181+
strategy:
182+
fail-fast: false
183+
matrix:
184+
include:
185+
- repository: pytorch/tensorrt
186+
package-name: torch_tensorrt
187+
pre-script: packaging/pre_build_script.sh
188+
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@gha-ci-infra
189+
with:
190+
job-name: tests-py-dynamo-core
191+
repository: "pytorch/tensorrt"
192+
ref: ""
193+
test-infra-repository: pytorch/test-infra
194+
test-infra-ref: main
195+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
196+
pre-script: ${{ matrix.pre-script }}
197+
script: |
198+
export USE_HOST_DEPS=1
199+
pushd .
200+
cd tests/py/dynamo
201+
${CONDA_RUN} python -m pip install --pre pytest-xdist timm transformers parameterized expecttest --use-deprecated=legacy-resolver
202+
${CONDA_RUN} python -m pytest -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml runtime/
203+
${CONDA_RUN} python -m pytest -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/
204+
${CONDA_RUN} python -m pytest -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/
205+
popd

‎.github/workflows/docgen.yml‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,32 +10,33 @@ on:
1010

1111
jobs:
1212
build-docs:
13-
runs-on: ubuntu-20.04
13+
runs-on: ubuntu-latest
1414
container:
1515
image: docker.io/pytorch/manylinux-builder:cuda12.1
1616
steps:
1717
- uses: actions/checkout@v3
1818
with:
1919
ref: ${{github.head_ref}}
20+
- name: Select python
21+
run: |
22+
export PATH=/opt/python/cp311-cp311/bin/:$PATH
2023
- name: Install base deps
2124
run: |
22-
./packaging/pre_build_script.sh
25+
python3 -m pip install pyyaml
26+
VERSION_SUFFIX=cu121 ./packaging/pre_build_script.sh
2327
- name: Get HEAD SHA
2428
id: vars
2529
run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
26-
- name: Get Bazel version
27-
id: bazel_info
28-
run: echo "version=$(cat .bazelversion)" >> $GITHUB_OUTPUT
2930
- name: Build Python Package
3031
run: |
31-
cp toolchains/ci_workspaces/WORKSPACE.x86_64.cu121.release.rhel WORKSPACE
32-
python -m pip install pip<=23
33-
python -m pip install --pre -e . --extra-index-url https://download.pytorch.org/whl/nightly/cu121
32+
python3 -m pip install pip --upgrade
33+
CUDA_HOME=/usr/local/cuda-12.1 CI_BUILD=1 python3 -m pip install --pre -e . --extra-index-url https://download.pytorch.org/whl/nightly/cu121
3434
- name: Generate New Docs
3535
run: |
3636
cd docsrc
37-
python -m pip install -r requirements.txt
38-
python -c "import torch_tensorrt; print(torch_tensorrt.__version__)"
37+
yum install -y doxygen pandoc
38+
python3 -m pip install -r requirements.txt
39+
python3 -c "import torch_tensorrt; print(torch_tensorrt.__version__)"
3940
make html
4041
cd ..
4142
- uses: stefanzweifel/git-auto-commit-action@v4

0 commit comments

Comments
 (0)