Skip to content

Commit 5c4613d

Browse files
driazatidriazati
committed
[ci] Rebuild Docker images if necessary
This rebuilds Docker images and uses them in later stages in the same build. If the build is running on `main`, then the images are uploaded to Docker Hub automatically once the run is complete. Images are always rebuilt, but Docker Hub functions as a cache. If there have been no changes to `docker/` since the last available hash on Docker Hub, then the build will just use the images from Hub.
1 parent 3e7916d commit 5c4613d

File tree

12 files changed

+739
-300
lines changed

12 files changed

+739
-300
lines changed

‎Jenkinsfile‎

Lines changed: 255 additions & 139 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎jenkins/Build.groovy.j2‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,30 @@ def add_hexagon_permissions() {
5252
{% endfor %}
5353
}
5454

55+
// Run make. First try to do an incremental make from a previous workspace in hope to
56+
// accelerate the compilation. If something is wrong, clean the workspace and then
57+
// build from scratch.
58+
def make(docker_type, path, make_flag) {
59+
timeout(time: max_time, unit: 'MINUTES') {
60+
try {
61+
cmake_build(docker_type, path, make_flag)
62+
// always run cpp test when build
63+
} catch (hudson.AbortException ae) {
64+
// script exited due to user abort, directly throw instead of retry
65+
if (ae.getMessage().contains('script returned exit code 143')) {
66+
throw ae
67+
}
68+
echo 'Incremental compilation failed. Fall back to build from scratch'
69+
sh (
70+
script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}",
71+
label: 'Clear old cmake workspace',
72+
)
73+
cmake_build(docker_type, path, make_flag)
74+
}
75+
}
76+
}
77+
78+
5579
def build() {
5680
stage('Build') {
5781
environment {

‎jenkins/Deploy.groovy.j2‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,25 @@ stage('Build packages') {
1616
}
1717
*/
1818

19+
20+
def update_docker(ecr_image, hub_image) {
21+
if (!ecr_image.contains("amazonaws.com")) {
22+
sh("echo Skipping '${ecr_image}' since it doesn't look like an ECR image")
23+
return
24+
}
25+
docker_init(ecr_image)
26+
sh(
27+
script: """
28+
set -eux
29+
docker tag \
30+
${ecr_image} \
31+
${hub_image}
32+
docker push ${hub_image}
33+
""",
34+
label: "Update ${hub_image} on Docker Hub",
35+
)
36+
}
37+
1938
def deploy_docs() {
2039
// Note: This code must stay in the Jenkinsfile to ensure that it runs
2140
// from a trusted context only
@@ -67,5 +86,36 @@ def deploy() {
6786
}
6887
}
6988
}
89+
if (env.BRANCH_NAME == 'main' && env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) {
90+
node('CPU') {
91+
ws({{ m.per_exec_ws('tvm/deploy-docker') }}) {
92+
try {
93+
withCredentials([string(
94+
credentialsId: 'dockerhub-tlcpackstaging-key',
95+
variable: 'DOCKERHUB_KEY',
96+
)]) {
97+
sh(
98+
script: 'docker login -u tlcpackstaging -p ${DOCKERHUB_KEY}',
99+
label: 'Log in to Docker Hub',
100+
)
101+
}
102+
def date_Ymd_HMS = sh(
103+
script: 'python3 -c \'import datetime; print(datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))\'',
104+
label: 'Determine date',
105+
returnStdout: true,
106+
).trim()
107+
def tag = "${date_Ymd_HMS}-${upstream_revision.substring(0, 8)}"
108+
{% for image in images %}
109+
update_docker({{ image.name }}, "tlcpackstaging/test_{{ image.name }}:${tag}")
110+
{% endfor %}
111+
} finally {
112+
sh(
113+
script: 'docker logout',
114+
label: 'Clean up login credentials'
115+
)
116+
}
117+
}
118+
}
119+
}
70120
}
71121
}

‎jenkins/DockerBuild.groovy.j2‎

Lines changed: 95 additions & 145 deletions
Original file line numberDiff line numberDiff line change
@@ -1,166 +1,116 @@
1-
def build_image(image_name) {
2-
hash = sh(
1+
def ecr_push(full_name) {
2+
aws_account_id = sh(
33
returnStdout: true,
4-
script: 'git log -1 --format=\'%h\''
4+
script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
5+
label: 'Get AWS ID'
56
).trim()
6-
def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}"
7-
sh(
8-
script: "${docker_build} ${image_name} --spec ${full_name}",
9-
label: 'Build docker image'
10-
)
7+
8+
def ecr_name = "${aws_account_id}.{{ aws_ecr_url }}/${full_name}"
9+
try {
10+
withEnv([
11+
"AWS_ACCOUNT_ID=${aws_account_id}",
12+
'AWS_DEFAULT_REGION={{ aws_default_region }}',
13+
"AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
14+
sh(
15+
script: '''
16+
set -eux
17+
aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
18+
''',
19+
label: 'Log in to ECR'
20+
)
21+
sh(
22+
script: """
23+
set -x
24+
docker tag ${full_name} \$AWS_ECR_REPO/${full_name}
25+
docker push \$AWS_ECR_REPO/${full_name}
26+
""",
27+
label: 'Upload image to ECR'
28+
)
29+
}
30+
} finally {
31+
withEnv([
32+
"AWS_ACCOUNT_ID=${aws_account_id}",
33+
'AWS_DEFAULT_REGION={{ aws_default_region }}',
34+
"AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
35+
sh(
36+
script: 'docker logout $AWS_ECR_REPO',
37+
label: 'Clean up login credentials'
38+
)
39+
}
40+
}
41+
return ecr_name
42+
}
43+
44+
def ecr_pull(full_name) {
1145
aws_account_id = sh(
1246
returnStdout: true,
1347
script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
1448
label: 'Get AWS ID'
1549
).trim()
1650

1751
try {
18-
// Use a credential so Jenkins knows to scrub the AWS account ID which is nice
19-
// (but so we don't have to rely it being hardcoded in Jenkins)
20-
withCredentials([string(
21-
credentialsId: 'aws-account-id',
22-
variable: '_ACCOUNT_ID_DO_NOT_USE',
23-
)]) {
24-
withEnv([
25-
"AWS_ACCOUNT_ID=${aws_account_id}",
26-
'AWS_DEFAULT_REGION=us-west-2']) {
27-
sh(
28-
script: '''
29-
set -x
30-
aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
31-
''',
32-
label: 'Log in to ECR'
33-
)
34-
sh(
35-
script: """
36-
set -x
37-
docker tag ${full_name} \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name}
38-
docker push \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name}
39-
""",
40-
label: 'Upload image to ECR'
41-
)
42-
}
52+
withEnv([
53+
"AWS_ACCOUNT_ID=${aws_account_id}",
54+
'AWS_DEFAULT_REGION={{ aws_default_region }}',
55+
"AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
56+
sh(
57+
script: '''
58+
set -eux
59+
aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
60+
''',
61+
label: 'Log in to ECR'
62+
)
63+
sh(
64+
script: """
65+
set -eux
66+
docker pull ${full_name}
67+
""",
68+
label: 'Pull image from ECR'
69+
)
4370
}
4471
} finally {
45-
sh(
46-
script: 'rm -f ~/.docker/config.json',
47-
label: 'Clean up login credentials'
48-
)
72+
withEnv([
73+
"AWS_ACCOUNT_ID=${aws_account_id}",
74+
'AWS_DEFAULT_REGION={{ aws_default_region }}',
75+
"AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
76+
sh(
77+
script: 'docker logout $AWS_ECR_REPO',
78+
label: 'Clean up login credentials'
79+
)
80+
}
4981
}
82+
}
83+
84+
def build_image(image_name) {
85+
hash = sh(
86+
returnStdout: true,
87+
script: 'git log -1 --format=\'%h\''
88+
).trim()
89+
def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}"
5090
sh(
51-
script: "docker rmi ${full_name}",
52-
label: 'Remove docker image'
91+
script: "${docker_build} ${image_name} --spec ${full_name}",
92+
label: 'Build docker image'
5393
)
94+
return ecr_push(full_name)
5495
}
5596

97+
5698
def build_docker_images() {
5799
stage('Docker Image Build') {
58-
// TODO in a follow up PR: Find ecr tag and use in subsequent builds
59-
parallel 'ci-lint': {
60-
node('CPU') {
61-
timeout(time: max_time, unit: 'MINUTES') {
62-
docker_init('none')
63-
init_git()
64-
build_image('ci_lint')
65-
}
66-
}
67-
}, 'ci-cpu': {
68-
node('CPU') {
69-
timeout(time: max_time, unit: 'MINUTES') {
70-
docker_init('none')
71-
init_git()
72-
build_image('ci_cpu')
100+
parallel(
101+
{% for image in images %}
102+
'{{ image.name }}': {
103+
node('{{ image.platform }}') {
104+
timeout(time: max_time, unit: 'MINUTES') {
105+
init_git()
106+
// We're purposefully not setting the built image here since they
107+
// are not yet being uploaded to tlcpack
108+
// {{ image.name }} = build_image('{{ image.name }}')
109+
build_image('{{ image.name }}')
110+
}
73111
}
74-
}
75-
}, 'ci-gpu': {
76-
node('GPU') {
77-
timeout(time: max_time, unit: 'MINUTES') {
78-
docker_init('none')
79-
init_git()
80-
build_image('ci_gpu')
81-
}
82-
}
83-
}, 'ci-qemu': {
84-
node('CPU') {
85-
timeout(time: max_time, unit: 'MINUTES') {
86-
docker_init('none')
87-
init_git()
88-
build_image('ci_qemu')
89-
}
90-
}
91-
}, 'ci-i386': {
92-
node('CPU') {
93-
timeout(time: max_time, unit: 'MINUTES') {
94-
docker_init('none')
95-
init_git()
96-
build_image('ci_i386')
97-
}
98-
}
99-
}, 'ci-arm': {
100-
node('ARM') {
101-
timeout(time: max_time, unit: 'MINUTES') {
102-
docker_init('none')
103-
init_git()
104-
build_image('ci_arm')
105-
}
106-
}
107-
}, 'ci-wasm': {
108-
node('CPU') {
109-
timeout(time: max_time, unit: 'MINUTES') {
110-
docker_init('none')
111-
init_git()
112-
build_image('ci_wasm')
113-
}
114-
}
115-
}, 'ci-hexagon': {
116-
node('CPU') {
117-
timeout(time: max_time, unit: 'MINUTES') {
118-
docker_init('none')
119-
init_git()
120-
build_image('ci_hexagon')
121-
}
122-
}
123-
}
124-
}
125-
// // TODO: Once we are able to use the built images, enable this step
126-
// // If the docker images changed, we need to run the image build before the lint
127-
// // can run since it requires a base docker image. Most of the time the images
128-
// // aren't build though so it's faster to use the same node that checks for
129-
// // docker changes to run the lint in the usual case.
130-
// stage('Sanity Check (re-run)') {
131-
// timeout(time: max_time, unit: 'MINUTES') {
132-
// node('CPU') {
133-
// ws({{ m.per_exec_ws('tvm/sanity') }}) {
134-
// init_git()
135-
// sh (
136-
// script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
137-
// label: 'Run lint',
138-
// )
139-
// }
140-
// }
141-
// }
142-
// }
143-
}
144-
145-
// Run make. First try to do an incremental make from a previous workspace in hope to
146-
// accelerate the compilation. If something is wrong, clean the workspace and then
147-
// build from scratch.
148-
def make(docker_type, path, make_flag) {
149-
timeout(time: max_time, unit: 'MINUTES') {
150-
try {
151-
cmake_build(docker_type, path, make_flag)
152-
// always run cpp test when build
153-
} catch (hudson.AbortException ae) {
154-
// script exited due to user abort, directly throw instead of retry
155-
if (ae.getMessage().contains('script returned exit code 143')) {
156-
throw ae
157-
}
158-
echo 'Incremental compilation failed. Fall back to build from scratch'
159-
sh (
160-
script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}",
161-
label: 'Clear old cmake workspace',
162-
)
163-
cmake_build(docker_type, path, make_flag)
164-
}
112+
},
113+
{% endfor %}
114+
)
165115
}
166116
}

‎jenkins/Jenkinsfile.j2‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) {
100100
{% set hexagon_api = ['build/hexagon_api_output',] %}
101101
s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
102102

103+
{% set aws_default_region = "us-west-2" %}
104+
{% set aws_ecr_url = "dkr.ecr." + aws_default_region + ".amazonaws.com" %}
105+
103106
// General note: Jenkins has limits on the size of a method (or top level code)
104107
// that are pretty strict, so most usage of groovy methods in these templates
105108
// are purely to satisfy the JVM

‎jenkins/Lint.groovy.j2‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ def lint() {
22
stage('Lint') {
33
parallel(
44
{% call m.sharded_lint_step(
5-
name='Lint',
6-
num_shards=2,
7-
node='CPU-SMALL',
8-
ws='tvm/lint',
9-
docker_image='ci_lint',
5+
name='Lint',
6+
num_shards=2,
7+
node='CPU-SMALL',
8+
ws='tvm/lint',
9+
docker_image='ci_lint',
1010
)
1111
%}
1212
sh (

0 commit comments

Comments
 (0)