Skip to content

Commit d53ae97

Browse files
driazatidriazati
committed
[ci] Split hexagon into 2 steps
This shards up the hexagon build since after #11016 it's the longest test step. This also drops the max runtime per stage down to 2 hours from 4 hours to make these kind of increases more obvious in the future.
1 parent 6846484 commit d53ae97

File tree

3 files changed

+159
-25
lines changed

3 files changed

+159
-25
lines changed

‎Jenkinsfile‎

Lines changed: 128 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
// 'python3 jenkins/generate.py'
4646
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
4747
// always rebased on main before merging:
48-
// Generated at 2022-04-22T12:59:15.071304
48+
// Generated at 2022-04-27T09:06:39.799194
4949

5050
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
5151
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. -->
@@ -92,7 +92,7 @@ upstream_revision = null
9292
docker_run = 'docker/bash.sh --env CI --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM'
9393
docker_build = 'docker/build.sh'
9494
// timeout in minutes
95-
max_time = 240
95+
max_time = 120
9696
rebuild_docker_images = false
9797

9898
def per_exec_ws(folder) {
@@ -196,7 +196,7 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) {
196196

197197
cancel_previous_build()
198198

199-
199+
def lint() {
200200
stage('Lint') {
201201
node('CPU') {
202202
timeout(time: max_time, unit: 'MINUTES') {
@@ -252,6 +252,12 @@ stage('Lint') {
252252
}
253253
}
254254
}
255+
}
256+
257+
// [note: method size]
258+
// This has to be extracted into a method due to JVM limitations on the size of
259+
// a method (so the code can't all be inlined)
260+
lint()
255261

256262
def build_image(image_name) {
257263
hash = sh(
@@ -465,6 +471,7 @@ def cpp_unittest(image) {
465471
)
466472
}
467473

474+
def build() {
468475
stage('Build') {
469476
environment {
470477
SKIP_SLOW_TESTS = "${skip_slow_tests}"
@@ -605,7 +612,12 @@ stage('Build') {
605612
}
606613
}
607614
}
615+
}
608616

617+
// [note: method size]
618+
build()
619+
620+
def test() {
609621
stage('Test') {
610622
environment {
611623
SKIP_SLOW_TESTS = "${skip_slow_tests}"
@@ -837,17 +849,52 @@ stage('Test') {
837849
Utils.markStageSkippedForConditional('python: i386 2 of 2')
838850
}
839851
},
840-
'test: Hexagon': {
852+
'test: Hexagon 1 of 4': {
841853
if (!skip_ci && is_docs_only_build != 1) {
842854
node('CPU') {
843855
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
844-
timeout(time: max_time, unit: 'MINUTES') {
845-
try {
846-
init_git()
847-
withEnv(['PLATFORM=hexagon'], {
856+
try {
857+
init_git()
858+
timeout(time: max_time, unit: 'MINUTES') {
859+
withEnv([
860+
'PLATFORM=hexagon',
861+
'TVM_NUM_SHARDS=4',
862+
'TVM_SHARD_INDEX=0'], {
863+
unpack_lib('hexagon', tvm_lib)
864+
ci_setup(ci_hexagon)
865+
cpp_unittest(ci_hexagon)
866+
sh (
867+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
868+
label: 'Build Hexagon API',
869+
)
870+
sh (
871+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
872+
label: 'Run Hexagon tests',
873+
)
874+
})
875+
}
876+
} finally {
877+
junit 'build/pytest-results/*.xml'
878+
}
879+
}
880+
}
881+
} else {
882+
Utils.markStageSkippedForConditional('test: Hexagon 1 of 4')
883+
}
884+
},
885+
'test: Hexagon 2 of 4': {
886+
if (!skip_ci && is_docs_only_build != 1) {
887+
node('CPU') {
888+
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
889+
try {
890+
init_git()
891+
timeout(time: max_time, unit: 'MINUTES') {
892+
withEnv([
893+
'PLATFORM=hexagon',
894+
'TVM_NUM_SHARDS=4',
895+
'TVM_SHARD_INDEX=1'], {
848896
unpack_lib('hexagon', tvm_lib)
849897
ci_setup(ci_hexagon)
850-
cpp_unittest(ci_hexagon)
851898
sh (
852899
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
853900
label: 'Build Hexagon API',
@@ -857,14 +904,78 @@ stage('Test') {
857904
label: 'Run Hexagon tests',
858905
)
859906
})
860-
} finally {
861-
junit 'build/pytest-results/*.xml'
862907
}
908+
} finally {
909+
junit 'build/pytest-results/*.xml'
863910
}
864911
}
865912
}
866913
} else {
867-
Utils.markStageSkippedForConditional('test: Hexagon')
914+
Utils.markStageSkippedForConditional('test: Hexagon 2 of 4')
915+
}
916+
},
917+
'test: Hexagon 3 of 4': {
918+
if (!skip_ci && is_docs_only_build != 1) {
919+
node('CPU') {
920+
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
921+
try {
922+
init_git()
923+
timeout(time: max_time, unit: 'MINUTES') {
924+
withEnv([
925+
'PLATFORM=hexagon',
926+
'TVM_NUM_SHARDS=4',
927+
'TVM_SHARD_INDEX=2'], {
928+
unpack_lib('hexagon', tvm_lib)
929+
ci_setup(ci_hexagon)
930+
sh (
931+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
932+
label: 'Build Hexagon API',
933+
)
934+
sh (
935+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
936+
label: 'Run Hexagon tests',
937+
)
938+
})
939+
}
940+
} finally {
941+
junit 'build/pytest-results/*.xml'
942+
}
943+
}
944+
}
945+
} else {
946+
Utils.markStageSkippedForConditional('test: Hexagon 3 of 4')
947+
}
948+
},
949+
'test: Hexagon 4 of 4': {
950+
if (!skip_ci && is_docs_only_build != 1) {
951+
node('CPU') {
952+
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
953+
try {
954+
init_git()
955+
timeout(time: max_time, unit: 'MINUTES') {
956+
withEnv([
957+
'PLATFORM=hexagon',
958+
'TVM_NUM_SHARDS=4',
959+
'TVM_SHARD_INDEX=3'], {
960+
unpack_lib('hexagon', tvm_lib)
961+
ci_setup(ci_hexagon)
962+
sh (
963+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
964+
label: 'Build Hexagon API',
965+
)
966+
sh (
967+
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
968+
label: 'Run Hexagon tests',
969+
)
970+
})
971+
}
972+
} finally {
973+
junit 'build/pytest-results/*.xml'
974+
}
975+
}
976+
}
977+
} else {
978+
Utils.markStageSkippedForConditional('test: Hexagon 4 of 4')
868979
}
869980
},
870981
'test: QEMU': {
@@ -1185,7 +1296,7 @@ stage('Test') {
11851296
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") {
11861297
init_git()
11871298
unpack_lib('gpu', tvm_multilib)
1188-
timeout(time: max_time, unit: 'MINUTES') {
1299+
timeout(time: 180, unit: 'MINUTES') {
11891300
ci_setup(ci_gpu)
11901301
sh (
11911302
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh",
@@ -1200,6 +1311,10 @@ stage('Test') {
12001311
},
12011312
)
12021313
}
1314+
}
1315+
1316+
// [note: method size]
1317+
test()
12031318

12041319
/*
12051320
stage('Build packages') {

‎jenkins/Jenkinsfile.j2‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ upstream_revision = null
8989
docker_run = 'docker/bash.sh --env CI --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM'
9090
docker_build = 'docker/build.sh'
9191
// timeout in minutes
92-
max_time = 240
92+
max_time = 120
9393
rebuild_docker_images = false
9494

9595
def per_exec_ws(folder) {
@@ -193,7 +193,7 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) {
193193

194194
cancel_previous_build()
195195

196-
196+
def lint() {
197197
stage('Lint') {
198198
node('CPU') {
199199
timeout(time: max_time, unit: 'MINUTES') {
@@ -249,6 +249,12 @@ stage('Lint') {
249249
}
250250
}
251251
}
252+
}
253+
254+
// [note: method size]
255+
// This has to be extracted into a method due to JVM limitations on the size of
256+
// a method (so the code can't all be inlined)
257+
lint()
252258

253259
def build_image(image_name) {
254260
hash = sh(
@@ -462,6 +468,7 @@ def cpp_unittest(image) {
462468
)
463469
}
464470

471+
def build() {
465472
stage('Build') {
466473
environment {
467474
SKIP_SLOW_TESTS = "${skip_slow_tests}"
@@ -602,13 +609,18 @@ stage('Build') {
602609
}
603610
}
604611
}
612+
}
613+
614+
// [note: method size]
615+
build()
605616

617+
def test() {
606618
stage('Test') {
607619
environment {
608620
SKIP_SLOW_TESTS = "${skip_slow_tests}"
609621
}
610622
parallel(
611-
{% call m.sharded_test_step(
623+
{% call(shard_index) m.sharded_test_step(
612624
name="unittest: GPU",
613625
num_shards=2,
614626
node="GPU",
@@ -634,7 +646,7 @@ stage('Test') {
634646
label: 'Run Python GPU integration tests',
635647
)
636648
{% endcall %}
637-
{% call m.sharded_test_step(
649+
{% call(shard_index) m.sharded_test_step(
638650
name="integration: CPU",
639651
node="CPU",
640652
num_shards=2,
@@ -663,7 +675,7 @@ stage('Test') {
663675
label: 'Run VTA tests in TSIM',
664676
)
665677
{% endcall %}
666-
{% call m.sharded_test_step(
678+
{% call(shard_index) m.sharded_test_step(
667679
name="python: i386",
668680
node="CPU",
669681
num_shards=2,
@@ -680,14 +692,17 @@ stage('Test') {
680692
)
681693
fsim_test(ci_i386)
682694
{% endcall %}
683-
{% call m.test_step(
695+
{% call(shard_index) m.sharded_test_step(
684696
name="test: Hexagon",
685697
node="CPU", ws="tvm/test-hexagon",
686698
platform="hexagon",
699+
num_shards=4,
687700
) %}
688701
unpack_lib('hexagon', tvm_lib)
689702
ci_setup(ci_hexagon)
690-
cpp_unittest(ci_hexagon)
703+
{% if shard_index == 1 %}
704+
cpp_unittest(ci_hexagon)
705+
{% endif %}
691706
sh (
692707
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh",
693708
label: 'Build Hexagon API',
@@ -736,7 +751,7 @@ stage('Test') {
736751
label: 'Run TOPI tests',
737752
)
738753
{% endcall %}
739-
{% call m.sharded_test_step(
754+
{% call(shard_index) m.sharded_test_step(
740755
name="integration: aarch64",
741756
num_shards=2,
742757
node="ARM", ws="tvm/ut-python-arm",
@@ -750,7 +765,7 @@ stage('Test') {
750765
label: 'Run CPU integration tests',
751766
)
752767
{% endcall %}
753-
{% call m.sharded_test_step(
768+
{% call(shard_index) m.sharded_test_step(
754769
name="topi: GPU",
755770
node="GPU",
756771
num_shards=2,
@@ -764,7 +779,7 @@ stage('Test') {
764779
label: 'Run TOPI tests',
765780
)
766781
{% endcall %}
767-
{% call m.sharded_test_step(
782+
{% call(shard_index) m.sharded_test_step(
768783
name="frontend: GPU", node="GPU",
769784
num_shards=3,
770785
ws="tvm/frontend-python-gpu",
@@ -809,7 +824,7 @@ stage('Test') {
809824
ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) {
810825
init_git()
811826
unpack_lib('gpu', tvm_multilib)
812-
timeout(time: max_time, unit: 'MINUTES') {
827+
timeout(time: 180, unit: 'MINUTES') {
813828
ci_setup(ci_gpu)
814829
sh (
815830
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh",
@@ -824,6 +839,10 @@ stage('Test') {
824839
},
825840
)
826841
}
842+
}
843+
844+
// [note: method size]
845+
test()
827846

828847
/*
829848
stage('Build packages') {

‎jenkins/macros.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
'PLATFORM={{ platform }}',
3333
'TVM_NUM_SHARDS={{ num_shards }}',
3434
'TVM_SHARD_INDEX={{ shard_index - 1 }}'], {
35-
{{ caller() | trim | indent(width=12) }}
35+
{{ caller(shard_index) | trim | indent(width=12) }}
3636
})
3737
}
3838
} finally {

0 commit comments

Comments
 (0)