diff --git a/ci/jenkins/data.py b/ci/jenkins/data.py
index 8cf762f16125..6eeb0bc7316c 100644
--- a/ci/jenkins/data.py
+++ b/ci/jenkins/data.py
@@ -21,8 +21,6 @@
 files_to_stash = {
     # Executables and build files needed to run c++ tests
     "cpptest": ["build/cpptest", "build/build.ninja", "build/CMakeFiles/rules.ninja"],
-    # Executables and build files needed to c runtime tests
-    "crttest": ["build/crttest"],
     # Folder for hexagon build
     "hexagon_api": [
         "build/hexagon_api_output",
@@ -33,12 +31,10 @@
     "tvm_runtime": ["build/libtvm_runtime.so", "build/config.cmake"],
     # compiler files
     "tvm_lib": ["build/libtvm.so", "build/libtvm_runtime.so", "build/config.cmake"],
-    # compiler files and fsim
-    "tvm_multilib": [
-        "build/libtvm.so",
-        "build/libvta_fsim.so",
-        "build/libtvm_runtime.so",
-        "build/config.cmake",
+    # gpu related compiler files
+    "tvm_lib_gpu_extra": [
+        "build/3rdparty/libflash_attn/src/libflash_attn.so",
+        "build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
     ],
 }
 
diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy
index c33be22a6f9c..95b1bb24a68f 100644
--- a/ci/jenkins/generated/arm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/arm_jenkinsfile.groovy
@@ -60,7 +60,7 @@
 // 'python3 jenkins/generate.py'
 // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
 // always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.226391
+// Generated at 2025-02-07T09:50:07.174590
 
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 // These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -514,63 +514,70 @@ try {
 } catch(Exception ex) {
   prepare('CPU-SMALL')
 }
-def build(node_type) {
-  stage('Build') {
-    if (!skip_ci && is_docs_only_build != 1) {
-      node(node_type) {
-        ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") {
-          init_git()
-          docker_init(ci_arm)
-          timeout(time: max_time, unit: 'MINUTES') {
-
-            withEnv([
-              'PLATFORM=arm',
-              ], {
-              sh (
+def run_build(node_type) {
+  if (!skip_ci && is_docs_only_build != 1) {
+    echo 'Begin running node_type ' + node_type
+    node(node_type) {
+      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") {
+        init_git()
+        docker_init(ci_arm)
+        timeout(time: max_time, unit: 'MINUTES') {
+
+          withEnv([
+            'PLATFORM=arm',
+            ], {
+            sh (
           script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build",
           label: 'Create ARM cmake config',
         )
         cmake_build(ci_arm, 'build', '-j4')
         make_cpp_tests(ci_arm, 'build')
         sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/arm --items build/libtvm.so build/libvta_fsim.so build/libtvm_runtime.so build/config.cmake build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/crttest build/build.ninja build/microtvm_template_projects",
+            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/arm --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/cpptest build/build.ninja build/CMakeFiles/rules.ninja",
             label: 'Upload artifacts to S3',
           )
-            })
-          }
+          })
         }
       }
-    } else {
-      Utils.markStageSkippedForConditional('BUILD: arm')
     }
+    echo 'End running node_type ' + node_type
+  } else {
+    Utils.markStageSkippedForConditional('BUILD: arm')
   }
 }
-try {
-    build('ARM-GRAVITON3-SPOT')
-} catch (Exception ex) {
-    build('ARM-GRAVITON3')
+def build() {
+  stage('Build') {
+    try {
+        run_build('ARM-GRAVITON3-SPOT')
+    } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        run_build('ARM-GRAVITON3')
+    }
+  }
 }
+build()
 
 
 
-def shard_run_integration_aarch64_1_of_4(node_type='ARM-GRAVITON3-SPOT', on_demand=false) {
+def shard_run_integration_aarch64_1_of_4(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'ARM-GRAVITON3'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
-        try {
-          init_git()
-          docker_init(ci_arm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=arm',
-              'TEST_STEP_NAME=integration: aarch64',
-              'TVM_NUM_SHARDS=4',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_arm)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=arm',
+            'TEST_STEP_NAME=integration: aarch64',
+            'TVM_NUM_SHARDS=4',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
                   label: 'Download artifacts from S3',
                 )
@@ -581,45 +588,43 @@ def shard_run_integration_aarch64_1_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
                 script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh",
                 label: 'Run CPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_aarch64 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('integration: aarch64 1 of 4')
   }
 }
 
-def shard_run_integration_aarch64_2_of_4(node_type='ARM-GRAVITON3-SPOT', on_demand=false) {
+def shard_run_integration_aarch64_2_of_4(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'ARM-GRAVITON3'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
-        try {
-          init_git()
-          docker_init(ci_arm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=arm',
-              'TEST_STEP_NAME=integration: aarch64',
-              'TVM_NUM_SHARDS=4',
-              'TVM_SHARD_INDEX=1',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_arm)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=arm',
+            'TEST_STEP_NAME=integration: aarch64',
+            'TVM_NUM_SHARDS=4',
+            'TVM_SHARD_INDEX=1',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
                   label: 'Download artifacts from S3',
                 )
@@ -630,45 +635,43 @@ def shard_run_integration_aarch64_2_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
                 script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh",
                 label: 'Run CPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_aarch64 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('integration: aarch64 2 of 4')
   }
 }
 
-def shard_run_integration_aarch64_3_of_4(node_type='ARM-GRAVITON3-SPOT', on_demand=false) {
+def shard_run_integration_aarch64_3_of_4(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'ARM-GRAVITON3'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
-        try {
-          init_git()
-          docker_init(ci_arm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=arm',
-              'TEST_STEP_NAME=integration: aarch64',
-              'TVM_NUM_SHARDS=4',
-              'TVM_SHARD_INDEX=2',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_arm)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=arm',
+            'TEST_STEP_NAME=integration: aarch64',
+            'TVM_NUM_SHARDS=4',
+            'TVM_SHARD_INDEX=2',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
                   label: 'Download artifacts from S3',
                 )
@@ -679,45 +682,43 @@ def shard_run_integration_aarch64_3_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
                 script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh",
                 label: 'Run CPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_aarch64 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('integration: aarch64 3 of 4')
   }
 }
 
-def shard_run_integration_aarch64_4_of_4(node_type='ARM-GRAVITON3-SPOT', on_demand=false) {
+def shard_run_integration_aarch64_4_of_4(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'ARM-GRAVITON3'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
-        try {
-          init_git()
-          docker_init(ci_arm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=arm',
-              'TEST_STEP_NAME=integration: aarch64',
-              'TVM_NUM_SHARDS=4',
-              'TVM_SHARD_INDEX=3',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_arm)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=arm',
+            'TEST_STEP_NAME=integration: aarch64',
+            'TVM_NUM_SHARDS=4',
+            'TVM_SHARD_INDEX=3',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
                   label: 'Download artifacts from S3',
                 )
@@ -728,22 +729,22 @@ def shard_run_integration_aarch64_4_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
                 script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh",
                 label: 'Run CPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_aarch64 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('integration: aarch64 4 of 4')
   }
@@ -751,24 +752,22 @@ def shard_run_integration_aarch64_4_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
 
 
 
-def shard_run_topi_aarch64_1_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=false) {
+def shard_run_topi_aarch64_1_of_2(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'ARM-GRAVITON3'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
-        try {
-          init_git()
-          docker_init(ci_arm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=arm',
-              'TEST_STEP_NAME=topi: aarch64',
-              'TVM_NUM_SHARDS=2',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_arm)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=arm',
+            'TEST_STEP_NAME=topi: aarch64',
+            'TVM_NUM_SHARDS=2',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
                   label: 'Download artifacts from S3',
                 )
@@ -783,45 +782,43 @@ def shard_run_topi_aarch64_1_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=fals
                 script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh",
                 label: 'Run TOPI tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_aarch64 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('topi: aarch64 1 of 2')
   }
 }
 
-def shard_run_topi_aarch64_2_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=false) {
+def shard_run_topi_aarch64_2_of_2(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'ARM-GRAVITON3'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
-        try {
-          init_git()
-          docker_init(ci_arm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=arm',
-              'TEST_STEP_NAME=topi: aarch64',
-              'TVM_NUM_SHARDS=2',
-              'TVM_SHARD_INDEX=1',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_arm)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=arm',
+            'TEST_STEP_NAME=topi: aarch64',
+            'TVM_NUM_SHARDS=2',
+            'TVM_SHARD_INDEX=1',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
                   label: 'Download artifacts from S3',
                 )
@@ -835,22 +832,22 @@ def shard_run_topi_aarch64_2_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=fals
                 script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh",
                 label: 'Run TOPI tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_aarch64 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('topi: aarch64 2 of 2')
   }
@@ -858,24 +855,22 @@ def shard_run_topi_aarch64_2_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=fals
 
 
 
-def shard_run_frontend_aarch64_1_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=false) {
+def shard_run_frontend_aarch64_1_of_2(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'ARM-GRAVITON3'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") {
-        try {
-          init_git()
-          docker_init(ci_arm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=arm',
-              'TEST_STEP_NAME=frontend: aarch64',
-              'TVM_NUM_SHARDS=2',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_arm)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=arm',
+            'TEST_STEP_NAME=frontend: aarch64',
+            'TVM_NUM_SHARDS=2',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
                   label: 'Download artifacts from S3',
                 )
@@ -885,45 +880,43 @@ def shard_run_frontend_aarch64_1_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=
                 script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh",
                 label: 'Run Python frontend tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_aarch64 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('frontend: aarch64 1 of 2')
   }
 }
 
-def shard_run_frontend_aarch64_2_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=false) {
+def shard_run_frontend_aarch64_2_of_2(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'ARM-GRAVITON3'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") {
-        try {
-          init_git()
-          docker_init(ci_arm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=arm',
-              'TEST_STEP_NAME=frontend: aarch64',
-              'TVM_NUM_SHARDS=2',
-              'TVM_SHARD_INDEX=1',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_arm)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=arm',
+            'TEST_STEP_NAME=frontend: aarch64',
+            'TVM_NUM_SHARDS=2',
+            'TVM_SHARD_INDEX=1',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
                   label: 'Download artifacts from S3',
                 )
@@ -933,22 +926,22 @@ def shard_run_frontend_aarch64_2_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=
                 script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh",
                 label: 'Run Python frontend tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_aarch64 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('frontend: aarch64 2 of 2')
   }
@@ -963,58 +956,90 @@ def test() {
     parallel(
     'integration: aarch64 1 of 4': {
       try {
-      shard_run_integration_aarch64_1_of_4()
-      } catch (Exception ex) {
-        shard_run_integration_aarch64_1_of_4(on_demand = true)
+      shard_run_integration_aarch64_1_of_4('ARM-GRAVITON3-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_integration_aarch64_1_of_4('ARM-GRAVITON3')
       }
     },
     'integration: aarch64 2 of 4': {
       try {
-      shard_run_integration_aarch64_2_of_4()
-      } catch (Exception ex) {
-        shard_run_integration_aarch64_2_of_4(on_demand = true)
+      shard_run_integration_aarch64_2_of_4('ARM-GRAVITON3-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_integration_aarch64_2_of_4('ARM-GRAVITON3')
       }
     },
     'integration: aarch64 3 of 4': {
       try {
-      shard_run_integration_aarch64_3_of_4()
-      } catch (Exception ex) {
-        shard_run_integration_aarch64_3_of_4(on_demand = true)
+      shard_run_integration_aarch64_3_of_4('ARM-GRAVITON3-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_integration_aarch64_3_of_4('ARM-GRAVITON3')
       }
     },
     'integration: aarch64 4 of 4': {
       try {
-      shard_run_integration_aarch64_4_of_4()
-      } catch (Exception ex) {
-        shard_run_integration_aarch64_4_of_4(on_demand = true)
+      shard_run_integration_aarch64_4_of_4('ARM-GRAVITON3-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_integration_aarch64_4_of_4('ARM-GRAVITON3')
       }
     },
     'topi: aarch64 1 of 2': {
       try {
-      shard_run_topi_aarch64_1_of_2()
-      } catch (Exception ex) {
-        shard_run_topi_aarch64_1_of_2(on_demand = true)
+      shard_run_topi_aarch64_1_of_2('ARM-GRAVITON3-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_topi_aarch64_1_of_2('ARM-GRAVITON3')
       }
     },
     'topi: aarch64 2 of 2': {
       try {
-      shard_run_topi_aarch64_2_of_2()
-      } catch (Exception ex) {
-        shard_run_topi_aarch64_2_of_2(on_demand = true)
+      shard_run_topi_aarch64_2_of_2('ARM-GRAVITON3-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_topi_aarch64_2_of_2('ARM-GRAVITON3')
       }
     },
     'frontend: aarch64 1 of 2': {
       try {
-      shard_run_frontend_aarch64_1_of_2()
-      } catch (Exception ex) {
-        shard_run_frontend_aarch64_1_of_2(on_demand = true)
+      shard_run_frontend_aarch64_1_of_2('ARM-GRAVITON3-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_frontend_aarch64_1_of_2('ARM-GRAVITON3')
       }
     },
     'frontend: aarch64 2 of 2': {
       try {
-      shard_run_frontend_aarch64_2_of_2()
-      } catch (Exception ex) {
-        shard_run_frontend_aarch64_2_of_2(on_demand = true)
+      shard_run_frontend_aarch64_2_of_2('ARM-GRAVITON3-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_frontend_aarch64_2_of_2('ARM-GRAVITON3')
       }
     },
     )
diff --git a/ci/jenkins/generated/cortexm_jenkinsfile.groovy b/ci/jenkins/generated/cortexm_jenkinsfile.groovy
deleted file mode 100644
index 6bd4b2e092b8..000000000000
--- a/ci/jenkins/generated/cortexm_jenkinsfile.groovy
+++ /dev/null
@@ -1,1231 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// -*- mode: groovy -*-
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-// Docker env used for testing
-// Different image may have different version tag
-// because some of them are more stable than anoter.
-//
-// Docker images are maintained by PMC, cached in dockerhub
-// and remains relatively stable over the time.
-// Flow for upgrading docker env(need commiter)
-//
-// - Send PR to upgrade build script in the repo
-// - Build the new docker image
-// - Tag the docker image with a new version and push to a binary cache.
-// - Update the version in the Jenkinsfile, send a PR
-// - Fix any issues wrt to the new image version in the PR
-// - Merge the PR and now we are in new version
-// - Tag the new version as the lates
-// - Periodically cleanup the old versions on local workers
-//
-
-// ============================= IMPORTANT NOTE =============================
-// This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
-// Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
-// 'python3 jenkins/generate.py'
-// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
-// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.121865
-
-import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-// These are set at runtime from data in ci/jenkins/docker-images.yml, update
-// image tags in that file
-ci_lint = ''
-ci_gpu = ''
-ci_cpu = ''
-ci_minimal = ''
-ci_wasm = ''
-ci_i386 = ''
-ci_cortexm = ''
-ci_arm = ''
-ci_hexagon = ''
-ci_riscv = ''
-
-// Parameters to allow overriding (in Jenkins UI), the images
-// to be used by a given build. When provided, they take precedence
-// over default values above.
-properties([
-  parameters([
-    string(name: 'ci_arm_param', defaultValue: ''),
-    string(name: 'ci_cortexm_param', defaultValue: ''),
-    string(name: 'ci_cpu_param', defaultValue: ''),
-    string(name: 'ci_gpu_param', defaultValue: ''),
-    string(name: 'ci_hexagon_param', defaultValue: ''),
-    string(name: 'ci_i386_param', defaultValue: ''),
-    string(name: 'ci_lint_param', defaultValue: ''),
-    string(name: 'ci_minimal_param', defaultValue: ''),
-    string(name: 'ci_riscv_param', defaultValue: ''),
-    string(name: 'ci_wasm_param', defaultValue: ''),
-  ])
-])
-
-// Placeholders for newly built Docker image names (if rebuild_docker_images
-// is used)
-  built_ci_arm = null;
-  built_ci_cortexm = null;
-  built_ci_cpu = null;
-  built_ci_gpu = null;
-  built_ci_hexagon = null;
-  built_ci_i386 = null;
-  built_ci_lint = null;
-  built_ci_minimal = null;
-  built_ci_riscv = null;
-  built_ci_wasm = null;
-
-// Global variable assigned during Sanity Check that holds the sha1 which should be
-// merged into the PR in all branches.
-upstream_revision = null
-
-// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
-docker_build = 'docker/build.sh'
-// timeout in minutes
-max_time = 180
-rebuild_docker_images = false
-
-s3_bucket = 'tvm-jenkins-artifacts-prod'
-s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
-// Jenkins script root directory
-jenkins_scripts_root = "ci/scripts/jenkins"
-
-
-// General note: Jenkins has limits on the size of a method (or top level code)
-// that are pretty strict, so most usage of groovy methods in these templates
-// are purely to satisfy the JVM
-def per_exec_ws(folder) {
-  return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
-}
-
-// initialize source codes
-def init_git() {
-  retry(5) {
-    checkout scm
-  }
-
-  // Add more info about job node
-  sh (
-    script: './tests/scripts/task_show_node_info.sh',
-    label: 'Show executor node info',
-  )
-
-  // Determine merge commit to use for all stages
-  if (env.BRANCH_NAME == 'main') {
-    // Only set upstream_revision to HEAD and skip merging to avoid a race with another commit merged to main.
-    update_upstream_revision("HEAD")
-  } else {
-    // This is PR branch so merge with latest main.
-    merge_with_main()
-  }
-
-  sh(
-    script: """
-      set -eux
-      . ${jenkins_scripts_root}/retry.sh
-      retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
-    """,
-    label: 'Update git submodules',
-  )
-  checkout_trusted_files()
-}
-
-def update_upstream_revision(git_ref) {
-  if (upstream_revision == null) {
-    upstream_revision = sh(
-      script: "git log -1 ${git_ref} --format=\'%H\'",
-      label: 'Determine upstream revision',
-      returnStdout: true,
-    ).trim()
-  }
-}
-
-def merge_with_main() {
-  sh (
-    script: 'git fetch origin main',
-    label: 'Fetch upstream',
-  )
-  update_upstream_revision("FETCH_HEAD")
-  sh (
-    script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
-    label: 'Merge to origin/main'
-  )
-}
-
-def docker_init(image) {
-  // Clear out all Docker images that aren't going to be used
-  sh(
-    script: """
-    set -eux
-    docker image ls --all
-    IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}}  {{.ID}}')
-
-    echo -e "Found images:\\n\$IMAGES"
-    echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
-    docker image ls --all
-    """,
-    label: 'Clean old Docker images',
-  )
-
-  if (image.contains("amazonaws.com")) {
-    // If this string is in the image name it's from ECR and needs to be pulled
-    // with the right credentials
-    ecr_pull(image)
-  } else {
-    sh(
-      script: """
-      set -eux
-      . ${jenkins_scripts_root}/retry.sh
-      retry 5 docker pull ${image}
-      """,
-      label: 'Pull docker image',
-    )
-  }
-}
-
-def ecr_pull(full_name) {
-  aws_account_id = sh(
-    returnStdout: true,
-    script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
-    label: 'Get AWS ID'
-  ).trim()
-
-  try {
-    withEnv([
-      "AWS_ACCOUNT_ID=${aws_account_id}",
-      'AWS_DEFAULT_REGION=us-west-2',
-      "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
-      sh(
-        script: '''
-          set -eux
-          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
-        ''',
-        label: 'Log in to ECR'
-      )
-      sh(
-        script: """
-          set -eux
-          . ${jenkins_scripts_root}/retry.sh
-          retry 5 docker pull ${full_name}
-        """,
-        label: 'Pull image from ECR'
-      )
-    }
-  } finally {
-    withEnv([
-      "AWS_ACCOUNT_ID=${aws_account_id}",
-      'AWS_DEFAULT_REGION=us-west-2',
-      "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
-      sh(
-        script: 'docker logout $AWS_ECR_REPO',
-        label: 'Clean up login credentials'
-      )
-    }
-  }
-}
-
-def should_skip_slow_tests(pr_number) {
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-  )]) {
-    // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
-    result = sh (
-      returnStatus: true,
-      script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
-      label: 'Check if CI should run slow tests',
-    )
-  }
-  return result == 0
-}
-
-def cancel_previous_build() {
-  // cancel previous build if it is not on main.
-  if (env.BRANCH_NAME != 'main') {
-    def buildNumber = env.BUILD_NUMBER as int
-    // Milestone API allows us to cancel previous build
-    // with the same milestone number
-    if (buildNumber > 1) milestone(buildNumber - 1)
-    milestone(buildNumber)
-  }
-}
-
-def checkout_trusted_files() {
-  // trust everything from branch builds
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    return;
-  }
-
-  // trust peoople listed in CONTRIBUTING.md
-  grep_code = sh(
-    returnStatus: true,
-    script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
-    label: 'Check if change is from a contributor',
-  )
-
-  if (grep_code == 1) {
-    // Any scripts that run on the bare host and not inside a Docker container
-    // (especially those that access secrets) should be checked out here so
-    // only trusted versions are used in CI
-    sh(
-      script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
-      label: 'Check out trusted files',
-    )
-  }
-}
-
-def should_skip_ci(pr_number) {
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    // never skip CI on build sourced from a branch
-    return false
-  }
-  glob_skip_ci_code = sh (
-    returnStatus: true,
-    script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
-    label: 'Check if CI should be skipped due to changed files',
-  )
-  if (glob_skip_ci_code == 0) {
-    return true
-  }
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-    )]) {
-    // Exit code of 1 means run full CI (or the script had an error, so run
-    // full CI just in case). Exit code of 0 means skip CI.
-    git_skip_ci_code = sh (
-      returnStatus: true,
-      script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
-      label: 'Check if CI should be skipped',
-    )
-  }
-  return git_skip_ci_code == 0
-}
-
-def check_pr(pr_number) {
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    // never skip CI on build sourced from a branch
-    return false
-  }
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-    )]) {
-    sh (
-      script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}",
-      label: 'Check PR title and body',
-    )
-  }
-
-}
-
-def prepare(node_type) {
-  stage('Prepare') {
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
-        init_git()
-
-        check_pr(env.CHANGE_ID)
-
-        if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
-          sh(
-            script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
-            label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
-          )
-          // Pull image names from the results of should_rebuild_docker.py
-          ci_arm = sh(
-            script: "cat .docker-image-names/ci_arm",
-            label: "Find docker image name for ci_arm",
-            returnStdout: true,
-          ).trim()
-          ci_cortexm = sh(
-            script: "cat .docker-image-names/ci_cortexm",
-            label: "Find docker image name for ci_cortexm",
-            returnStdout: true,
-          ).trim()
-          ci_cpu = sh(
-            script: "cat .docker-image-names/ci_cpu",
-            label: "Find docker image name for ci_cpu",
-            returnStdout: true,
-          ).trim()
-          ci_gpu = sh(
-            script: "cat .docker-image-names/ci_gpu",
-            label: "Find docker image name for ci_gpu",
-            returnStdout: true,
-          ).trim()
-          ci_hexagon = sh(
-            script: "cat .docker-image-names/ci_hexagon",
-            label: "Find docker image name for ci_hexagon",
-            returnStdout: true,
-          ).trim()
-          ci_i386 = sh(
-            script: "cat .docker-image-names/ci_i386",
-            label: "Find docker image name for ci_i386",
-            returnStdout: true,
-          ).trim()
-          ci_lint = sh(
-            script: "cat .docker-image-names/ci_lint",
-            label: "Find docker image name for ci_lint",
-            returnStdout: true,
-          ).trim()
-          ci_minimal = sh(
-            script: "cat .docker-image-names/ci_minimal",
-            label: "Find docker image name for ci_minimal",
-            returnStdout: true,
-          ).trim()
-          ci_riscv = sh(
-            script: "cat .docker-image-names/ci_riscv",
-            label: "Find docker image name for ci_riscv",
-            returnStdout: true,
-          ).trim()
-          ci_wasm = sh(
-            script: "cat .docker-image-names/ci_wasm",
-            label: "Find docker image name for ci_wasm",
-            returnStdout: true,
-          ).trim()
-        }
-
-        ci_arm = params.ci_arm_param ?: ci_arm
-        ci_cortexm = params.ci_cortexm_param ?: ci_cortexm
-        ci_cpu = params.ci_cpu_param ?: ci_cpu
-        ci_gpu = params.ci_gpu_param ?: ci_gpu
-        ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
-        ci_i386 = params.ci_i386_param ?: ci_i386
-        ci_lint = params.ci_lint_param ?: ci_lint
-        ci_minimal = params.ci_minimal_param ?: ci_minimal
-        ci_riscv = params.ci_riscv_param ?: ci_riscv
-        ci_wasm = params.ci_wasm_param ?: ci_wasm
-
-        sh (script: """
-          echo "Docker images being used in this build:"
-          echo " ci_arm = ${ci_arm}"
-          echo " ci_cortexm = ${ci_cortexm}"
-          echo " ci_cpu = ${ci_cpu}"
-          echo " ci_gpu = ${ci_gpu}"
-          echo " ci_hexagon = ${ci_hexagon}"
-          echo " ci_i386 = ${ci_i386}"
-          echo " ci_lint = ${ci_lint}"
-          echo " ci_minimal = ${ci_minimal}"
-          echo " ci_riscv = ${ci_riscv}"
-          echo " ci_wasm = ${ci_wasm}"
-        """, label: 'Docker image names')
-
-        is_docs_only_build = sh (
-          returnStatus: true,
-          script: "./${jenkins_scripts_root}/git_change_docs.sh",
-          label: 'Check for docs only changes',
-        )
-        skip_ci = should_skip_ci(env.CHANGE_ID)
-        skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
-        rebuild_docker_images = sh (
-          returnStatus: true,
-          script: "./${jenkins_scripts_root}/git_change_docker.sh",
-          label: 'Check for any docker changes',
-        )
-
-        if (skip_ci) {
-          // Don't rebuild when skipping CI
-          rebuild_docker_images = false
-        }
-      }
-    }
-  }
-}
-def ci_setup(image) {
-  sh (
-    script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
-    label: 'Clean up old workspace',
-  )
-}
-
-def python_unittest(image) {
-  sh (
-    script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
-    label: 'Run Python unit tests',
-  )
-}
-
-def make_cpp_tests(image, build_dir) {
-  sh (
-    script: """
-      set -eux
-      ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
-        --sccache-bucket tvm-sccache-prod \
-        --sccache-region us-west-2 \
-        --cmake-target cpptest \
-        --build-dir ${build_dir}
-      """,
-    label: 'Make C++ tests',
-  )
-}
-
-def cmake_build(image, path, make_flag) {
-  sh (
-    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir ${path}",
-    label: 'Run cmake build',
-  )
-}
-def cpp_unittest(image) {
-  sh (
-    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
-    label: 'Run C++ tests',
-  )
-}
-
-cancel_previous_build()
-
-try {
-    prepare('CPU-SMALL-SPOT')
-} catch(Exception ex) {
-  prepare('CPU-SMALL')
-}
-def build(node_type) {
-  stage('Build') {
-    if (!skip_ci && is_docs_only_build != 1) {
-      node(node_type) {
-        ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cortexm") {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-
-            withEnv([
-              'PLATFORM=cortexm',
-              ], {
-              sh (
-          script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_config_build_cortexm.sh build",
-          label: 'Create Cortex-M cmake config',
-        )
-        cmake_build(ci_cortexm, 'build', '-j2')
-        make_cpp_tests(ci_cortexm, 'build')
-        sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/crttest build/build.ninja build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/microtvm_template_projects",
-            label: 'Upload artifacts to S3',
-          )
-            })
-          }
-        }
-      }
-    } else {
-      Utils.markStageSkippedForConditional('BUILD: Cortex-M')
-    }
-  }
-}
-try {
-    build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
-    build('CPU-SMALL')
-}
-
-
-
-def shard_run_test_Cortex_M_1_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              cpp_unittest(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_demo_microtvm.sh",
-                label: 'Run microTVM demos',
-              )
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 1 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_2_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=1',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 2 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_3_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=2',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 3 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_4_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=3',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 4 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_5_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=4',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 5 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_6_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=5',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 6 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_7_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=6',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 7 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_8_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=7',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 8 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_9_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=8',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 9 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_10_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=9',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 10 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_11_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=10',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 11 of 12')
-  }
-}
-
-def shard_run_test_Cortex_M_12_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
-        try {
-          init_git()
-          docker_init(ci_cortexm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cortexm',
-              'TEST_STEP_NAME=test: Cortex-M',
-              'TVM_NUM_SHARDS=12',
-              'TVM_SHARD_INDEX=11',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_cortexm)
-              sh (
-                script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: Cortex-M 12 of 12')
-  }
-}
-
-
-def test() {
-  stage('Test') {
-    environment {
-      SKIP_SLOW_TESTS = "${skip_slow_tests}"
-    }
-    parallel(
-    'test: Cortex-M 1 of 12': {
-      try {
-      shard_run_test_Cortex_M_1_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_1_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 2 of 12': {
-      try {
-      shard_run_test_Cortex_M_2_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_2_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 3 of 12': {
-      try {
-      shard_run_test_Cortex_M_3_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_3_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 4 of 12': {
-      try {
-      shard_run_test_Cortex_M_4_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_4_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 5 of 12': {
-      try {
-      shard_run_test_Cortex_M_5_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_5_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 6 of 12': {
-      try {
-      shard_run_test_Cortex_M_6_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_6_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 7 of 12': {
-      try {
-      shard_run_test_Cortex_M_7_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_7_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 8 of 12': {
-      try {
-      shard_run_test_Cortex_M_8_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_8_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 9 of 12': {
-      try {
-      shard_run_test_Cortex_M_9_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_9_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 10 of 12': {
-      try {
-      shard_run_test_Cortex_M_10_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_10_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 11 of 12': {
-      try {
-      shard_run_test_Cortex_M_11_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_11_of_12(on_demand = true)
-      }
-    },
-    'test: Cortex-M 12 of 12': {
-      try {
-      shard_run_test_Cortex_M_12_of_12()
-      } catch (Exception ex) {
-        shard_run_test_Cortex_M_12_of_12(on_demand = true)
-      }
-    },
-    )
-  }
-}
-test()
diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy
index 04ab19f40414..2b2a17d95170 100644
--- a/ci/jenkins/generated/cpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
 // 'python3 jenkins/generate.py'
 // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
 // always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.103852
+// Generated at 2025-02-07T09:50:07.191497
 
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 // These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -514,26 +514,26 @@ try {
 } catch(Exception ex) {
   prepare('CPU-SMALL')
 }
-def build(node_type) {
-  stage('Build') {
-    if (!skip_ci && is_docs_only_build != 1) {
-      node(node_type) {
-        ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") {
-          init_git()
-          docker_init(ci_cpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-
-            withEnv([
-              'PLATFORM=cpu',
-              ], {
-              sh (
+def run_build(node_type) {
+  if (!skip_ci && is_docs_only_build != 1) {
+    echo 'Begin running node_type ' + node_type
+    node(node_type) {
+      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") {
+        init_git()
+        docker_init(ci_cpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+
+          withEnv([
+            'PLATFORM=cpu',
+            ], {
+            sh (
           script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build",
           label: 'Create CPU cmake config',
         )
         cmake_build(ci_cpu, 'build', '-j2')
         make_cpp_tests(ci_cpu, 'build')
         sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu --items build/libvta_tsim.so build/libtvm.so build/libvta_fsim.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/crttest build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/build.ninja build/microtvm_template_projects",
+            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/cpptest build/build.ninja build/CMakeFiles/rules.ninja",
             label: 'Upload artifacts to S3',
           )
 
@@ -541,41 +541,48 @@ def build(node_type) {
         // sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh"
         // TODO(@jroesch): need to resolve CI issue will turn back on in follow up patch
         sh (script: "${docker_run} ${ci_cpu} ./tests/scripts/task_rust.sh", label: 'Rust build and test')
-            })
-          }
+          })
         }
       }
-    } else {
-      Utils.markStageSkippedForConditional('BUILD: CPU')
     }
+    echo 'End running node_type ' + node_type
+  } else {
+    Utils.markStageSkippedForConditional('BUILD: CPU')
   }
 }
-try {
-    build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
-    build('CPU-SMALL')
+def build() {
+  stage('Build') {
+    try {
+        run_build('CPU-SMALL-SPOT')
+    } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        run_build('CPU-SMALL')
+    }
+  }
 }
+build()
 
 
 
-def shard_run_integration_CPU_1_of_4(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_integration_CPU_1_of_4(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
-        try {
-          init_git()
-          docker_init(ci_cpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cpu',
-              'TEST_STEP_NAME=integration: CPU',
-              'TVM_NUM_SHARDS=4',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_cpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=cpu',
+            'TEST_STEP_NAME=integration: CPU',
+            'TVM_NUM_SHARDS=4',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
                   label: 'Download artifacts from S3',
                 )
@@ -585,45 +592,43 @@ def shard_run_integration_CPU_1_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
                 script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
                 label: 'Run CPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_CPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('integration: CPU 1 of 4')
   }
 }
 
-def shard_run_integration_CPU_2_of_4(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_integration_CPU_2_of_4(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
-        try {
-          init_git()
-          docker_init(ci_cpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cpu',
-              'TEST_STEP_NAME=integration: CPU',
-              'TVM_NUM_SHARDS=4',
-              'TVM_SHARD_INDEX=1',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_cpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=cpu',
+            'TEST_STEP_NAME=integration: CPU',
+            'TVM_NUM_SHARDS=4',
+            'TVM_SHARD_INDEX=1',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
                   label: 'Download artifacts from S3',
                 )
@@ -633,45 +638,43 @@ def shard_run_integration_CPU_2_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
                 script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
                 label: 'Run CPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_CPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('integration: CPU 2 of 4')
   }
 }
 
-def shard_run_integration_CPU_3_of_4(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_integration_CPU_3_of_4(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
-        try {
-          init_git()
-          docker_init(ci_cpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cpu',
-              'TEST_STEP_NAME=integration: CPU',
-              'TVM_NUM_SHARDS=4',
-              'TVM_SHARD_INDEX=2',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_cpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=cpu',
+            'TEST_STEP_NAME=integration: CPU',
+            'TVM_NUM_SHARDS=4',
+            'TVM_SHARD_INDEX=2',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
                   label: 'Download artifacts from S3',
                 )
@@ -681,45 +684,43 @@ def shard_run_integration_CPU_3_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
                 script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
                 label: 'Run CPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_CPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('integration: CPU 3 of 4')
   }
 }
 
-def shard_run_integration_CPU_4_of_4(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_integration_CPU_4_of_4(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
-        try {
-          init_git()
-          docker_init(ci_cpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cpu',
-              'TEST_STEP_NAME=integration: CPU',
-              'TVM_NUM_SHARDS=4',
-              'TVM_SHARD_INDEX=3',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_cpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=cpu',
+            'TEST_STEP_NAME=integration: CPU',
+            'TVM_NUM_SHARDS=4',
+            'TVM_SHARD_INDEX=3',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
                   label: 'Download artifacts from S3',
                 )
@@ -729,22 +730,22 @@ def shard_run_integration_CPU_4_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
                 script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
                 label: 'Run CPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_CPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('integration: CPU 4 of 4')
   }
@@ -752,24 +753,22 @@ def shard_run_integration_CPU_4_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
 
 
 
-def shard_run_unittest_CPU_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_unittest_CPU_1_of_1(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") {
-        try {
-          init_git()
-          docker_init(ci_cpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cpu',
-              'TEST_STEP_NAME=unittest: CPU',
-              'TVM_NUM_SHARDS=1',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_cpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=cpu',
+            'TEST_STEP_NAME=unittest: CPU',
+            'TVM_NUM_SHARDS=1',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
                   label: 'Download artifacts from S3',
                 )
@@ -777,46 +776,44 @@ def shard_run_unittest_CPU_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
               ci_setup(ci_cpu)
               cpp_unittest(ci_cpu)
               python_unittest(ci_cpu)
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('unittest: CPU 1 of 1')
   }
 }
 
 
-def shard_run_frontend_CPU_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_frontend_CPU_1_of_1(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-cpu") {
-        try {
-          init_git()
-          docker_init(ci_cpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=cpu',
-              'TEST_STEP_NAME=frontend: CPU',
-              'TVM_NUM_SHARDS=1',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_cpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=cpu',
+            'TEST_STEP_NAME=frontend: CPU',
+            'TVM_NUM_SHARDS=1',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
                   label: 'Download artifacts from S3',
                 )
@@ -826,22 +823,22 @@ def shard_run_frontend_CPU_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh",
                 label: 'Run Python frontend tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_CPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('frontend: CPU 1 of 1')
   }
@@ -856,44 +853,68 @@ def test() {
     parallel(
     'integration: CPU 1 of 4': {
       try {
-      shard_run_integration_CPU_1_of_4()
-      } catch (Exception ex) {
-        shard_run_integration_CPU_1_of_4(on_demand = true)
+      shard_run_integration_CPU_1_of_4('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_integration_CPU_1_of_4('CPU-SMALL')
       }
     },
     'integration: CPU 2 of 4': {
       try {
-      shard_run_integration_CPU_2_of_4()
-      } catch (Exception ex) {
-        shard_run_integration_CPU_2_of_4(on_demand = true)
+      shard_run_integration_CPU_2_of_4('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_integration_CPU_2_of_4('CPU-SMALL')
       }
     },
     'integration: CPU 3 of 4': {
       try {
-      shard_run_integration_CPU_3_of_4()
-      } catch (Exception ex) {
-        shard_run_integration_CPU_3_of_4(on_demand = true)
+      shard_run_integration_CPU_3_of_4('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_integration_CPU_3_of_4('CPU-SMALL')
       }
     },
     'integration: CPU 4 of 4': {
       try {
-      shard_run_integration_CPU_4_of_4()
-      } catch (Exception ex) {
-        shard_run_integration_CPU_4_of_4(on_demand = true)
+      shard_run_integration_CPU_4_of_4('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_integration_CPU_4_of_4('CPU-SMALL')
       }
     },
     'unittest: CPU 1 of 1': {
       try {
-      shard_run_unittest_CPU_1_of_1()
-      } catch (Exception ex) {
-        shard_run_unittest_CPU_1_of_1(on_demand = true)
+      shard_run_unittest_CPU_1_of_1('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_unittest_CPU_1_of_1('CPU-SMALL')
       }
     },
     'frontend: CPU 1 of 1': {
       try {
-      shard_run_frontend_CPU_1_of_1()
-      } catch (Exception ex) {
-        shard_run_frontend_CPU_1_of_1(on_demand = true)
+      shard_run_frontend_CPU_1_of_1('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_frontend_CPU_1_of_1('CPU-SMALL')
       }
     },
     )
diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy
index 796fbe0c2d64..553678b38c5a 100644
--- a/ci/jenkins/generated/docker_jenkinsfile.groovy
+++ b/ci/jenkins/generated/docker_jenkinsfile.groovy
@@ -60,7 +60,7 @@
 // 'python3 jenkins/generate.py'
 // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
 // always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.207618
+// Generated at 2025-02-06T12:16:02.541252
 
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 // These are set at runtime from data in ci/jenkins/docker-images.yml, update
diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy
index 014377f05d36..63e68cb072d9 100644
--- a/ci/jenkins/generated/gpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
 // 'python3 jenkins/generate.py'
 // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
 // always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.186261
+// Generated at 2025-02-07T09:50:07.217941
 
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 // These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -514,22 +514,22 @@ try {
 } catch(Exception ex) {
   prepare('CPU-SMALL')
 }
-def build(node_type) {
-  stage('Build') {
-    if (!skip_ci) {
-      node(node_type) {
-        ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
+def run_build(node_type) {
+  if (!skip_ci) {
+    echo 'Begin running node_type ' + node_type
+    node(node_type) {
+      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") {
+        init_git()
+        docker_init(ci_gpu)
+        timeout(time: max_time, unit: 'MINUTES') {
 
-            withEnv([
-              'PLATFORM=gpu',
-              ], {
-              sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
+          withEnv([
+            'PLATFORM=gpu',
+            ], {
+            sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
         cmake_build("${ci_gpu} --no-gpu", 'build', '-j2')
         sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu --items build/libtvm.so build/libvta_fsim.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/microtvm_template_projects build/crttest build/build.ninja build/3rdparty/libflash_attn/src/libflash_attn.so build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
+            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/3rdparty/libflash_attn/src/libflash_attn.so build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
             label: 'Upload artifacts to S3',
           )
 
@@ -539,44 +539,51 @@ def build(node_type) {
         sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
         cmake_build("${ci_gpu} --no-gpu", 'build', '-j2')
         sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu2 --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/crttest build/build.ninja build/3rdparty/libflash_attn/src/libflash_attn.so build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
+            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu2 --items build/libtvm.so build/libtvm_runtime.so build/config.cmake",
             label: 'Upload artifacts to S3',
           )
-            })
-          }
+          })
         }
       }
-    } else {
-      Utils.markStageSkippedForConditional('BUILD: GPU')
     }
+    echo 'End running node_type ' + node_type
+  } else {
+    Utils.markStageSkippedForConditional('BUILD: GPU')
   }
 }
-try {
-    build('CPU')
-} catch (Exception ex) {
-    build('CPU-SMALL')
+def build() {
+  stage('Build') {
+    try {
+        run_build('CPU-SPOT')
+    } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        run_build('CPU')
+    }
+  }
 }
+build()
 
 
 
-def shard_run_unittest_GPU_1_of_3(node_type='GPU-SPOT', on_demand=false) {
+def shard_run_unittest_GPU_1_of_3(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=unittest: GPU',
-              'TVM_NUM_SHARDS=3',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_gpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=gpu',
+            'TEST_STEP_NAME=unittest: GPU',
+            'TVM_NUM_SHARDS=3',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu2",
                   label: 'Download artifacts from S3',
                 )
@@ -612,45 +619,43 @@ def shard_run_unittest_GPU_1_of_3(node_type='GPU-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh",
                 label: 'Run Python GPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('unittest: GPU 1 of 3')
   }
 }
 
-def shard_run_unittest_GPU_2_of_3(node_type='GPU-SPOT', on_demand=false) {
+def shard_run_unittest_GPU_2_of_3(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=unittest: GPU',
-              'TVM_NUM_SHARDS=3',
-              'TVM_SHARD_INDEX=1',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_gpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=gpu',
+            'TEST_STEP_NAME=unittest: GPU',
+            'TVM_NUM_SHARDS=3',
+            'TVM_SHARD_INDEX=1',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
                   label: 'Download artifacts from S3',
                 )
@@ -668,45 +673,43 @@ def shard_run_unittest_GPU_2_of_3(node_type='GPU-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh",
                 label: 'Run Python GPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('unittest: GPU 2 of 3')
   }
 }
 
-def shard_run_unittest_GPU_3_of_3(node_type='GPU-SPOT', on_demand=false) {
+def shard_run_unittest_GPU_3_of_3(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=unittest: GPU',
-              'TVM_NUM_SHARDS=3',
-              'TVM_SHARD_INDEX=2',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_gpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=gpu',
+            'TEST_STEP_NAME=unittest: GPU',
+            'TVM_NUM_SHARDS=3',
+            'TVM_SHARD_INDEX=2',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
                   label: 'Download artifacts from S3',
                 )
@@ -720,22 +723,22 @@ def shard_run_unittest_GPU_3_of_3(node_type='GPU-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh",
                 label: 'Run Python GPU integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('unittest: GPU 3 of 3')
   }
@@ -743,24 +746,22 @@ def shard_run_unittest_GPU_3_of_3(node_type='GPU-SPOT', on_demand=false) {
 
 
 
-def shard_run_topi_GPU_1_of_3(node_type='GPU-SPOT', on_demand=false) {
+def shard_run_topi_GPU_1_of_3(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=topi: GPU',
-              'TVM_NUM_SHARDS=3',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_gpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=gpu',
+            'TEST_STEP_NAME=topi: GPU',
+            'TVM_NUM_SHARDS=3',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
                   label: 'Download artifacts from S3',
                 )
@@ -770,45 +771,43 @@ def shard_run_topi_GPU_1_of_3(node_type='GPU-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
                 label: 'Run TOPI tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_GPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('topi: GPU 1 of 3')
   }
 }
 
-def shard_run_topi_GPU_2_of_3(node_type='GPU-SPOT', on_demand=false) {
+def shard_run_topi_GPU_2_of_3(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=topi: GPU',
-              'TVM_NUM_SHARDS=3',
-              'TVM_SHARD_INDEX=1',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_gpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=gpu',
+            'TEST_STEP_NAME=topi: GPU',
+            'TVM_NUM_SHARDS=3',
+            'TVM_SHARD_INDEX=1',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
                   label: 'Download artifacts from S3',
                 )
@@ -818,45 +817,43 @@ def shard_run_topi_GPU_2_of_3(node_type='GPU-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
                 label: 'Run TOPI tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_GPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('topi: GPU 2 of 3')
   }
 }
 
-def shard_run_topi_GPU_3_of_3(node_type='GPU-SPOT', on_demand=false) {
+def shard_run_topi_GPU_3_of_3(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=topi: GPU',
-              'TVM_NUM_SHARDS=3',
-              'TVM_SHARD_INDEX=2',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_gpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=gpu',
+            'TEST_STEP_NAME=topi: GPU',
+            'TVM_NUM_SHARDS=3',
+            'TVM_SHARD_INDEX=2',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
                   label: 'Download artifacts from S3',
                 )
@@ -866,337 +863,45 @@ def shard_run_topi_GPU_3_of_3(node_type='GPU-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
                 label: 'Run TOPI tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_GPU --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('topi: GPU 3 of 3')
-  }
-}
-
-
-
-def shard_run_frontend_GPU_1_of_6(node_type='GPU-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=frontend: GPU',
-              'TVM_NUM_SHARDS=6',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_gpu)
-              sh (
-                script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
-                label: 'Run Python frontend tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          })
         }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('frontend: GPU 1 of 6')
-  }
-}
-
-def shard_run_frontend_GPU_2_of_6(node_type='GPU-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
+        // only run upload if things are successful
         try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=frontend: GPU',
-              'TVM_NUM_SHARDS=6',
-              'TVM_SHARD_INDEX=1',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_gpu)
-              sh (
-                script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
-                label: 'Run Python frontend tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('frontend: GPU 2 of 6')
-  }
-}
-
-def shard_run_frontend_GPU_3_of_6(node_type='GPU-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=frontend: GPU',
-              'TVM_NUM_SHARDS=6',
-              'TVM_SHARD_INDEX=2',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_gpu)
-              sh (
-                script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
-                label: 'Run Python frontend tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('frontend: GPU 3 of 6')
-  }
-}
-
-def shard_run_frontend_GPU_4_of_6(node_type='GPU-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=frontend: GPU',
-              'TVM_NUM_SHARDS=6',
-              'TVM_SHARD_INDEX=3',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_gpu)
-              sh (
-                script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
-                label: 'Run Python frontend tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('frontend: GPU 4 of 6')
-  }
-}
-
-def shard_run_frontend_GPU_5_of_6(node_type='GPU-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=frontend: GPU',
-              'TVM_NUM_SHARDS=6',
-              'TVM_SHARD_INDEX=4',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_gpu)
-              sh (
-                script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
-                label: 'Run Python frontend tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('frontend: GPU 5 of 6')
-  }
-}
-
-def shard_run_frontend_GPU_6_of_6(node_type='GPU-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=frontend: GPU',
-              'TVM_NUM_SHARDS=6',
-              'TVM_SHARD_INDEX=5',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_gpu)
-              sh (
-                script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
-                label: 'Run Python frontend tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
+          sh(
+            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_GPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
-    Utils.markStageSkippedForConditional('frontend: GPU 6 of 6')
+    Utils.markStageSkippedForConditional('topi: GPU 3 of 3')
   }
 }
 
 
 
-def shard_run_docs_GPU_1_of_1(node_type='GPU-SPOT', on_demand=false) {
+def shard_run_docs_GPU_1_of_1(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'GPU'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") {
-        try {
-          init_git()
-          docker_init(ci_gpu)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=gpu',
-              'TEST_STEP_NAME=docs: GPU',
-              'TVM_NUM_SHARDS=1',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_gpu)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=gpu',
+            'TEST_STEP_NAME=docs: GPU',
+            'TVM_NUM_SHARDS=1',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
                   label: 'Download artifacts from S3',
                 )
@@ -1215,22 +920,22 @@ def shard_run_docs_GPU_1_of_1(node_type='GPU-SPOT', on_demand=false) {
                 script: "aws s3 cp --no-progress _docs s3://${s3_bucket}/${s3_prefix}/docs --recursive",
                 label: 'Upload docs to S3',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/docs_GPU --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('docs: GPU 1 of 1')
   }
@@ -1246,93 +951,79 @@ def test() {
     parallel(
     'unittest: GPU 1 of 3': {
       try {
-      shard_run_unittest_GPU_1_of_3()
-      } catch (Exception ex) {
-        shard_run_unittest_GPU_1_of_3(on_demand = true)
+      shard_run_unittest_GPU_1_of_3('GPU-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_unittest_GPU_1_of_3('GPU')
       }
     },
     'unittest: GPU 2 of 3': {
       try {
-      shard_run_unittest_GPU_2_of_3()
-      } catch (Exception ex) {
-        shard_run_unittest_GPU_2_of_3(on_demand = true)
+      shard_run_unittest_GPU_2_of_3('GPU-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_unittest_GPU_2_of_3('GPU')
       }
     },
     'unittest: GPU 3 of 3': {
       try {
-      shard_run_unittest_GPU_3_of_3()
-      } catch (Exception ex) {
-        shard_run_unittest_GPU_3_of_3(on_demand = true)
+      shard_run_unittest_GPU_3_of_3('GPU-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_unittest_GPU_3_of_3('GPU')
       }
     },
     'topi: GPU 1 of 3': {
       try {
-      shard_run_topi_GPU_1_of_3()
-      } catch (Exception ex) {
-        shard_run_topi_GPU_1_of_3(on_demand = true)
+      shard_run_topi_GPU_1_of_3('GPU-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_topi_GPU_1_of_3('GPU')
       }
     },
     'topi: GPU 2 of 3': {
       try {
-      shard_run_topi_GPU_2_of_3()
-      } catch (Exception ex) {
-        shard_run_topi_GPU_2_of_3(on_demand = true)
+      shard_run_topi_GPU_2_of_3('GPU-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_topi_GPU_2_of_3('GPU')
       }
     },
     'topi: GPU 3 of 3': {
       try {
-      shard_run_topi_GPU_3_of_3()
-      } catch (Exception ex) {
-        shard_run_topi_GPU_3_of_3(on_demand = true)
-      }
-    },
-    'frontend: GPU 1 of 6': {
-      try {
-      shard_run_frontend_GPU_1_of_6()
-      } catch (Exception ex) {
-        shard_run_frontend_GPU_1_of_6(on_demand = true)
-      }
-    },
-    'frontend: GPU 2 of 6': {
-      try {
-      shard_run_frontend_GPU_2_of_6()
-      } catch (Exception ex) {
-        shard_run_frontend_GPU_2_of_6(on_demand = true)
-      }
-    },
-    'frontend: GPU 3 of 6': {
-      try {
-      shard_run_frontend_GPU_3_of_6()
-      } catch (Exception ex) {
-        shard_run_frontend_GPU_3_of_6(on_demand = true)
-      }
-    },
-    'frontend: GPU 4 of 6': {
-      try {
-      shard_run_frontend_GPU_4_of_6()
-      } catch (Exception ex) {
-        shard_run_frontend_GPU_4_of_6(on_demand = true)
-      }
-    },
-    'frontend: GPU 5 of 6': {
-      try {
-      shard_run_frontend_GPU_5_of_6()
-      } catch (Exception ex) {
-        shard_run_frontend_GPU_5_of_6(on_demand = true)
-      }
-    },
-    'frontend: GPU 6 of 6': {
-      try {
-      shard_run_frontend_GPU_6_of_6()
-      } catch (Exception ex) {
-        shard_run_frontend_GPU_6_of_6(on_demand = true)
+      shard_run_topi_GPU_3_of_3('GPU-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_topi_GPU_3_of_3('GPU')
       }
     },
     'docs: GPU 1 of 1': {
       try {
-      shard_run_docs_GPU_1_of_1()
-      } catch (Exception ex) {
-        shard_run_docs_GPU_1_of_1(on_demand = true)
+      shard_run_docs_GPU_1_of_1('GPU-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_docs_GPU_1_of_1('GPU')
       }
     },
     )
diff --git a/ci/jenkins/generated/hexagon_jenkinsfile.groovy b/ci/jenkins/generated/hexagon_jenkinsfile.groovy
index a48b12ecd25b..75331a67183a 100644
--- a/ci/jenkins/generated/hexagon_jenkinsfile.groovy
+++ b/ci/jenkins/generated/hexagon_jenkinsfile.groovy
@@ -60,7 +60,7 @@
 // 'python3 jenkins/generate.py'
 // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
 // always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.087221
+// Generated at 2025-02-07T09:50:07.160234
 
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 // These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -514,19 +514,19 @@ try {
 } catch(Exception ex) {
   prepare('CPU-SMALL')
 }
-def build(node_type) {
-  stage('Build') {
-    if (!skip_ci && is_docs_only_build != 1) {
-      node(node_type) {
-        ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") {
-          init_git()
-          docker_init(ci_hexagon)
-          timeout(time: max_time, unit: 'MINUTES') {
-
-            withEnv([
-              'PLATFORM=hexagon',
-              ], {
-              sh (
+def run_build(node_type) {
+  if (!skip_ci && is_docs_only_build != 1) {
+    echo 'Begin running node_type ' + node_type
+    node(node_type) {
+      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") {
+        init_git()
+        docker_init(ci_hexagon)
+        timeout(time: max_time, unit: 'MINUTES') {
+
+          withEnv([
+            'PLATFORM=hexagon',
+            ], {
+            sh (
           script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build",
           label: 'Create Hexagon cmake config',
         )
@@ -540,42 +540,49 @@ def build(node_type) {
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/hexagon_api_output",
             label: 'Upload artifacts to S3',
           )
-            })
-          }
+          })
         }
       }
-    } else {
-      Utils.markStageSkippedForConditional('BUILD: Hexagon')
     }
+    echo 'End running node_type ' + node_type
+  } else {
+    Utils.markStageSkippedForConditional('BUILD: Hexagon')
   }
 }
-try {
-    build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
-    build('CPU-SMALL')
+def build() {
+  stage('Build') {
+    try {
+        run_build('CPU-SMALL-SPOT')
+    } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        run_build('CPU-SMALL')
+    }
+  }
 }
+build()
 
 
 
 
-def shard_run_test_Hexagon_1_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_test_Hexagon_1_of_8(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
-        try {
-          init_git()
-          docker_init(ci_hexagon)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=hexagon',
-              'TEST_STEP_NAME=test: Hexagon',
-              'TVM_NUM_SHARDS=8',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_hexagon)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=hexagon',
+            'TEST_STEP_NAME=test: Hexagon',
+            'TVM_NUM_SHARDS=8',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
                   label: 'Download artifacts from S3',
                 )
@@ -586,45 +593,43 @@ def shard_run_test_Hexagon_1_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
                 label: 'Run Hexagon tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('test: Hexagon 1 of 8')
   }
 }
 
-def shard_run_test_Hexagon_2_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_test_Hexagon_2_of_8(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
-        try {
-          init_git()
-          docker_init(ci_hexagon)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=hexagon',
-              'TEST_STEP_NAME=test: Hexagon',
-              'TVM_NUM_SHARDS=8',
-              'TVM_SHARD_INDEX=1',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_hexagon)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=hexagon',
+            'TEST_STEP_NAME=test: Hexagon',
+            'TVM_NUM_SHARDS=8',
+            'TVM_SHARD_INDEX=1',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
                   label: 'Download artifacts from S3',
                 )
@@ -634,45 +639,43 @@ def shard_run_test_Hexagon_2_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
                 label: 'Run Hexagon tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('test: Hexagon 2 of 8')
   }
 }
 
-def shard_run_test_Hexagon_3_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_test_Hexagon_3_of_8(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
-        try {
-          init_git()
-          docker_init(ci_hexagon)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=hexagon',
-              'TEST_STEP_NAME=test: Hexagon',
-              'TVM_NUM_SHARDS=8',
-              'TVM_SHARD_INDEX=2',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_hexagon)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=hexagon',
+            'TEST_STEP_NAME=test: Hexagon',
+            'TVM_NUM_SHARDS=8',
+            'TVM_SHARD_INDEX=2',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
                   label: 'Download artifacts from S3',
                 )
@@ -682,45 +685,43 @@ def shard_run_test_Hexagon_3_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
                 label: 'Run Hexagon tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('test: Hexagon 3 of 8')
   }
 }
 
-def shard_run_test_Hexagon_4_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_test_Hexagon_4_of_8(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
-        try {
-          init_git()
-          docker_init(ci_hexagon)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=hexagon',
-              'TEST_STEP_NAME=test: Hexagon',
-              'TVM_NUM_SHARDS=8',
-              'TVM_SHARD_INDEX=3',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_hexagon)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=hexagon',
+            'TEST_STEP_NAME=test: Hexagon',
+            'TVM_NUM_SHARDS=8',
+            'TVM_SHARD_INDEX=3',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
                   label: 'Download artifacts from S3',
                 )
@@ -730,45 +731,43 @@ def shard_run_test_Hexagon_4_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
                 label: 'Run Hexagon tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('test: Hexagon 4 of 8')
   }
 }
 
-def shard_run_test_Hexagon_5_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_test_Hexagon_5_of_8(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
-        try {
-          init_git()
-          docker_init(ci_hexagon)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=hexagon',
-              'TEST_STEP_NAME=test: Hexagon',
-              'TVM_NUM_SHARDS=8',
-              'TVM_SHARD_INDEX=4',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_hexagon)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=hexagon',
+            'TEST_STEP_NAME=test: Hexagon',
+            'TVM_NUM_SHARDS=8',
+            'TVM_SHARD_INDEX=4',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
                   label: 'Download artifacts from S3',
                 )
@@ -778,45 +777,43 @@ def shard_run_test_Hexagon_5_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
                 label: 'Run Hexagon tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('test: Hexagon 5 of 8')
   }
 }
 
-def shard_run_test_Hexagon_6_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_test_Hexagon_6_of_8(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
-        try {
-          init_git()
-          docker_init(ci_hexagon)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=hexagon',
-              'TEST_STEP_NAME=test: Hexagon',
-              'TVM_NUM_SHARDS=8',
-              'TVM_SHARD_INDEX=5',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_hexagon)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=hexagon',
+            'TEST_STEP_NAME=test: Hexagon',
+            'TVM_NUM_SHARDS=8',
+            'TVM_SHARD_INDEX=5',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
                   label: 'Download artifacts from S3',
                 )
@@ -826,45 +823,43 @@ def shard_run_test_Hexagon_6_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
                 label: 'Run Hexagon tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('test: Hexagon 6 of 8')
   }
 }
 
-def shard_run_test_Hexagon_7_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_test_Hexagon_7_of_8(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
-        try {
-          init_git()
-          docker_init(ci_hexagon)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=hexagon',
-              'TEST_STEP_NAME=test: Hexagon',
-              'TVM_NUM_SHARDS=8',
-              'TVM_SHARD_INDEX=6',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_hexagon)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=hexagon',
+            'TEST_STEP_NAME=test: Hexagon',
+            'TVM_NUM_SHARDS=8',
+            'TVM_SHARD_INDEX=6',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
                   label: 'Download artifacts from S3',
                 )
@@ -874,45 +869,43 @@ def shard_run_test_Hexagon_7_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
                 label: 'Run Hexagon tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('test: Hexagon 7 of 8')
   }
 }
 
-def shard_run_test_Hexagon_8_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_test_Hexagon_8_of_8(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
-        try {
-          init_git()
-          docker_init(ci_hexagon)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=hexagon',
-              'TEST_STEP_NAME=test: Hexagon',
-              'TVM_NUM_SHARDS=8',
-              'TVM_SHARD_INDEX=7',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_hexagon)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=hexagon',
+            'TEST_STEP_NAME=test: Hexagon',
+            'TVM_NUM_SHARDS=8',
+            'TVM_SHARD_INDEX=7',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
                   label: 'Download artifacts from S3',
                 )
@@ -922,22 +915,22 @@ def shard_run_test_Hexagon_8_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
                 label: 'Run Hexagon tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('test: Hexagon 8 of 8')
   }
@@ -952,58 +945,90 @@ def test() {
     parallel(
     'test: Hexagon 1 of 8': {
       try {
-      shard_run_test_Hexagon_1_of_8()
-      } catch (Exception ex) {
-        shard_run_test_Hexagon_1_of_8(on_demand = true)
+      shard_run_test_Hexagon_1_of_8('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_test_Hexagon_1_of_8('CPU-SMALL')
       }
     },
     'test: Hexagon 2 of 8': {
       try {
-      shard_run_test_Hexagon_2_of_8()
-      } catch (Exception ex) {
-        shard_run_test_Hexagon_2_of_8(on_demand = true)
+      shard_run_test_Hexagon_2_of_8('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_test_Hexagon_2_of_8('CPU-SMALL')
       }
     },
     'test: Hexagon 3 of 8': {
       try {
-      shard_run_test_Hexagon_3_of_8()
-      } catch (Exception ex) {
-        shard_run_test_Hexagon_3_of_8(on_demand = true)
+      shard_run_test_Hexagon_3_of_8('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_test_Hexagon_3_of_8('CPU-SMALL')
       }
     },
     'test: Hexagon 4 of 8': {
       try {
-      shard_run_test_Hexagon_4_of_8()
-      } catch (Exception ex) {
-        shard_run_test_Hexagon_4_of_8(on_demand = true)
+      shard_run_test_Hexagon_4_of_8('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_test_Hexagon_4_of_8('CPU-SMALL')
       }
     },
     'test: Hexagon 5 of 8': {
       try {
-      shard_run_test_Hexagon_5_of_8()
-      } catch (Exception ex) {
-        shard_run_test_Hexagon_5_of_8(on_demand = true)
+      shard_run_test_Hexagon_5_of_8('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_test_Hexagon_5_of_8('CPU-SMALL')
       }
     },
     'test: Hexagon 6 of 8': {
       try {
-      shard_run_test_Hexagon_6_of_8()
-      } catch (Exception ex) {
-        shard_run_test_Hexagon_6_of_8(on_demand = true)
+      shard_run_test_Hexagon_6_of_8('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_test_Hexagon_6_of_8('CPU-SMALL')
       }
     },
     'test: Hexagon 7 of 8': {
       try {
-      shard_run_test_Hexagon_7_of_8()
-      } catch (Exception ex) {
-        shard_run_test_Hexagon_7_of_8(on_demand = true)
+      shard_run_test_Hexagon_7_of_8('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_test_Hexagon_7_of_8('CPU-SMALL')
       }
     },
     'test: Hexagon 8 of 8': {
       try {
-      shard_run_test_Hexagon_8_of_8()
-      } catch (Exception ex) {
-        shard_run_test_Hexagon_8_of_8(on_demand = true)
+      shard_run_test_Hexagon_8_of_8('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_test_Hexagon_8_of_8('CPU-SMALL')
       }
     },
     )
diff --git a/ci/jenkins/generated/i386_jenkinsfile.groovy b/ci/jenkins/generated/i386_jenkinsfile.groovy
index 47d20564730f..183245838cfb 100644
--- a/ci/jenkins/generated/i386_jenkinsfile.groovy
+++ b/ci/jenkins/generated/i386_jenkinsfile.groovy
@@ -60,7 +60,7 @@
 // 'python3 jenkins/generate.py'
 // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
 // always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.169799
+// Generated at 2025-02-07T09:50:07.135264
 
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 // These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -507,7 +507,6 @@ def cpp_unittest(image) {
   )
 }
 
-
 cancel_previous_build()
 
 try {
@@ -515,64 +514,71 @@ try {
 } catch(Exception ex) {
   prepare('CPU-SMALL')
 }
-def build(node_type) {
-  stage('Build') {
-    if (!skip_ci && is_docs_only_build != 1) {
-      node(node_type) {
-        ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") {
-          init_git()
-          docker_init(ci_i386)
-          timeout(time: max_time, unit: 'MINUTES') {
-
-            withEnv([
-              'PLATFORM=i386',
-              ], {
-              sh (
+def run_build(node_type) {
+  if (!skip_ci && is_docs_only_build != 1) {
+    echo 'Begin running node_type ' + node_type
+    node(node_type) {
+      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") {
+        init_git()
+        docker_init(ci_i386)
+        timeout(time: max_time, unit: 'MINUTES') {
+
+          withEnv([
+            'PLATFORM=i386',
+            ], {
+            sh (
           script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build",
           label: 'Create i386 cmake config',
         )
         cmake_build(ci_i386, 'build', '-j2')
         make_cpp_tests(ci_i386, 'build')
         sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/i386 --items build/libvta_tsim.so build/libtvm.so build/libvta_fsim.so build/libtvm_runtime.so build/config.cmake build/build.ninja build/crttest build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/microtvm_template_projects",
+            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/i386 --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/cpptest build/build.ninja build/CMakeFiles/rules.ninja",
             label: 'Upload artifacts to S3',
           )
-            })
-          }
+          })
         }
       }
-    } else {
-      Utils.markStageSkippedForConditional('BUILD: i386')
     }
+    echo 'End running node_type ' + node_type
+  } else {
+    Utils.markStageSkippedForConditional('BUILD: i386')
   }
 }
-try {
-    build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
-    build('CPU-SMALL')
+def build() {
+  stage('Build') {
+    try {
+        run_build('CPU-SMALL-SPOT')
+    } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        run_build('CPU-SMALL')
+    }
+  }
 }
+build()
 
 
 
 
-def shard_run_python_i386_1_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_python_i386_1_of_3(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
-        try {
-          init_git()
-          docker_init(ci_i386)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=i386',
-              'TEST_STEP_NAME=python: i386',
-              'TVM_NUM_SHARDS=3',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_i386)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=i386',
+            'TEST_STEP_NAME=python: i386',
+            'TVM_NUM_SHARDS=3',
+            'TVM_SHARD_INDEX=0',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/i386",
                   label: 'Download artifacts from S3',
                 )
@@ -584,45 +590,43 @@ def shard_run_python_i386_1_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh",
                 label: 'Run i386 integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/python_i386 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('python: i386 1 of 3')
   }
 }
 
-def shard_run_python_i386_2_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_python_i386_2_of_3(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
-        try {
-          init_git()
-          docker_init(ci_i386)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=i386',
-              'TEST_STEP_NAME=python: i386',
-              'TVM_NUM_SHARDS=3',
-              'TVM_SHARD_INDEX=1',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_i386)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=i386',
+            'TEST_STEP_NAME=python: i386',
+            'TVM_NUM_SHARDS=3',
+            'TVM_SHARD_INDEX=1',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/i386",
                   label: 'Download artifacts from S3',
                 )
@@ -633,45 +637,43 @@ def shard_run_python_i386_2_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh",
                 label: 'Run i386 integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/python_i386 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('python: i386 2 of 3')
   }
 }
 
-def shard_run_python_i386_3_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
+def shard_run_python_i386_3_of_3(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
     node(node_type) {
       ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
-        try {
-          init_git()
-          docker_init(ci_i386)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=i386',
-              'TEST_STEP_NAME=python: i386',
-              'TVM_NUM_SHARDS=3',
-              'TVM_SHARD_INDEX=2',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init(ci_i386)
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM=i386',
+            'TEST_STEP_NAME=python: i386',
+            'TVM_NUM_SHARDS=3',
+            'TVM_SHARD_INDEX=2',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            sh(
                   script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/i386",
                   label: 'Download artifacts from S3',
                 )
@@ -682,22 +684,22 @@ def shard_run_python_i386_3_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
                 script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh",
                 label: 'Run i386 integration tests',
               )
-            })
-          }
-        } finally {
-          try {
-            sh(
+          })
+        }
+        // only run upload if things are successful
+        try {
+          sh(
             script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/python_i386 --items build/pytest-results",
             label: 'Upload JUnits to S3',
           )
 
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('python: i386 3 of 3')
   }
@@ -712,23 +714,35 @@ def test() {
     parallel(
     'python: i386 1 of 3': {
       try {
-      shard_run_python_i386_1_of_3()
-      } catch (Exception ex) {
-        shard_run_python_i386_1_of_3(on_demand = true)
+      shard_run_python_i386_1_of_3('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_python_i386_1_of_3('CPU-SMALL')
       }
     },
     'python: i386 2 of 3': {
       try {
-      shard_run_python_i386_2_of_3()
-      } catch (Exception ex) {
-        shard_run_python_i386_2_of_3(on_demand = true)
+      shard_run_python_i386_2_of_3('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_python_i386_2_of_3('CPU-SMALL')
       }
     },
     'python: i386 3 of 3': {
       try {
-      shard_run_python_i386_3_of_3()
-      } catch (Exception ex) {
-        shard_run_python_i386_3_of_3(on_demand = true)
+      shard_run_python_i386_3_of_3('CPU-SMALL-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        shard_run_python_i386_3_of_3('CPU-SMALL')
       }
     },
     )
diff --git a/ci/jenkins/generated/lint_jenkinsfile.groovy b/ci/jenkins/generated/lint_jenkinsfile.groovy
index b1e300d7125d..bc8118c5463a 100644
--- a/ci/jenkins/generated/lint_jenkinsfile.groovy
+++ b/ci/jenkins/generated/lint_jenkinsfile.groovy
@@ -60,7 +60,7 @@
 // 'python3 jenkins/generate.py'
 // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
 // always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.245060
+// Generated at 2025-02-06T12:16:02.571072
 
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 // These are set at runtime from data in ci/jenkins/docker-images.yml, update
diff --git a/ci/jenkins/generated/minimal_cross_isa_jenkinsfile.groovy b/ci/jenkins/generated/minimal_cross_isa_jenkinsfile.groovy
deleted file mode 100644
index 3645cf90364c..000000000000
--- a/ci/jenkins/generated/minimal_cross_isa_jenkinsfile.groovy
+++ /dev/null
@@ -1,622 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// -*- mode: groovy -*-
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-// Docker env used for testing
-// Different image may have different version tag
-// because some of them are more stable than anoter.
-//
-// Docker images are maintained by PMC, cached in dockerhub
-// and remains relatively stable over the time.
-// Flow for upgrading docker env(need commiter)
-//
-// - Send PR to upgrade build script in the repo
-// - Build the new docker image
-// - Tag the docker image with a new version and push to a binary cache.
-// - Update the version in the Jenkinsfile, send a PR
-// - Fix any issues wrt to the new image version in the PR
-// - Merge the PR and now we are in new version
-// - Tag the new version as the lates
-// - Periodically cleanup the old versions on local workers
-//
-
-// ============================= IMPORTANT NOTE =============================
-// This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
-// Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
-// 'python3 jenkins/generate.py'
-// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
-// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.045689
-
-import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-// These are set at runtime from data in ci/jenkins/docker-images.yml, update
-// image tags in that file
-ci_lint = ''
-ci_gpu = ''
-ci_cpu = ''
-ci_minimal = ''
-ci_wasm = ''
-ci_i386 = ''
-ci_cortexm = ''
-ci_arm = ''
-ci_hexagon = ''
-ci_riscv = ''
-
-// Parameters to allow overriding (in Jenkins UI), the images
-// to be used by a given build. When provided, they take precedence
-// over default values above.
-properties([
-  parameters([
-    string(name: 'ci_arm_param', defaultValue: ''),
-    string(name: 'ci_cortexm_param', defaultValue: ''),
-    string(name: 'ci_cpu_param', defaultValue: ''),
-    string(name: 'ci_gpu_param', defaultValue: ''),
-    string(name: 'ci_hexagon_param', defaultValue: ''),
-    string(name: 'ci_i386_param', defaultValue: ''),
-    string(name: 'ci_lint_param', defaultValue: ''),
-    string(name: 'ci_minimal_param', defaultValue: ''),
-    string(name: 'ci_riscv_param', defaultValue: ''),
-    string(name: 'ci_wasm_param', defaultValue: ''),
-  ])
-])
-
-// Placeholders for newly built Docker image names (if rebuild_docker_images
-// is used)
-  built_ci_arm = null;
-  built_ci_cortexm = null;
-  built_ci_cpu = null;
-  built_ci_gpu = null;
-  built_ci_hexagon = null;
-  built_ci_i386 = null;
-  built_ci_lint = null;
-  built_ci_minimal = null;
-  built_ci_riscv = null;
-  built_ci_wasm = null;
-
-// Global variable assigned during Sanity Check that holds the sha1 which should be
-// merged into the PR in all branches.
-upstream_revision = null
-
-// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
-docker_build = 'docker/build.sh'
-// timeout in minutes
-max_time = 180
-rebuild_docker_images = false
-
-s3_bucket = 'tvm-jenkins-artifacts-prod'
-s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
-// Jenkins script root directory
-jenkins_scripts_root = "ci/scripts/jenkins"
-
-
-// General note: Jenkins has limits on the size of a method (or top level code)
-// that are pretty strict, so most usage of groovy methods in these templates
-// are purely to satisfy the JVM
-def per_exec_ws(folder) {
-  return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
-}
-
-// initialize source codes
-def init_git() {
-  retry(5) {
-    checkout scm
-  }
-
-  // Add more info about job node
-  sh (
-    script: './tests/scripts/task_show_node_info.sh',
-    label: 'Show executor node info',
-  )
-
-  // Determine merge commit to use for all stages
-  if (env.BRANCH_NAME == 'main') {
-    // Only set upstream_revision to HEAD and skip merging to avoid a race with another commit merged to main.
-    update_upstream_revision("HEAD")
-  } else {
-    // This is PR branch so merge with latest main.
-    merge_with_main()
-  }
-
-  sh(
-    script: """
-      set -eux
-      . ${jenkins_scripts_root}/retry.sh
-      retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
-    """,
-    label: 'Update git submodules',
-  )
-  checkout_trusted_files()
-}
-
-def update_upstream_revision(git_ref) {
-  if (upstream_revision == null) {
-    upstream_revision = sh(
-      script: "git log -1 ${git_ref} --format=\'%H\'",
-      label: 'Determine upstream revision',
-      returnStdout: true,
-    ).trim()
-  }
-}
-
-def merge_with_main() {
-  sh (
-    script: 'git fetch origin main',
-    label: 'Fetch upstream',
-  )
-  update_upstream_revision("FETCH_HEAD")
-  sh (
-    script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
-    label: 'Merge to origin/main'
-  )
-}
-
-def docker_init(image) {
-  // Clear out all Docker images that aren't going to be used
-  sh(
-    script: """
-    set -eux
-    docker image ls --all
-    IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}}  {{.ID}}')
-
-    echo -e "Found images:\\n\$IMAGES"
-    echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
-    docker image ls --all
-    """,
-    label: 'Clean old Docker images',
-  )
-
-  if (image.contains("amazonaws.com")) {
-    // If this string is in the image name it's from ECR and needs to be pulled
-    // with the right credentials
-    ecr_pull(image)
-  } else {
-    sh(
-      script: """
-      set -eux
-      . ${jenkins_scripts_root}/retry.sh
-      retry 5 docker pull ${image}
-      """,
-      label: 'Pull docker image',
-    )
-  }
-}
-
-def ecr_pull(full_name) {
-  aws_account_id = sh(
-    returnStdout: true,
-    script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
-    label: 'Get AWS ID'
-  ).trim()
-
-  try {
-    withEnv([
-      "AWS_ACCOUNT_ID=${aws_account_id}",
-      'AWS_DEFAULT_REGION=us-west-2',
-      "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
-      sh(
-        script: '''
-          set -eux
-          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
-        ''',
-        label: 'Log in to ECR'
-      )
-      sh(
-        script: """
-          set -eux
-          . ${jenkins_scripts_root}/retry.sh
-          retry 5 docker pull ${full_name}
-        """,
-        label: 'Pull image from ECR'
-      )
-    }
-  } finally {
-    withEnv([
-      "AWS_ACCOUNT_ID=${aws_account_id}",
-      'AWS_DEFAULT_REGION=us-west-2',
-      "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
-      sh(
-        script: 'docker logout $AWS_ECR_REPO',
-        label: 'Clean up login credentials'
-      )
-    }
-  }
-}
-
-def should_skip_slow_tests(pr_number) {
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-  )]) {
-    // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
-    result = sh (
-      returnStatus: true,
-      script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
-      label: 'Check if CI should run slow tests',
-    )
-  }
-  return result == 0
-}
-
-def cancel_previous_build() {
-  // cancel previous build if it is not on main.
-  if (env.BRANCH_NAME != 'main') {
-    def buildNumber = env.BUILD_NUMBER as int
-    // Milestone API allows us to cancel previous build
-    // with the same milestone number
-    if (buildNumber > 1) milestone(buildNumber - 1)
-    milestone(buildNumber)
-  }
-}
-
-def checkout_trusted_files() {
-  // trust everything from branch builds
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    return;
-  }
-
-  // trust peoople listed in CONTRIBUTING.md
-  grep_code = sh(
-    returnStatus: true,
-    script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
-    label: 'Check if change is from a contributor',
-  )
-
-  if (grep_code == 1) {
-    // Any scripts that run on the bare host and not inside a Docker container
-    // (especially those that access secrets) should be checked out here so
-    // only trusted versions are used in CI
-    sh(
-      script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
-      label: 'Check out trusted files',
-    )
-  }
-}
-
-def should_skip_ci(pr_number) {
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    // never skip CI on build sourced from a branch
-    return false
-  }
-  glob_skip_ci_code = sh (
-    returnStatus: true,
-    script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
-    label: 'Check if CI should be skipped due to changed files',
-  )
-  if (glob_skip_ci_code == 0) {
-    return true
-  }
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-    )]) {
-    // Exit code of 1 means run full CI (or the script had an error, so run
-    // full CI just in case). Exit code of 0 means skip CI.
-    git_skip_ci_code = sh (
-      returnStatus: true,
-      script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
-      label: 'Check if CI should be skipped',
-    )
-  }
-  return git_skip_ci_code == 0
-}
-
-def check_pr(pr_number) {
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    // never skip CI on build sourced from a branch
-    return false
-  }
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-    )]) {
-    sh (
-      script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}",
-      label: 'Check PR title and body',
-    )
-  }
-
-}
-
-def prepare(node_type) {
-  stage('Prepare') {
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
-        init_git()
-
-        check_pr(env.CHANGE_ID)
-
-        if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
-          sh(
-            script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
-            label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
-          )
-          // Pull image names from the results of should_rebuild_docker.py
-          ci_arm = sh(
-            script: "cat .docker-image-names/ci_arm",
-            label: "Find docker image name for ci_arm",
-            returnStdout: true,
-          ).trim()
-          ci_cortexm = sh(
-            script: "cat .docker-image-names/ci_cortexm",
-            label: "Find docker image name for ci_cortexm",
-            returnStdout: true,
-          ).trim()
-          ci_cpu = sh(
-            script: "cat .docker-image-names/ci_cpu",
-            label: "Find docker image name for ci_cpu",
-            returnStdout: true,
-          ).trim()
-          ci_gpu = sh(
-            script: "cat .docker-image-names/ci_gpu",
-            label: "Find docker image name for ci_gpu",
-            returnStdout: true,
-          ).trim()
-          ci_hexagon = sh(
-            script: "cat .docker-image-names/ci_hexagon",
-            label: "Find docker image name for ci_hexagon",
-            returnStdout: true,
-          ).trim()
-          ci_i386 = sh(
-            script: "cat .docker-image-names/ci_i386",
-            label: "Find docker image name for ci_i386",
-            returnStdout: true,
-          ).trim()
-          ci_lint = sh(
-            script: "cat .docker-image-names/ci_lint",
-            label: "Find docker image name for ci_lint",
-            returnStdout: true,
-          ).trim()
-          ci_minimal = sh(
-            script: "cat .docker-image-names/ci_minimal",
-            label: "Find docker image name for ci_minimal",
-            returnStdout: true,
-          ).trim()
-          ci_riscv = sh(
-            script: "cat .docker-image-names/ci_riscv",
-            label: "Find docker image name for ci_riscv",
-            returnStdout: true,
-          ).trim()
-          ci_wasm = sh(
-            script: "cat .docker-image-names/ci_wasm",
-            label: "Find docker image name for ci_wasm",
-            returnStdout: true,
-          ).trim()
-        }
-
-        ci_arm = params.ci_arm_param ?: ci_arm
-        ci_cortexm = params.ci_cortexm_param ?: ci_cortexm
-        ci_cpu = params.ci_cpu_param ?: ci_cpu
-        ci_gpu = params.ci_gpu_param ?: ci_gpu
-        ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
-        ci_i386 = params.ci_i386_param ?: ci_i386
-        ci_lint = params.ci_lint_param ?: ci_lint
-        ci_minimal = params.ci_minimal_param ?: ci_minimal
-        ci_riscv = params.ci_riscv_param ?: ci_riscv
-        ci_wasm = params.ci_wasm_param ?: ci_wasm
-
-        sh (script: """
-          echo "Docker images being used in this build:"
-          echo " ci_arm = ${ci_arm}"
-          echo " ci_cortexm = ${ci_cortexm}"
-          echo " ci_cpu = ${ci_cpu}"
-          echo " ci_gpu = ${ci_gpu}"
-          echo " ci_hexagon = ${ci_hexagon}"
-          echo " ci_i386 = ${ci_i386}"
-          echo " ci_lint = ${ci_lint}"
-          echo " ci_minimal = ${ci_minimal}"
-          echo " ci_riscv = ${ci_riscv}"
-          echo " ci_wasm = ${ci_wasm}"
-        """, label: 'Docker image names')
-
-        is_docs_only_build = sh (
-          returnStatus: true,
-          script: "./${jenkins_scripts_root}/git_change_docs.sh",
-          label: 'Check for docs only changes',
-        )
-        skip_ci = should_skip_ci(env.CHANGE_ID)
-        skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
-        rebuild_docker_images = sh (
-          returnStatus: true,
-          script: "./${jenkins_scripts_root}/git_change_docker.sh",
-          label: 'Check for any docker changes',
-        )
-
-        if (skip_ci) {
-          // Don't rebuild when skipping CI
-          rebuild_docker_images = false
-        }
-      }
-    }
-  }
-}
-def ci_setup(image) {
-  sh (
-    script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
-    label: 'Clean up old workspace',
-  )
-}
-
-def python_unittest(image) {
-  sh (
-    script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
-    label: 'Run Python unit tests',
-  )
-}
-
-def make_cpp_tests(image, build_dir) {
-  sh (
-    script: """
-      set -eux
-      ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
-        --sccache-bucket tvm-sccache-prod \
-        --sccache-region us-west-2 \
-        --cmake-target cpptest \
-        --build-dir ${build_dir}
-      """,
-    label: 'Make C++ tests',
-  )
-}
-
-def cmake_build(image, path, make_flag) {
-  sh (
-    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir ${path}",
-    label: 'Run cmake build',
-  )
-}
-def cpp_unittest(image) {
-  sh (
-    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
-    label: 'Run C++ tests',
-  )
-}
-
-cancel_previous_build()
-
-try {
-    prepare('CPU-SMALL-SPOT')
-} catch(Exception ex) {
-  prepare('CPU-SMALL')
-}
-def build(node_type) {
-  stage('Build') {
-    if (!skip_ci && is_docs_only_build != 1) {
-      node(node_type) {
-        ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu-minimal-cross-isa") {
-          init_git()
-          docker_init(ci_minimal)
-          timeout(time: max_time, unit: 'MINUTES') {
-
-            withEnv([
-              'PLATFORM=arm',
-              ], {
-              sh (
-          script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal_cross_isa.sh build",
-          label: 'Create CPU minimal cmake config',
-        )
-        cmake_build(ci_minimal, 'build', '-j2')
-        sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu-minimal-cross-isa --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/build.ninja build/microtvm_template_projects",
-            label: 'Upload artifacts to S3',
-          )
-            })
-          }
-        }
-      }
-    } else {
-      Utils.markStageSkippedForConditional('BUILD: CPU MINIMAL CROSS ISA')
-    }
-  }
-}
-try {
-    build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
-    build('CPU-SMALL')
-}
-
-
-
-
-def shard_run_unittest_CPU_MINIMAL_CROSS_ISA_1_of_1(node_type='ARM-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'ARM-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-cpp-arm-cross-isa") {
-        try {
-          init_git()
-          docker_init(ci_arm)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=arm',
-              'TEST_STEP_NAME=unittest: CPU MINIMAL CROSS ISA',
-              'TVM_NUM_SHARDS=1',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu-minimal-cross-isa",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_arm)
-              sh "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_minimal_cross_isa.sh build"
-              make_cpp_tests(ci_arm, 'build')
-              cpp_unittest(ci_arm)
-              python_unittest(ci_arm)
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU_MINIMAL_CROSS_ISA --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('unittest: CPU MINIMAL CROSS ISA 1 of 1')
-  }
-}
-
-
-
-def test() {
-  stage('Test') {
-    environment {
-      SKIP_SLOW_TESTS = "${skip_slow_tests}"
-    }
-    parallel(
-    'unittest: CPU MINIMAL CROSS ISA 1 of 1': {
-      try {
-      shard_run_unittest_CPU_MINIMAL_CROSS_ISA_1_of_1()
-      } catch (Exception ex) {
-        shard_run_unittest_CPU_MINIMAL_CROSS_ISA_1_of_1(on_demand = true)
-      }
-    },
-    )
-  }
-}
-test()
diff --git a/ci/jenkins/generated/minimal_jenkinsfile.groovy b/ci/jenkins/generated/minimal_jenkinsfile.groovy
deleted file mode 100644
index 9ae238012235..000000000000
--- a/ci/jenkins/generated/minimal_jenkinsfile.groovy
+++ /dev/null
@@ -1,620 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// -*- mode: groovy -*-
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-// Docker env used for testing
-// Different image may have different version tag
-// because some of them are more stable than anoter.
-//
-// Docker images are maintained by PMC, cached in dockerhub
-// and remains relatively stable over the time.
-// Flow for upgrading docker env(need commiter)
-//
-// - Send PR to upgrade build script in the repo
-// - Build the new docker image
-// - Tag the docker image with a new version and push to a binary cache.
-// - Update the version in the Jenkinsfile, send a PR
-// - Fix any issues wrt to the new image version in the PR
-// - Merge the PR and now we are in new version
-// - Tag the new version as the lates
-// - Periodically cleanup the old versions on local workers
-//
-
-// ============================= IMPORTANT NOTE =============================
-// This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
-// Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
-// 'python3 jenkins/generate.py'
-// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
-// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.139264
-
-import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-// These are set at runtime from data in ci/jenkins/docker-images.yml, update
-// image tags in that file
-ci_lint = ''
-ci_gpu = ''
-ci_cpu = ''
-ci_minimal = ''
-ci_wasm = ''
-ci_i386 = ''
-ci_cortexm = ''
-ci_arm = ''
-ci_hexagon = ''
-ci_riscv = ''
-
-// Parameters to allow overriding (in Jenkins UI), the images
-// to be used by a given build. When provided, they take precedence
-// over default values above.
-properties([
-  parameters([
-    string(name: 'ci_arm_param', defaultValue: ''),
-    string(name: 'ci_cortexm_param', defaultValue: ''),
-    string(name: 'ci_cpu_param', defaultValue: ''),
-    string(name: 'ci_gpu_param', defaultValue: ''),
-    string(name: 'ci_hexagon_param', defaultValue: ''),
-    string(name: 'ci_i386_param', defaultValue: ''),
-    string(name: 'ci_lint_param', defaultValue: ''),
-    string(name: 'ci_minimal_param', defaultValue: ''),
-    string(name: 'ci_riscv_param', defaultValue: ''),
-    string(name: 'ci_wasm_param', defaultValue: ''),
-  ])
-])
-
-// Placeholders for newly built Docker image names (if rebuild_docker_images
-// is used)
-  built_ci_arm = null;
-  built_ci_cortexm = null;
-  built_ci_cpu = null;
-  built_ci_gpu = null;
-  built_ci_hexagon = null;
-  built_ci_i386 = null;
-  built_ci_lint = null;
-  built_ci_minimal = null;
-  built_ci_riscv = null;
-  built_ci_wasm = null;
-
-// Global variable assigned during Sanity Check that holds the sha1 which should be
-// merged into the PR in all branches.
-upstream_revision = null
-
-// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
-docker_build = 'docker/build.sh'
-// timeout in minutes
-max_time = 180
-rebuild_docker_images = false
-
-s3_bucket = 'tvm-jenkins-artifacts-prod'
-s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
-// Jenkins script root directory
-jenkins_scripts_root = "ci/scripts/jenkins"
-
-
-// General note: Jenkins has limits on the size of a method (or top level code)
-// that are pretty strict, so most usage of groovy methods in these templates
-// are purely to satisfy the JVM
-def per_exec_ws(folder) {
-  return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
-}
-
-// initialize source codes
-def init_git() {
-  retry(5) {
-    checkout scm
-  }
-
-  // Add more info about job node
-  sh (
-    script: './tests/scripts/task_show_node_info.sh',
-    label: 'Show executor node info',
-  )
-
-  // Determine merge commit to use for all stages
-  if (env.BRANCH_NAME == 'main') {
-    // Only set upstream_revision to HEAD and skip merging to avoid a race with another commit merged to main.
-    update_upstream_revision("HEAD")
-  } else {
-    // This is PR branch so merge with latest main.
-    merge_with_main()
-  }
-
-  sh(
-    script: """
-      set -eux
-      . ${jenkins_scripts_root}/retry.sh
-      retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
-    """,
-    label: 'Update git submodules',
-  )
-  checkout_trusted_files()
-}
-
-def update_upstream_revision(git_ref) {
-  if (upstream_revision == null) {
-    upstream_revision = sh(
-      script: "git log -1 ${git_ref} --format=\'%H\'",
-      label: 'Determine upstream revision',
-      returnStdout: true,
-    ).trim()
-  }
-}
-
-def merge_with_main() {
-  sh (
-    script: 'git fetch origin main',
-    label: 'Fetch upstream',
-  )
-  update_upstream_revision("FETCH_HEAD")
-  sh (
-    script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
-    label: 'Merge to origin/main'
-  )
-}
-
-def docker_init(image) {
-  // Clear out all Docker images that aren't going to be used
-  sh(
-    script: """
-    set -eux
-    docker image ls --all
-    IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}}  {{.ID}}')
-
-    echo -e "Found images:\\n\$IMAGES"
-    echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
-    docker image ls --all
-    """,
-    label: 'Clean old Docker images',
-  )
-
-  if (image.contains("amazonaws.com")) {
-    // If this string is in the image name it's from ECR and needs to be pulled
-    // with the right credentials
-    ecr_pull(image)
-  } else {
-    sh(
-      script: """
-      set -eux
-      . ${jenkins_scripts_root}/retry.sh
-      retry 5 docker pull ${image}
-      """,
-      label: 'Pull docker image',
-    )
-  }
-}
-
-def ecr_pull(full_name) {
-  aws_account_id = sh(
-    returnStdout: true,
-    script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
-    label: 'Get AWS ID'
-  ).trim()
-
-  try {
-    withEnv([
-      "AWS_ACCOUNT_ID=${aws_account_id}",
-      'AWS_DEFAULT_REGION=us-west-2',
-      "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
-      sh(
-        script: '''
-          set -eux
-          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
-        ''',
-        label: 'Log in to ECR'
-      )
-      sh(
-        script: """
-          set -eux
-          . ${jenkins_scripts_root}/retry.sh
-          retry 5 docker pull ${full_name}
-        """,
-        label: 'Pull image from ECR'
-      )
-    }
-  } finally {
-    withEnv([
-      "AWS_ACCOUNT_ID=${aws_account_id}",
-      'AWS_DEFAULT_REGION=us-west-2',
-      "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
-      sh(
-        script: 'docker logout $AWS_ECR_REPO',
-        label: 'Clean up login credentials'
-      )
-    }
-  }
-}
-
-def should_skip_slow_tests(pr_number) {
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-  )]) {
-    // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
-    result = sh (
-      returnStatus: true,
-      script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
-      label: 'Check if CI should run slow tests',
-    )
-  }
-  return result == 0
-}
-
-def cancel_previous_build() {
-  // cancel previous build if it is not on main.
-  if (env.BRANCH_NAME != 'main') {
-    def buildNumber = env.BUILD_NUMBER as int
-    // Milestone API allows us to cancel previous build
-    // with the same milestone number
-    if (buildNumber > 1) milestone(buildNumber - 1)
-    milestone(buildNumber)
-  }
-}
-
-def checkout_trusted_files() {
-  // trust everything from branch builds
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    return;
-  }
-
-  // trust peoople listed in CONTRIBUTING.md
-  grep_code = sh(
-    returnStatus: true,
-    script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
-    label: 'Check if change is from a contributor',
-  )
-
-  if (grep_code == 1) {
-    // Any scripts that run on the bare host and not inside a Docker container
-    // (especially those that access secrets) should be checked out here so
-    // only trusted versions are used in CI
-    sh(
-      script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
-      label: 'Check out trusted files',
-    )
-  }
-}
-
-def should_skip_ci(pr_number) {
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    // never skip CI on build sourced from a branch
-    return false
-  }
-  glob_skip_ci_code = sh (
-    returnStatus: true,
-    script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
-    label: 'Check if CI should be skipped due to changed files',
-  )
-  if (glob_skip_ci_code == 0) {
-    return true
-  }
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-    )]) {
-    // Exit code of 1 means run full CI (or the script had an error, so run
-    // full CI just in case). Exit code of 0 means skip CI.
-    git_skip_ci_code = sh (
-      returnStatus: true,
-      script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
-      label: 'Check if CI should be skipped',
-    )
-  }
-  return git_skip_ci_code == 0
-}
-
-def check_pr(pr_number) {
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    // never skip CI on build sourced from a branch
-    return false
-  }
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-    )]) {
-    sh (
-      script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}",
-      label: 'Check PR title and body',
-    )
-  }
-
-}
-
-def prepare(node_type) {
-  stage('Prepare') {
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
-        init_git()
-
-        check_pr(env.CHANGE_ID)
-
-        if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
-          sh(
-            script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
-            label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
-          )
-          // Pull image names from the results of should_rebuild_docker.py
-          ci_arm = sh(
-            script: "cat .docker-image-names/ci_arm",
-            label: "Find docker image name for ci_arm",
-            returnStdout: true,
-          ).trim()
-          ci_cortexm = sh(
-            script: "cat .docker-image-names/ci_cortexm",
-            label: "Find docker image name for ci_cortexm",
-            returnStdout: true,
-          ).trim()
-          ci_cpu = sh(
-            script: "cat .docker-image-names/ci_cpu",
-            label: "Find docker image name for ci_cpu",
-            returnStdout: true,
-          ).trim()
-          ci_gpu = sh(
-            script: "cat .docker-image-names/ci_gpu",
-            label: "Find docker image name for ci_gpu",
-            returnStdout: true,
-          ).trim()
-          ci_hexagon = sh(
-            script: "cat .docker-image-names/ci_hexagon",
-            label: "Find docker image name for ci_hexagon",
-            returnStdout: true,
-          ).trim()
-          ci_i386 = sh(
-            script: "cat .docker-image-names/ci_i386",
-            label: "Find docker image name for ci_i386",
-            returnStdout: true,
-          ).trim()
-          ci_lint = sh(
-            script: "cat .docker-image-names/ci_lint",
-            label: "Find docker image name for ci_lint",
-            returnStdout: true,
-          ).trim()
-          ci_minimal = sh(
-            script: "cat .docker-image-names/ci_minimal",
-            label: "Find docker image name for ci_minimal",
-            returnStdout: true,
-          ).trim()
-          ci_riscv = sh(
-            script: "cat .docker-image-names/ci_riscv",
-            label: "Find docker image name for ci_riscv",
-            returnStdout: true,
-          ).trim()
-          ci_wasm = sh(
-            script: "cat .docker-image-names/ci_wasm",
-            label: "Find docker image name for ci_wasm",
-            returnStdout: true,
-          ).trim()
-        }
-
-        ci_arm = params.ci_arm_param ?: ci_arm
-        ci_cortexm = params.ci_cortexm_param ?: ci_cortexm
-        ci_cpu = params.ci_cpu_param ?: ci_cpu
-        ci_gpu = params.ci_gpu_param ?: ci_gpu
-        ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
-        ci_i386 = params.ci_i386_param ?: ci_i386
-        ci_lint = params.ci_lint_param ?: ci_lint
-        ci_minimal = params.ci_minimal_param ?: ci_minimal
-        ci_riscv = params.ci_riscv_param ?: ci_riscv
-        ci_wasm = params.ci_wasm_param ?: ci_wasm
-
-        sh (script: """
-          echo "Docker images being used in this build:"
-          echo " ci_arm = ${ci_arm}"
-          echo " ci_cortexm = ${ci_cortexm}"
-          echo " ci_cpu = ${ci_cpu}"
-          echo " ci_gpu = ${ci_gpu}"
-          echo " ci_hexagon = ${ci_hexagon}"
-          echo " ci_i386 = ${ci_i386}"
-          echo " ci_lint = ${ci_lint}"
-          echo " ci_minimal = ${ci_minimal}"
-          echo " ci_riscv = ${ci_riscv}"
-          echo " ci_wasm = ${ci_wasm}"
-        """, label: 'Docker image names')
-
-        is_docs_only_build = sh (
-          returnStatus: true,
-          script: "./${jenkins_scripts_root}/git_change_docs.sh",
-          label: 'Check for docs only changes',
-        )
-        skip_ci = should_skip_ci(env.CHANGE_ID)
-        skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
-        rebuild_docker_images = sh (
-          returnStatus: true,
-          script: "./${jenkins_scripts_root}/git_change_docker.sh",
-          label: 'Check for any docker changes',
-        )
-
-        if (skip_ci) {
-          // Don't rebuild when skipping CI
-          rebuild_docker_images = false
-        }
-      }
-    }
-  }
-}
-def ci_setup(image) {
-  sh (
-    script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
-    label: 'Clean up old workspace',
-  )
-}
-
-def python_unittest(image) {
-  sh (
-    script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
-    label: 'Run Python unit tests',
-  )
-}
-
-def make_cpp_tests(image, build_dir) {
-  sh (
-    script: """
-      set -eux
-      ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
-        --sccache-bucket tvm-sccache-prod \
-        --sccache-region us-west-2 \
-        --cmake-target cpptest \
-        --build-dir ${build_dir}
-      """,
-    label: 'Make C++ tests',
-  )
-}
-
-def cmake_build(image, path, make_flag) {
-  sh (
-    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir ${path}",
-    label: 'Run cmake build',
-  )
-}
-def cpp_unittest(image) {
-  sh (
-    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
-    label: 'Run C++ tests',
-  )
-}
-
-cancel_previous_build()
-
-try {
-    prepare('CPU-SMALL-SPOT')
-} catch(Exception ex) {
-  prepare('CPU-SMALL')
-}
-def build(node_type) {
-  stage('Build') {
-    if (!skip_ci && is_docs_only_build != 1) {
-      node(node_type) {
-        ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu-minimal") {
-          init_git()
-          docker_init(ci_minimal)
-          timeout(time: max_time, unit: 'MINUTES') {
-
-            withEnv([
-              'PLATFORM=minimal',
-              ], {
-              sh (
-          script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal.sh build",
-          label: 'Create CPU minimal cmake config',
-        )
-        cmake_build(ci_minimal, 'build', '-j2')
-        make_cpp_tests(ci_minimal, 'build')
-        sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu-minimal --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/crttest build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/build.ninja build/microtvm_template_projects",
-            label: 'Upload artifacts to S3',
-          )
-            })
-          }
-        }
-      }
-    } else {
-      Utils.markStageSkippedForConditional('BUILD: CPU MINIMAL')
-    }
-  }
-}
-try {
-    build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
-    build('CPU-SMALL')
-}
-
-
-
-
-def shard_run_unittest_CPU_MINIMAL_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu-minimal") {
-        try {
-          init_git()
-          docker_init(ci_minimal)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=minimal',
-              'TEST_STEP_NAME=unittest: CPU MINIMAL',
-              'TVM_NUM_SHARDS=1',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu-minimal",
-                  label: 'Download artifacts from S3',
-                )
-
-              cpp_unittest(ci_minimal)
-              python_unittest(ci_minimal)
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU_MINIMAL --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('unittest: CPU MINIMAL 1 of 1')
-  }
-}
-
-
-
-def test() {
-  stage('Test') {
-    environment {
-      SKIP_SLOW_TESTS = "${skip_slow_tests}"
-    }
-    parallel(
-    'unittest: CPU MINIMAL 1 of 1': {
-      try {
-      shard_run_unittest_CPU_MINIMAL_1_of_1()
-      } catch (Exception ex) {
-        shard_run_unittest_CPU_MINIMAL_1_of_1(on_demand = true)
-      }
-    },
-    )
-  }
-}
-test()
diff --git a/ci/jenkins/generated/riscv_jenkinsfile.groovy b/ci/jenkins/generated/riscv_jenkinsfile.groovy
deleted file mode 100644
index f836d71630f9..000000000000
--- a/ci/jenkins/generated/riscv_jenkinsfile.groovy
+++ /dev/null
@@ -1,624 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// -*- mode: groovy -*-
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-// Docker env used for testing
-// Different image may have different version tag
-// because some of them are more stable than anoter.
-//
-// Docker images are maintained by PMC, cached in dockerhub
-// and remains relatively stable over the time.
-// Flow for upgrading docker env(need commiter)
-//
-// - Send PR to upgrade build script in the repo
-// - Build the new docker image
-// - Tag the docker image with a new version and push to a binary cache.
-// - Update the version in the Jenkinsfile, send a PR
-// - Fix any issues wrt to the new image version in the PR
-// - Merge the PR and now we are in new version
-// - Tag the new version as the lates
-// - Periodically cleanup the old versions on local workers
-//
-
-// ============================= IMPORTANT NOTE =============================
-// This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
-// Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
-// 'python3 jenkins/generate.py'
-// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
-// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.070888
-
-import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-// These are set at runtime from data in ci/jenkins/docker-images.yml, update
-// image tags in that file
-ci_lint = ''
-ci_gpu = ''
-ci_cpu = ''
-ci_minimal = ''
-ci_wasm = ''
-ci_i386 = ''
-ci_cortexm = ''
-ci_arm = ''
-ci_hexagon = ''
-ci_riscv = ''
-
-// Parameters to allow overriding (in Jenkins UI), the images
-// to be used by a given build. When provided, they take precedence
-// over default values above.
-properties([
-  parameters([
-    string(name: 'ci_arm_param', defaultValue: ''),
-    string(name: 'ci_cortexm_param', defaultValue: ''),
-    string(name: 'ci_cpu_param', defaultValue: ''),
-    string(name: 'ci_gpu_param', defaultValue: ''),
-    string(name: 'ci_hexagon_param', defaultValue: ''),
-    string(name: 'ci_i386_param', defaultValue: ''),
-    string(name: 'ci_lint_param', defaultValue: ''),
-    string(name: 'ci_minimal_param', defaultValue: ''),
-    string(name: 'ci_riscv_param', defaultValue: ''),
-    string(name: 'ci_wasm_param', defaultValue: ''),
-  ])
-])
-
-// Placeholders for newly built Docker image names (if rebuild_docker_images
-// is used)
-  built_ci_arm = null;
-  built_ci_cortexm = null;
-  built_ci_cpu = null;
-  built_ci_gpu = null;
-  built_ci_hexagon = null;
-  built_ci_i386 = null;
-  built_ci_lint = null;
-  built_ci_minimal = null;
-  built_ci_riscv = null;
-  built_ci_wasm = null;
-
-// Global variable assigned during Sanity Check that holds the sha1 which should be
-// merged into the PR in all branches.
-upstream_revision = null
-
-// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
-docker_build = 'docker/build.sh'
-// timeout in minutes
-max_time = 180
-rebuild_docker_images = false
-
-s3_bucket = 'tvm-jenkins-artifacts-prod'
-s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
-// Jenkins script root directory
-jenkins_scripts_root = "ci/scripts/jenkins"
-
-
-// General note: Jenkins has limits on the size of a method (or top level code)
-// that are pretty strict, so most usage of groovy methods in these templates
-// are purely to satisfy the JVM
-def per_exec_ws(folder) {
-  return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
-}
-
-// initialize source codes
-def init_git() {
-  retry(5) {
-    checkout scm
-  }
-
-  // Add more info about job node
-  sh (
-    script: './tests/scripts/task_show_node_info.sh',
-    label: 'Show executor node info',
-  )
-
-  // Determine merge commit to use for all stages
-  if (env.BRANCH_NAME == 'main') {
-    // Only set upstream_revision to HEAD and skip merging to avoid a race with another commit merged to main.
-    update_upstream_revision("HEAD")
-  } else {
-    // This is PR branch so merge with latest main.
-    merge_with_main()
-  }
-
-  sh(
-    script: """
-      set -eux
-      . ${jenkins_scripts_root}/retry.sh
-      retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
-    """,
-    label: 'Update git submodules',
-  )
-  checkout_trusted_files()
-}
-
-def update_upstream_revision(git_ref) {
-  if (upstream_revision == null) {
-    upstream_revision = sh(
-      script: "git log -1 ${git_ref} --format=\'%H\'",
-      label: 'Determine upstream revision',
-      returnStdout: true,
-    ).trim()
-  }
-}
-
-def merge_with_main() {
-  sh (
-    script: 'git fetch origin main',
-    label: 'Fetch upstream',
-  )
-  update_upstream_revision("FETCH_HEAD")
-  sh (
-    script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
-    label: 'Merge to origin/main'
-  )
-}
-
-def docker_init(image) {
-  // Clear out all Docker images that aren't going to be used
-  sh(
-    script: """
-    set -eux
-    docker image ls --all
-    IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}}  {{.ID}}')
-
-    echo -e "Found images:\\n\$IMAGES"
-    echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
-    docker image ls --all
-    """,
-    label: 'Clean old Docker images',
-  )
-
-  if (image.contains("amazonaws.com")) {
-    // If this string is in the image name it's from ECR and needs to be pulled
-    // with the right credentials
-    ecr_pull(image)
-  } else {
-    sh(
-      script: """
-      set -eux
-      . ${jenkins_scripts_root}/retry.sh
-      retry 5 docker pull ${image}
-      """,
-      label: 'Pull docker image',
-    )
-  }
-}
-
-def ecr_pull(full_name) {
-  aws_account_id = sh(
-    returnStdout: true,
-    script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
-    label: 'Get AWS ID'
-  ).trim()
-
-  try {
-    withEnv([
-      "AWS_ACCOUNT_ID=${aws_account_id}",
-      'AWS_DEFAULT_REGION=us-west-2',
-      "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
-      sh(
-        script: '''
-          set -eux
-          aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
-        ''',
-        label: 'Log in to ECR'
-      )
-      sh(
-        script: """
-          set -eux
-          . ${jenkins_scripts_root}/retry.sh
-          retry 5 docker pull ${full_name}
-        """,
-        label: 'Pull image from ECR'
-      )
-    }
-  } finally {
-    withEnv([
-      "AWS_ACCOUNT_ID=${aws_account_id}",
-      'AWS_DEFAULT_REGION=us-west-2',
-      "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
-      sh(
-        script: 'docker logout $AWS_ECR_REPO',
-        label: 'Clean up login credentials'
-      )
-    }
-  }
-}
-
-def should_skip_slow_tests(pr_number) {
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-  )]) {
-    // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
-    result = sh (
-      returnStatus: true,
-      script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
-      label: 'Check if CI should run slow tests',
-    )
-  }
-  return result == 0
-}
-
-def cancel_previous_build() {
-  // cancel previous build if it is not on main.
-  if (env.BRANCH_NAME != 'main') {
-    def buildNumber = env.BUILD_NUMBER as int
-    // Milestone API allows us to cancel previous build
-    // with the same milestone number
-    if (buildNumber > 1) milestone(buildNumber - 1)
-    milestone(buildNumber)
-  }
-}
-
-def checkout_trusted_files() {
-  // trust everything from branch builds
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    return;
-  }
-
-  // trust peoople listed in CONTRIBUTING.md
-  grep_code = sh(
-    returnStatus: true,
-    script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
-    label: 'Check if change is from a contributor',
-  )
-
-  if (grep_code == 1) {
-    // Any scripts that run on the bare host and not inside a Docker container
-    // (especially those that access secrets) should be checked out here so
-    // only trusted versions are used in CI
-    sh(
-      script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
-      label: 'Check out trusted files',
-    )
-  }
-}
-
-def should_skip_ci(pr_number) {
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    // never skip CI on build sourced from a branch
-    return false
-  }
-  glob_skip_ci_code = sh (
-    returnStatus: true,
-    script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
-    label: 'Check if CI should be skipped due to changed files',
-  )
-  if (glob_skip_ci_code == 0) {
-    return true
-  }
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-    )]) {
-    // Exit code of 1 means run full CI (or the script had an error, so run
-    // full CI just in case). Exit code of 0 means skip CI.
-    git_skip_ci_code = sh (
-      returnStatus: true,
-      script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
-      label: 'Check if CI should be skipped',
-    )
-  }
-  return git_skip_ci_code == 0
-}
-
-def check_pr(pr_number) {
-  if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
-    // never skip CI on build sourced from a branch
-    return false
-  }
-  withCredentials([string(
-    credentialsId: 'tvm-bot-jenkins-reader',
-    variable: 'GITHUB_TOKEN',
-    )]) {
-    sh (
-      script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}",
-      label: 'Check PR title and body',
-    )
-  }
-
-}
-
-def prepare(node_type) {
-  stage('Prepare') {
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
-        init_git()
-
-        check_pr(env.CHANGE_ID)
-
-        if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
-          sh(
-            script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
-            label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
-          )
-          // Pull image names from the results of should_rebuild_docker.py
-          ci_arm = sh(
-            script: "cat .docker-image-names/ci_arm",
-            label: "Find docker image name for ci_arm",
-            returnStdout: true,
-          ).trim()
-          ci_cortexm = sh(
-            script: "cat .docker-image-names/ci_cortexm",
-            label: "Find docker image name for ci_cortexm",
-            returnStdout: true,
-          ).trim()
-          ci_cpu = sh(
-            script: "cat .docker-image-names/ci_cpu",
-            label: "Find docker image name for ci_cpu",
-            returnStdout: true,
-          ).trim()
-          ci_gpu = sh(
-            script: "cat .docker-image-names/ci_gpu",
-            label: "Find docker image name for ci_gpu",
-            returnStdout: true,
-          ).trim()
-          ci_hexagon = sh(
-            script: "cat .docker-image-names/ci_hexagon",
-            label: "Find docker image name for ci_hexagon",
-            returnStdout: true,
-          ).trim()
-          ci_i386 = sh(
-            script: "cat .docker-image-names/ci_i386",
-            label: "Find docker image name for ci_i386",
-            returnStdout: true,
-          ).trim()
-          ci_lint = sh(
-            script: "cat .docker-image-names/ci_lint",
-            label: "Find docker image name for ci_lint",
-            returnStdout: true,
-          ).trim()
-          ci_minimal = sh(
-            script: "cat .docker-image-names/ci_minimal",
-            label: "Find docker image name for ci_minimal",
-            returnStdout: true,
-          ).trim()
-          ci_riscv = sh(
-            script: "cat .docker-image-names/ci_riscv",
-            label: "Find docker image name for ci_riscv",
-            returnStdout: true,
-          ).trim()
-          ci_wasm = sh(
-            script: "cat .docker-image-names/ci_wasm",
-            label: "Find docker image name for ci_wasm",
-            returnStdout: true,
-          ).trim()
-        }
-
-        ci_arm = params.ci_arm_param ?: ci_arm
-        ci_cortexm = params.ci_cortexm_param ?: ci_cortexm
-        ci_cpu = params.ci_cpu_param ?: ci_cpu
-        ci_gpu = params.ci_gpu_param ?: ci_gpu
-        ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
-        ci_i386 = params.ci_i386_param ?: ci_i386
-        ci_lint = params.ci_lint_param ?: ci_lint
-        ci_minimal = params.ci_minimal_param ?: ci_minimal
-        ci_riscv = params.ci_riscv_param ?: ci_riscv
-        ci_wasm = params.ci_wasm_param ?: ci_wasm
-
-        sh (script: """
-          echo "Docker images being used in this build:"
-          echo " ci_arm = ${ci_arm}"
-          echo " ci_cortexm = ${ci_cortexm}"
-          echo " ci_cpu = ${ci_cpu}"
-          echo " ci_gpu = ${ci_gpu}"
-          echo " ci_hexagon = ${ci_hexagon}"
-          echo " ci_i386 = ${ci_i386}"
-          echo " ci_lint = ${ci_lint}"
-          echo " ci_minimal = ${ci_minimal}"
-          echo " ci_riscv = ${ci_riscv}"
-          echo " ci_wasm = ${ci_wasm}"
-        """, label: 'Docker image names')
-
-        is_docs_only_build = sh (
-          returnStatus: true,
-          script: "./${jenkins_scripts_root}/git_change_docs.sh",
-          label: 'Check for docs only changes',
-        )
-        skip_ci = should_skip_ci(env.CHANGE_ID)
-        skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
-        rebuild_docker_images = sh (
-          returnStatus: true,
-          script: "./${jenkins_scripts_root}/git_change_docker.sh",
-          label: 'Check for any docker changes',
-        )
-
-        if (skip_ci) {
-          // Don't rebuild when skipping CI
-          rebuild_docker_images = false
-        }
-      }
-    }
-  }
-}
-def ci_setup(image) {
-  sh (
-    script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
-    label: 'Clean up old workspace',
-  )
-}
-
-def python_unittest(image) {
-  sh (
-    script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
-    label: 'Run Python unit tests',
-  )
-}
-
-def make_cpp_tests(image, build_dir) {
-  sh (
-    script: """
-      set -eux
-      ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
-        --sccache-bucket tvm-sccache-prod \
-        --sccache-region us-west-2 \
-        --cmake-target cpptest \
-        --build-dir ${build_dir}
-      """,
-    label: 'Make C++ tests',
-  )
-}
-
-def cmake_build(image, path, make_flag) {
-  sh (
-    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir ${path}",
-    label: 'Run cmake build',
-  )
-}
-def cpp_unittest(image) {
-  sh (
-    script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
-    label: 'Run C++ tests',
-  )
-}
-
-cancel_previous_build()
-
-try {
-    prepare('CPU-SMALL-SPOT')
-} catch(Exception ex) {
-  prepare('CPU-SMALL')
-}
-def build(node_type) {
-  stage('Build') {
-    if (!skip_ci && is_docs_only_build != 1) {
-      node(node_type) {
-        ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-riscv") {
-          init_git()
-          docker_init(ci_riscv)
-          timeout(time: max_time, unit: 'MINUTES') {
-
-            withEnv([
-              'PLATFORM=riscv',
-              ], {
-              sh (
-          script: "${docker_run} ${ci_riscv} ./tests/scripts/task_config_build_riscv.sh build",
-          label: 'Create RISC-V cmake config',
-        )
-        cmake_build(ci_riscv, 'build', '-j2')
-        make_cpp_tests(ci_riscv, 'build')
-        sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/riscv --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/build.ninja build/crttest build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/microtvm_template_projects",
-            label: 'Upload artifacts to S3',
-          )
-            })
-          }
-        }
-      }
-    } else {
-      Utils.markStageSkippedForConditional('BUILD: RISC-V')
-    }
-  }
-}
-try {
-    build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
-    build('CPU-SMALL')
-}
-
-
-
-
-
-def shard_run_test_RISC_V_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
-  if (!skip_ci && is_docs_only_build != 1) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = 'CPU-SMALL'
-    }
-    node(node_type) {
-      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-riscv") {
-        try {
-          init_git()
-          docker_init(ci_riscv)
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM=riscv',
-              'TEST_STEP_NAME=test: RISC-V',
-              'TVM_NUM_SHARDS=1',
-              'TVM_SHARD_INDEX=0',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              sh(
-                  script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/riscv",
-                  label: 'Download artifacts from S3',
-                )
-
-              ci_setup(ci_riscv)
-              cpp_unittest(ci_riscv)
-              sh (
-                script: "${docker_run} ${ci_riscv} ./tests/scripts/task_riscv_microtvm.sh",
-                label: 'Run microTVM tests',
-              )
-            })
-          }
-        } finally {
-          try {
-            sh(
-            script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_RISC_V --items build/pytest-results",
-            label: 'Upload JUnits to S3',
-          )
-
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
-        }
-      }
-    }
-  } else {
-    Utils.markStageSkippedForConditional('test: RISC-V 1 of 1')
-  }
-}
-
-
-def test() {
-  stage('Test') {
-    environment {
-      SKIP_SLOW_TESTS = "${skip_slow_tests}"
-    }
-    parallel(
-    'test: RISC-V 1 of 1': {
-      try {
-      shard_run_test_RISC_V_1_of_1()
-      } catch (Exception ex) {
-        shard_run_test_RISC_V_1_of_1(on_demand = true)
-      }
-    },
-    )
-  }
-}
-test()
diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy
index da7cdfd5c1f3..4cf67de233c9 100644
--- a/ci/jenkins/generated/wasm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy
@@ -60,7 +60,7 @@
 // 'python3 jenkins/generate.py'
 // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
 // always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.155555
+// Generated at 2025-02-06T20:53:50.442520
 
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 // These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -514,19 +514,19 @@ try {
 } catch(Exception ex) {
   prepare('CPU-SMALL')
 }
-def build(node_type) {
-  stage('Build') {
-    if (!skip_ci && is_docs_only_build != 1) {
-      node(node_type) {
-        ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") {
-          init_git()
-          docker_init(ci_wasm)
-          timeout(time: max_time, unit: 'MINUTES') {
-
-            withEnv([
-              'PLATFORM=wasm',
-              ], {
-              sh (
+def run_build(node_type) {
+  if (!skip_ci && is_docs_only_build != 1) {
+    echo 'Begin running node_type ' + node_type
+    node(node_type) {
+      ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") {
+        init_git()
+        docker_init(ci_wasm)
+        timeout(time: max_time, unit: 'MINUTES') {
+
+          withEnv([
+            'PLATFORM=wasm',
+            ], {
+            sh (
           script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
           label: 'Create WASM cmake config',
         )
@@ -538,17 +538,26 @@ def build(node_type) {
           script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh",
           label: 'Run WASM lint and tests',
         )
-            })
-          }
+          })
         }
       }
-    } else {
-      Utils.markStageSkippedForConditional('BUILD: WASM')
     }
+    echo 'End running node_type ' + node_type
+  } else {
+    Utils.markStageSkippedForConditional('BUILD: WASM')
   }
 }
-try {
-    build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
-    build('CPU-SMALL')
+def build() {
+  stage('Build') {
+    try {
+        run_build('CPU-SMALL-SPOT')
+    } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        run_build('CPU-SMALL')
+    }
+  }
 }
+build()
diff --git a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2 b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
index c512ddd0ce12..418f786743bd 100644
--- a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
   )
   cmake_build(ci_arm, 'build', '-j4')
   make_cpp_tests(ci_arm, 'build')
-  {{ m.upload_artifacts(tag='arm', filenames=tvm_multilib + cpptest + crttest + microtvm_template_projects) }}
+  {{ m.upload_artifacts(tag='arm', filenames=tvm_lib + cpptest) }}
 {% endcall %}
 
 {% set test_method_names = [] %}
@@ -39,7 +39,6 @@
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="integration: aarch64",
   num_shards=4,
-  node="ARM-GRAVITON3",
   ws="tvm/ut-python-arm",
   platform="arm",
   docker_image="ci_arm",
@@ -56,7 +55,6 @@
 
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="topi: aarch64",
-  node="ARM-GRAVITON3",
   ws="tvm/ut-python-arm",
   platform="arm",
   docker_image="ci_arm",
@@ -80,7 +78,6 @@
 
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="frontend: aarch64",
-  node="ARM-GRAVITON3",
   ws="tvm/frontend-python-arm",
   platform="arm",
   docker_image="ci_arm",
@@ -95,4 +92,4 @@
   )
 {% endcall %}
 
-{{ m.invoke_tests(test_method_names) -}}
+{{ m.invoke_tests(node="ARM-GRAVITON3", test_method_names=test_method_names) -}}
diff --git a/ci/jenkins/templates/cortexm_jenkinsfile.groovy.j2 b/ci/jenkins/templates/cortexm_jenkinsfile.groovy.j2
deleted file mode 100644
index 4f3830670114..000000000000
--- a/ci/jenkins/templates/cortexm_jenkinsfile.groovy.j2
+++ /dev/null
@@ -1,63 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-{% include "utils/base.groovy.j2" with context %}
-{% import 'utils/macros.j2' as m with context -%}
-
-{% call m.invoke_build(
-  name='BUILD: Cortex-M',
-  node='CPU-SMALL',
-  condition='!skip_ci && is_docs_only_build != 1',
-  ws='tvm/build-cortexm',
-  docker_image='ci_cortexm',
-  platform="cortexm",
-) %}
-  sh (
-    script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_config_build_cortexm.sh build",
-    label: 'Create Cortex-M cmake config',
-  )
-  cmake_build(ci_cortexm, 'build', '-j2')
-  make_cpp_tests(ci_cortexm, 'build')
-  {{ m.upload_artifacts(tag='cortexm', filenames=tvm_lib + tvm_allvisible + crttest + cpptest + microtvm_template_projects) }}
-{% endcall %}
-
-{% set test_method_names = [] %}
-
-{% call(shard_index, num_shards) m.sharded_test_step(
-  name="test: Cortex-M",
-  node="CPU-SMALL",
-  ws="tvm/test-cortexm",
-  platform="cortexm",
-  docker_image="ci_cortexm",
-  num_shards=12,
-  test_method_names=test_method_names,
-) %}
-  {{ m.download_artifacts(tag='cortexm') }}
-  ci_setup(ci_cortexm)
-  {% if shard_index == 1%}
-  cpp_unittest(ci_cortexm)
-  sh (
-    script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_demo_microtvm.sh",
-    label: 'Run microTVM demos',
-  )
-  {% endif %}
-  sh (
-    script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
-    label: 'Run microTVM tests',
-  )
-{% endcall %}
-
-{{ m.invoke_tests(test_method_names) -}}
diff --git a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
index 4d6081f3f4af..7e578884f8cd 100644
--- a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
   )
   cmake_build(ci_cpu, 'build', '-j2')
   make_cpp_tests(ci_cpu, 'build')
-  {{ m.upload_artifacts(tag='cpu', filenames=tvm_multilib_tsim + tvm_allvisible + crttest + cpptest + microtvm_template_projects) }}
+  {{ m.upload_artifacts(tag='cpu', filenames=tvm_lib + tvm_allvisible + cpptest) }}
   ci_setup(ci_cpu)
   // sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh"
   // TODO(@jroesch): need to resolve CI issue will turn back on in follow up patch
@@ -42,7 +42,6 @@
 
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="integration: CPU",
-  node="CPU-SMALL",
   num_shards=4,
   ws="tvm/integration-python-cpu",
   platform="cpu",
@@ -59,7 +58,6 @@
 
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="unittest: CPU",
-  node="CPU-SMALL",
   ws="tvm/ut-python-cpu",
   platform="cpu",
   num_shards=1,
@@ -73,7 +71,6 @@
 {% endcall %}
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="frontend: CPU",
-  node="CPU-SMALL",
   ws="tvm/frontend-python-cpu",
   platform="cpu",
   num_shards=1,
@@ -88,4 +85,4 @@
   )
 {% endcall %}
 
-{{ m.invoke_tests(test_method_names) -}}
+{{ m.invoke_tests(node="CPU-SMALL", test_method_names=test_method_names) -}}
diff --git a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
index b6c7c2cecb60..fdef5c96e616 100644
--- a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
@@ -19,7 +19,7 @@
 
 {% call m.invoke_build(
   name='BUILD: GPU',
-  node='CPU-SMALL',
+  node='CPU',
   condition='!skip_ci',
   ws='tvm/build-gpu',
   docker_image='ci_gpu',
@@ -27,13 +27,13 @@
 ) %}
   sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
   cmake_build("${ci_gpu} --no-gpu", 'build', '-j2')
-  {{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib + tvm_allvisible + microtvm_template_projects + crttest) }}
+  {{ m.upload_artifacts(tag='gpu', filenames=tvm_lib + tvm_allvisible + tvm_lib_gpu_extra) }}
 
   // compiler test
   sh "rm -rf build"
   sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
   cmake_build("${ci_gpu} --no-gpu", 'build', '-j2')
-  {{ m.upload_artifacts(tag='gpu2', filenames=tvm_lib + crttest) }}
+  {{ m.upload_artifacts(tag='gpu2', filenames=tvm_lib) }}
 {% endcall %}
 
 {% set test_method_names = [] %}
@@ -41,7 +41,6 @@
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="unittest: GPU",
   num_shards=3,
-  node="GPU",
   ws="tvm/ut-python-gpu",
   platform="gpu",
   docker_image="ci_gpu",
@@ -90,7 +89,6 @@
 
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="topi: GPU",
-  node="GPU",
   num_shards=3,
   ws="tvm/topi-python-gpu",
   platform="gpu",
@@ -105,26 +103,8 @@
   )
 {% endcall %}
 
-{% call(shard_index, num_shards) m.sharded_test_step(
-  name="frontend: GPU",
-  node="GPU",
-  num_shards=6,
-  ws="tvm/frontend-python-gpu",
-  platform="gpu",
-  docker_image="ci_gpu",
-  test_method_names=test_method_names,
-) %}
-  {{ m.download_artifacts(tag='gpu') }}
-  ci_setup(ci_gpu)
-  sh (
-    script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
-    label: 'Run Python frontend tests',
-  )
-{% endcall %}
-
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="docs: GPU",
-  node="GPU",
   num_shards=1,
   ws="tvm/docs-python-gpu",
   platform="gpu",
@@ -146,7 +126,7 @@
 {% endcall %}
 
 
-{{ m.invoke_tests(test_method_names) }}
+{{ m.invoke_tests(node="GPU", test_method_names=test_method_names) }}
 
 
 def deploy_docs() {
diff --git a/ci/jenkins/templates/hexagon_jenkinsfile.groovy.j2 b/ci/jenkins/templates/hexagon_jenkinsfile.groovy.j2
index 94ea7dba04d6..1d3f37131c58 100644
--- a/ci/jenkins/templates/hexagon_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/hexagon_jenkinsfile.groovy.j2
@@ -43,7 +43,6 @@
 
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="test: Hexagon",
-  node="CPU-SMALL",
   ws="tvm/test-hexagon",
   platform="hexagon",
   docker_image="ci_hexagon",
@@ -61,4 +60,4 @@
   )
 {% endcall %}
 
-{{ m.invoke_tests(test_method_names) -}}
+{{ m.invoke_tests(node="CPU-SMALL", test_method_names=test_method_names) -}}
diff --git a/ci/jenkins/templates/i386_jenkinsfile.groovy.j2 b/ci/jenkins/templates/i386_jenkinsfile.groovy.j2
index 5b181367452d..382e3bfbe6e7 100644
--- a/ci/jenkins/templates/i386_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/i386_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
   )
   cmake_build(ci_i386, 'build', '-j2')
   make_cpp_tests(ci_i386, 'build')
-  {{ m.upload_artifacts(tag='i386', filenames=tvm_multilib_tsim + crttest + cpptest + microtvm_template_projects) }}
+  {{ m.upload_artifacts(tag='i386', filenames=tvm_lib + cpptest) }}
 {% endcall %}
 
 
@@ -39,7 +39,6 @@
 
 {% call(shard_index, num_shards) m.sharded_test_step(
   name="python: i386",
-  node="CPU-SMALL",
   num_shards=3,
   ws="tvm/integration-python-i386",
   platform="i386",
@@ -58,4 +57,4 @@
   )
 {% endcall %}
 
-{{ m.invoke_tests(test_method_names) -}}
+{{ m.invoke_tests(node="CPU-SMALL", test_method_names=test_method_names) -}}
diff --git a/ci/jenkins/templates/minimal_cross_isa_jenkinsfile.groovy.j2 b/ci/jenkins/templates/minimal_cross_isa_jenkinsfile.groovy.j2
deleted file mode 100644
index 07c3890dfce5..000000000000
--- a/ci/jenkins/templates/minimal_cross_isa_jenkinsfile.groovy.j2
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-{% include "utils/base.groovy.j2" with context %}
-{% import 'utils/macros.j2' as m with context -%}
-
-{% call m.invoke_build(
-  name='BUILD: CPU MINIMAL CROSS ISA',
-  node='CPU-SMALL',
-  condition='!skip_ci && is_docs_only_build != 1',
-  ws='tvm/build-cpu-minimal-cross-isa',
-  docker_image='ci_minimal',
-  platform="arm",
-) %}
-  sh (
-    script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal_cross_isa.sh build",
-    label: 'Create CPU minimal cmake config',
-  )
-  cmake_build(ci_minimal, 'build', '-j2')
-  {{ m.upload_artifacts(tag='cpu-minimal-cross-isa', filenames=tvm_lib + tvm_allvisible + microtvm_template_projects) }}
-{% endcall %}
-
-
-{% set test_method_names = [] %}
-
-{% call(shard_index, num_shards) m.sharded_test_step(
-  name="unittest: CPU MINIMAL CROSS ISA",
-  node="ARM-SMALL",
-  num_shards=1,
-  ws="tvm/ut-cpp-arm-cross-isa",
-  platform="arm",
-  docker_image="ci_arm",
-  test_method_names=test_method_names,
-) %}
-  {{ m.download_artifacts(tag='cpu-minimal-cross-isa') }}
-  ci_setup(ci_arm)
-  sh "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_minimal_cross_isa.sh build"
-  make_cpp_tests(ci_arm, 'build')
-  cpp_unittest(ci_arm)
-  python_unittest(ci_arm)
-{% endcall %}
-
-
-{{ m.invoke_tests(test_method_names) -}}
diff --git a/ci/jenkins/templates/minimal_jenkinsfile.groovy.j2 b/ci/jenkins/templates/minimal_jenkinsfile.groovy.j2
deleted file mode 100644
index dd4dd02fc85d..000000000000
--- a/ci/jenkins/templates/minimal_jenkinsfile.groovy.j2
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-{% include "utils/base.groovy.j2" with context %}
-{% import 'utils/macros.j2' as m with context -%}
-
-{% call m.invoke_build(
-  name='BUILD: CPU MINIMAL',
-  node='CPU-SMALL',
-  condition='!skip_ci && is_docs_only_build != 1',
-  ws='tvm/build-cpu-minimal',
-  docker_image='ci_minimal',
-  platform="minimal",
-) %}
-  sh (
-    script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal.sh build",
-    label: 'Create CPU minimal cmake config',
-  )
-  cmake_build(ci_minimal, 'build', '-j2')
-  make_cpp_tests(ci_minimal, 'build')
-  {{ m.upload_artifacts(tag='cpu-minimal', filenames=tvm_lib + tvm_allvisible + crttest + cpptest + microtvm_template_projects) }}
-{% endcall %}
-
-
-{% set test_method_names = [] %}
-
-{% call(shard_index, num_shards) m.sharded_test_step(
-  name="unittest: CPU MINIMAL",
-  node="CPU-SMALL",
-  num_shards=1,
-  ws="tvm/ut-python-cpu-minimal",
-  platform="minimal",
-  docker_image="ci_minimal",
-  test_method_names=test_method_names,
-) %}
-  {{ m.download_artifacts(tag='cpu-minimal') }}
-  cpp_unittest(ci_minimal)
-  python_unittest(ci_minimal)
-{% endcall %}
-
-
-{{ m.invoke_tests(test_method_names) -}}
diff --git a/ci/jenkins/templates/riscv_jenkinsfile.groovy.j2 b/ci/jenkins/templates/riscv_jenkinsfile.groovy.j2
deleted file mode 100644
index 902e912f7561..000000000000
--- a/ci/jenkins/templates/riscv_jenkinsfile.groovy.j2
+++ /dev/null
@@ -1,61 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-{% include "utils/base.groovy.j2" with context %}
-{% import 'utils/macros.j2' as m with context -%}
-
-{% call m.invoke_build(
-  name='BUILD: RISC-V',
-  node='CPU-SMALL',
-  condition='!skip_ci && is_docs_only_build != 1',
-  ws='tvm/build-riscv',
-  docker_image='ci_riscv',
-  platform="riscv",
-) %}
-  sh (
-    script: "${docker_run} ${ci_riscv} ./tests/scripts/task_config_build_riscv.sh build",
-    label: 'Create RISC-V cmake config',
-  )
-  cmake_build(ci_riscv, 'build', '-j2')
-  make_cpp_tests(ci_riscv, 'build')
-  {{ m.upload_artifacts(tag='riscv', filenames=tvm_lib + tvm_allvisible + crttest + cpptest + microtvm_template_projects) }}
-{% endcall %}
-
-
-
-{% set test_method_names = [] %}
-
-{% call(shard_index, num_shards) m.sharded_test_step(
-  name="test: RISC-V",
-  node="CPU-SMALL",
-  ws="tvm/test-riscv",
-  platform="riscv",
-  docker_image="ci_riscv",
-  num_shards=1,
-  test_method_names=test_method_names,
-) %}
-  {{ m.download_artifacts(tag='riscv') }}
-  ci_setup(ci_riscv)
-  {% if shard_index == 1%}
-  cpp_unittest(ci_riscv)
-  {% endif %}
-  sh (
-    script: "${docker_run} ${ci_riscv} ./tests/scripts/task_riscv_microtvm.sh",
-    label: 'Run microTVM tests',
-  )
-{% endcall %}
-
-{{ m.invoke_tests(test_method_names) -}}
diff --git a/ci/jenkins/templates/utils/macros.j2 b/ci/jenkins/templates/utils/macros.j2
index 6de4bd6d656d..81eeaa1fdff8 100644
--- a/ci/jenkins/templates/utils/macros.j2
+++ b/ci/jenkins/templates/utils/macros.j2
@@ -26,41 +26,39 @@ sh(
           )
 {% endmacro %}
 
-{% macro sharded_test_step(name, num_shards, node, ws, docker_image, platform, test_method_names, condition="!skip_ci && is_docs_only_build != 1") %}
+{% macro sharded_test_step(name, num_shards, ws, docker_image, platform, test_method_names, condition="!skip_ci && is_docs_only_build != 1") %}
 
 {% for shard_index in range(1, num_shards + 1) %}
 {% set method_name = "shard_run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") + "_" + shard_index|string + "_of_" + num_shards|string %}
 {% set test_dir_name = name.replace(":", "").replace(" ", "-").replace("-", "_")|string %}
-def {{ method_name }}(node_type='{{ node }}-SPOT', on_demand=false) {
+def {{ method_name }}(node_type) {
+  echo 'Begin running on node_type ' + node_type
   if ({{ condition }}) {
-    if (on_demand==true || node_type.contains('ARM')) {
-        node_type = '{{ node }}'
-    }
     node(node_type) {
       ws({{ per_exec_ws(ws) }}) {
+        // NOTE: if exception happens, it will be caught outside
+        init_git()
+        docker_init({{ docker_image }})
+        timeout(time: max_time, unit: 'MINUTES') {
+          withEnv([
+            'PLATFORM={{ platform }}',
+            'TEST_STEP_NAME={{ name }}',
+            'TVM_NUM_SHARDS={{ num_shards }}',
+            'TVM_SHARD_INDEX={{ shard_index - 1 }}',
+            "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+            {{ caller(shard_index, num_shards) | trim | indent(width=12) }}
+          })
+        }
+        // only run upload if things are successful
         try {
-          init_git()
-          docker_init({{ docker_image }})
-          timeout(time: max_time, unit: 'MINUTES') {
-            withEnv([
-              'PLATFORM={{ platform }}',
-              'TEST_STEP_NAME={{ name }}',
-              'TVM_NUM_SHARDS={{ num_shards }}',
-              'TVM_SHARD_INDEX={{ shard_index - 1 }}',
-              "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
-              {{ caller(shard_index, num_shards) | trim | indent(width=12) }}
-            })
-          }
-        } finally {
-          try {
-            {{ junit_to_s3(test_dir_name) }}
-            junit 'build/pytest-results/*.xml'
-          } catch (Exception e) {
-            echo 'Exception during JUnit upload: ' + e.toString()
-          }
+          {{ junit_to_s3(test_dir_name) }}
+          junit 'build/pytest-results/*.xml'
+        } catch (Exception e) {
+          echo 'Exception during JUnit upload: ' + e.toString()
         }
       }
     }
+    echo 'End running on node_type ' + node_type
   } else {
     Utils.markStageSkippedForConditional('{{ name }} {{ shard_index }} of {{ num_shards }}')
   }
@@ -71,36 +69,45 @@ def {{ method_name }}(node_type='{{ node }}-SPOT', on_demand=false) {
 {% endmacro %}
 
 {% macro invoke_build(name, condition, node, docker_image, ws, platform) %}
-def build(node_type) {
-  stage('Build') {
-    if ({{ condition }}) {
-      node(node_type) {
-        ws({{ per_exec_ws(ws) }}) {
-          init_git()
-          docker_init({{ docker_image }})
-          timeout(time: max_time, unit: 'MINUTES') {
+def run_build(node_type) {
+  if ({{ condition }}) {
+    echo 'Begin running node_type ' + node_type
+    node(node_type) {
+      ws({{ per_exec_ws(ws) }}) {
+        init_git()
+        docker_init({{ docker_image }})
+        timeout(time: max_time, unit: 'MINUTES') {
 
-            withEnv([
-              'PLATFORM={{ platform }}',
-              ], {
-              {{ caller() | trim | indent(width=6) }}
-            })
-          }
+          withEnv([
+            'PLATFORM={{ platform }}',
+            ], {
+            {{ caller() | trim | indent(width=6) }}
+          })
         }
       }
-    } else {
-      Utils.markStageSkippedForConditional('{{ name }}')
     }
+    echo 'End running node_type ' + node_type
+  } else {
+    Utils.markStageSkippedForConditional('{{ name }}')
   }
 }
-try {
-    build('{{ node }}-SPOT')
-} catch (Exception ex) {
-    build('{{ node }}')
+def build() {
+  stage('Build') {
+    try {
+        run_build('{{ node }}-SPOT')
+    } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        run_build('{{ node }}')
+    }
+  }
 }
+build()
 {% endmacro %}
 
-{% macro invoke_tests(test_method_names) %}
+{% macro invoke_tests(node, test_method_names) %}
 def test() {
   stage('Test') {
     environment {
@@ -110,9 +117,13 @@ def test() {
     {% for stage_name, method_name in test_method_names %}
     '{{ stage_name }}': {
       try {
-      {{ method_name }}()
-      } catch (Exception ex) {
-        {{ method_name }}(on_demand = true)
+      {{ method_name }}('{{ node }}-SPOT')
+      } catch (Throwable ex) {
+        // mark the current stage as success
+        // and try again via on demand node
+        echo 'Exception during SPOT run ' + ex.toString() + ' retry on-demand'
+        currentBuild.result = 'SUCCESS'
+        {{ method_name }}('{{ node }}')
       }
     },
     {% endfor %}
diff --git a/ci/jenkins/unity_jenkinsfile.groovy b/ci/jenkins/unity_jenkinsfile.groovy
index d6a5d46f6f8d..a0c1120fa3e7 100755
--- a/ci/jenkins/unity_jenkinsfile.groovy
+++ b/ci/jenkins/unity_jenkinsfile.groovy
@@ -348,7 +348,7 @@ stage('Build and Test') {
       }
     },
     'BUILD: CPU': {
-      node('CPU-SMALL-SPOT') {
+      node('CPU-SMALL') {
         ws(per_exec_ws('tvm/build-cpu')) {
           init_git()
           sh "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build"