diff --git a/docker/Dockerfile.ci_minimal b/docker/Dockerfile.ci_minimal
index ec6aa78b2fc0..5ea832454a8b 100644
--- a/docker/Dockerfile.ci_minimal
+++ b/docker/Dockerfile.ci_minimal
@@ -28,6 +28,10 @@ RUN bash /install/ubuntu_setup_tz.sh
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
 RUN bash /install/ubuntu_install_core.sh
 
+# Install libraries for cross-compiling TVM to Aarch64
+COPY install/ubuntu1804_install_aarch64_cross_compile.sh /install/ubuntu1804_install_aarch64_cross_compile.sh
+RUN bash /install/ubuntu1804_install_aarch64_cross_compile.sh
+
 COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
 RUN bash /install/ubuntu_install_cmake_source.sh
 
@@ -47,6 +51,10 @@ RUN bash /install/ubuntu_install_python_package.sh
 COPY install/ubuntu1804_manual_install_llvm.sh /install/ubuntu1804_manual_install_llvm.sh
 RUN bash /install/ubuntu1804_manual_install_llvm.sh
 
+# Cross build LLVM to Aarch64
+COPY install/ubuntu1804_manual_install_llvm_cross_aarch64.sh /install/ubuntu1804_manual_install_llvm_cross_aarch64.sh
+RUN bash /install/ubuntu1804_manual_install_llvm_cross_aarch64.sh
+
 # Rust env (build early; takes a while)
 COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh
 RUN bash /install/ubuntu_install_rust.sh
diff --git a/docker/install/ubuntu1804_install_aarch64_cross_compile.sh b/docker/install/ubuntu1804_install_aarch64_cross_compile.sh
new file mode 100644
index 000000000000..faa4aa3f23e6
--- /dev/null
+++ b/docker/install/ubuntu1804_install_aarch64_cross_compile.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -u
+# Used for debugging RVM build
+set -x
+set -o pipefail
+
+architecture_type=$(uname -i)
+if [ "$architecture_type" != "aarch64" ]; then
+  # Install gcc and g++ for cross-compiling c++ on ubuntu
+  apt-get update && apt-install-and-clear -y --no-install-recommends \
+      g++-aarch64-linux-gnu \
+      gcc-aarch64-linux-gnu \
+
+  # Add Aarch64 packages to the apt sources list
+  echo >> /etc/apt/sources.list.d/arm64.list
+  echo "deb [arch=arm64] http://ports.ubuntu.com/ bionic main restricted" >> /etc/apt/sources.list.d/arm64.list
+  echo "deb [arch=arm64] http://ports.ubuntu.com/ bionic-updates main restricted" >> /etc/apt/sources.list.d/arm64.list
+  echo "deb [arch=arm64] http://ports.ubuntu.com/ bionic universe" >> /etc/apt/sources.list.d/arm64.list
+  echo "deb [arch=arm64] http://ports.ubuntu.com/ bionic-updates universe" >> /etc/apt/sources.list.d/arm64.list
+  echo "deb [arch=arm64] http://ports.ubuntu.com/ bionic multiverse" >> /etc/apt/sources.list.d/arm64.list
+  echo "deb [arch=arm64] http://ports.ubuntu.com/ bionic-updates multiverse" >> /etc/apt/sources.list.d/arm64.list
+  echo "deb [arch=arm64] http://ports.ubuntu.com/ bionic-backports main restricted universe multiverse" >> /etc/apt/sources.list.d/arm64.list
+
+  # Fix apt-get update by specifying the amd64 architecture in sources.list
+  sed -i -e 's/deb /deb [arch=amd64] /g' /etc/apt/sources.list
+
+  # Install the required packages for cross-compiling
+  dpkg --add-architecture arm64
+  apt-get update && apt-install-and-clear -y --no-install-recommends \
+      zlib1g-dev:arm64 \
+      libtinfo-dev:arm64
+
+fi
diff --git a/docker/install/ubuntu1804_manual_install_llvm_cross_aarch64.sh b/docker/install/ubuntu1804_manual_install_llvm_cross_aarch64.sh
new file mode 100644
index 000000000000..b1ef38a4095f
--- /dev/null
+++ b/docker/install/ubuntu1804_manual_install_llvm_cross_aarch64.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -u
+set -o pipefail
+
+architecture_type=$(uname -i)
+# Cross-build LLVM for aarch64 when not building natively.
+if [ "$architecture_type" != "aarch64" ]; then
+  git clone --depth 1 --branch release/11.x https://github.com/llvm/llvm-project.git
+  pushd llvm-project
+
+  # First build clang-tblgen and llvm-tblgen
+  mkdir build-host
+  pushd build-host
+  cmake \
+    -G Ninja \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_ENABLE_ASSERTIONS=ON \
+    -DLLVM_ENABLE_PROJECTS="llvm;clang;clang-tools-extra" \
+    ../llvm
+  ninja clang-tblgen llvm-tblgen
+  popd
+
+  # Then cross-compile LLVM for Aarch64
+  mkdir build
+  pushd build
+  CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ cmake \
+    -G Ninja \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=/usr/llvm-aarch64 \
+    -DLLVM_ENABLE_ASSERTIONS=ON \
+    -DLLVM_ENABLE_PROJECTS="llvm;clang" \
+    -DCMAKE_SYSTEM_NAME=Linux \
+    -DLLVM_TABLEGEN=/llvm-project/build-host/bin/llvm-tblgen \
+    -DCLANG_TABLEGEN=/llvm-project/build-host/bin/clang-tblgen \
+    -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-linux-gnu \
+    -DLLVM_TARGET_ARCH=AArch64 \
+    -DCMAKE_CXX_FLAGS='-march=armv8-a -mtune=cortex-a72' \
+    ../llvm
+  ninja install
+  popd
+  popd
+  rm -rf llvm-project
+
+  # This is a hack. Cross-compiling LLVM with gcc will cause the llvm-config to be an Aarch64 executable.
+  # We need it to be x86 to be able to call it when building TVM. We just copy and use the x86 one instead.
+  cp /usr/bin/llvm-config /usr/llvm-aarch64/bin/llvm-config
+fi