diff --git a/apps/microtvm/cmsisnn/Makefile b/apps/microtvm/cmsisnn/Makefile
index db72ab889663..e7d1b7081d54 100644
--- a/apps/microtvm/cmsisnn/Makefile
+++ b/apps/microtvm/cmsisnn/Makefile
@@ -31,6 +31,11 @@ CMAKE ?= cmake
 CC = arm-none-eabi-gcc
 AR = arm-none-eabi-ar
 RANLIB = arm-none-eabi-ranlib
+ifeq ($(shell [ -d ${CMSIS_PATH}/CMSIS-NN ]; echo $$?), 0)
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS-NN
+else
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS/NN
+endif
 PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	-I${STANDALONE_CRT_PATH}/include \
 	-I${STANDALONE_CRT_PATH}/src/runtime/crt/include \
@@ -38,7 +43,7 @@ PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	-I${CORSTONE_300_PATH} \
 	-I${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Include/ \
 	-I${CMSIS_PATH}/CMSIS/Core/Include \
-	-I${CMSIS_PATH}/CMSIS/NN/Include \
+	-I${CMSIS_NN_PATH}/Include \
 	-I${CMSIS_PATH}/CMSIS/DSP/Include \
 	-I$(abspath $(BUILD_DIR))/codegen/host/include
 CMSIS_NN_CMAKE_FLAGS = -DCMAKE_TOOLCHAIN_FILE=$(abspath $(BUILD_DIR))/../arm-none-eabi-gcc.cmake \
@@ -56,7 +61,7 @@ DEMO_MAIN = src/demo_bare_metal.c
 CODEGEN_SRCS = $(wildcard $(abspath $(BUILD_DIR))/codegen/host/src/*.c)
 CODEGEN_OBJS = $(subst .c,.o,$(CODEGEN_SRCS))
 CMSIS_STARTUP_SRCS = $(wildcard ${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Source/*.c)
-CMSIS_NN_SRCS = $(shell find ${CMSIS_PATH}/CMSIS/NN/Source/*/*.c)
+CMSIS_NN_SRCS = $(shell find ${CMSIS_NN_PATH}/Source/*/*.c)
 UART_SRCS = $(wildcard ${CORSTONE_300_PATH}/*.c)
 
 demo: $(BUILD_DIR)/demo
diff --git a/apps/microtvm/cmsisnn/README.md b/apps/microtvm/cmsisnn/README.md
index befcda6bb063..3f32c44baed5 100644
--- a/apps/microtvm/cmsisnn/README.md
+++ b/apps/microtvm/cmsisnn/README.md
@@ -35,6 +35,7 @@ If the demo is not run in the ci_cpu Docker container, then you will need the fo
   - [GCC toolchain from Arm(R)](https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2)
   - [Arm(R) Ethos(TM)-U NPU driver stack](https://review.mlplatform.org)
   - [CMSIS](https://github.com/ARM-software/CMSIS_5)
+  - [CMSIS NN](https://github.com/ARM-software/CMSIS-NN)
 - The python libraries listed in the requirements.txt of this directory
   - These can be installed by running the following from the current directory:
     ```bash
diff --git a/apps/microtvm/ethosu/Makefile b/apps/microtvm/ethosu/Makefile
index 1b79548eaf62..63f8adbc2790 100644
--- a/apps/microtvm/ethosu/Makefile
+++ b/apps/microtvm/ethosu/Makefile
@@ -32,6 +32,11 @@ CMAKE ?= cmake
 CC = arm-none-eabi-gcc
 AR = arm-none-eabi-ar
 RANLIB = arm-none-eabi-ranlib
+ifeq ($(shell [ -d ${CMSIS_PATH}/CMSIS-NN ]; echo $$?), 0)
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS-NN
+else
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS/NN
+endif
 PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	-I${STANDALONE_CRT_PATH}/include \
 	-I${STANDALONE_CRT_PATH}/src/runtime/crt/include \
@@ -40,7 +45,7 @@ PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	-I${ETHOSU_PATH}/core_driver/include \
 	-I${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Include/ \
 	-I${CMSIS_PATH}/CMSIS/Core/Include \
-	-I${CMSIS_PATH}/CMSIS/NN/Include \
+	-I${CMSIS_NN_PATH}/Include \
 	-I${CMSIS_PATH}/CMSIS/DSP/Include \
 	-I$(abspath $(BUILD_DIR))/codegen/host/include \
 	-DETHOSU_TEST_RUNNER_TOL=${ETHOSU_TEST_RUNNER_TOL}
@@ -78,7 +83,7 @@ endif
 CODEGEN_SRCS = $(wildcard $(abspath $(BUILD_DIR))/codegen/host/src/*.c)
 CODEGEN_OBJS = $(subst .c,.o,$(CODEGEN_SRCS))
 CMSIS_STARTUP_SRCS = $(wildcard ${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Source/*.c)
-CMSIS_NN_SOFTMAX_SRCS = $(shell find ${CMSIS_PATH}/CMSIS/NN/Source/SoftmaxFunctions/*.c)
+CMSIS_NN_SOFTMAX_SRCS = $(shell find ${CMSIS_NN_PATH}/Source/SoftmaxFunctions/*.c)
 UART_SRCS = $(wildcard ${CORSTONE_300_PATH}/*.c)
 
 demo: $(BUILD_DIR)/demo
diff --git a/apps/microtvm/ethosu/README.md b/apps/microtvm/ethosu/README.md
index 1f08928a6ee4..69834837ac42 100644
--- a/apps/microtvm/ethosu/README.md
+++ b/apps/microtvm/ethosu/README.md
@@ -35,6 +35,7 @@ If the demo is not run in the ci_cpu Docker container, then you will need the fo
   - [GCC toolchain from Arm(R)](https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2)
   - [Arm(R) Ethos(TM)-U NPU driver stack](https://review.mlplatform.org)
   - [CMSIS](https://github.com/ARM-software/CMSIS_5)
+  - [CMSIS NN](https://github.com/ARM-software/CMSIS-NN)
 - The python libraries listed in the requirements.txt of this directory
   - These can be installed by running the following from the current directory:
     ```bash
diff --git a/apps/microtvm/zephyr/template_project/CMakeLists.txt.template b/apps/microtvm/zephyr/template_project/CMakeLists.txt.template
index a41d68a134ef..17200f7e9704 100644
--- a/apps/microtvm/zephyr/template_project/CMakeLists.txt.template
+++ b/apps/microtvm/zephyr/template_project/CMakeLists.txt.template
@@ -31,21 +31,27 @@ find_package(Zephyr HINTS $ENV{ZEPHYR_BASE})
 project(microtvm_autogenerated_project)
 
 if(DEFINED CMSIS_PATH)
+  if (EXISTS ${CMSIS_PATH}/CMSIS-NN)
+      set(CMSIS_NN_PATH ${CMSIS_PATH}/CMSIS-NN)
+  else()
+      set(CMSIS_NN_PATH ${CMSIS_PATH}/CMSIS/NN)
+  endif()
+
   file(GLOB_RECURSE cmsis_lib_srcs
-    ${CMSIS_PATH}/CMSIS/NN/Source/ActivationFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/BasicMathFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConcatenationFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/FullyConnectedFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/PoolingFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ReshapeFunctions/*.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/SoftmaxFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/ActivationFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/BasicMathFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/ConcatenationFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/FullyConnectedFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/PoolingFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/ReshapeFunctions/*.c
+    ${CMSIS_NN_PATH}/Source/SoftmaxFunctions/*.c
   )
 
   set(cmsis_includes
     ${CMSIS_PATH}/CMSIS/Core/Include
-    ${CMSIS_PATH}/CMSIS/NN/Include
+    ${CMSIS_NN_PATH}/Include
     ${CMSIS_PATH}/CMSIS/DSP/Include
     ${CMSIS_PATH}/CMSIS/DSP/Include/dsp
   )
diff --git a/apps/microtvm/zephyr_cmsisnn/CMakeLists.txt b/apps/microtvm/zephyr_cmsisnn/CMakeLists.txt
index 0ca000d48e6a..9dec75dc5030 100644
--- a/apps/microtvm/zephyr_cmsisnn/CMakeLists.txt
+++ b/apps/microtvm/zephyr_cmsisnn/CMakeLists.txt
@@ -51,22 +51,34 @@ set(DATA_FILES
     ${CMAKE_CURRENT_BINARY_DIR}/outputs.c
     ${CMAKE_CURRENT_BINARY_DIR}/labels.c
 )
+
+if (EXISTS ${CMSIS_PATH}/CMSIS-NN)
+    set(CMSIS_NN_PATH ${CMSIS_PATH}/CMSIS-NN)
+    set(CMSIS_NN_ADDITIONAL_SOURCES
+        ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_convolve_1x1_s8.c
+        ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_mat_mult_nt_t_s8.c
+    )
+else()
+    set(CMSIS_NN_PATH ${CMSIS_PATH}/CMSIS/NN)
+endif()
+
 set(CMSIS_SOURCES
-    ${CMSIS_PATH}/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_wrapper_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_s8_opt.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_depthwise_conv_nt_t_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_depthwise_conv_nt_t_padded_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_wrapper_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1_x_n_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1x1_s8_fast.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mat_mul_core_4x_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_vec_mat_mult_t_s8.c
-    ${CMSIS_PATH}/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mat_mul_core_1x_s8.c
+    ${CMSIS_NN_PATH}/Source/SoftmaxFunctions/arm_softmax_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_depthwise_conv_wrapper_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_depthwise_conv_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_depthwise_conv_s8_opt.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_depthwise_conv_nt_t_s8.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_depthwise_conv_nt_t_padded_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_convolve_wrapper_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_convolve_1_x_n_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_convolve_1x1_s8_fast.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_convolve_s8.c
+    ${CMSIS_NN_PATH}/Source/ConvolutionFunctions/arm_nn_mat_mult_s8.c
+    ${CMSIS_NN_PATH}/Source/FullyConnectedFunctions/arm_fully_connected_s8.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_mat_mul_core_4x_s8.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_vec_mat_mult_t_s8.c
+    ${CMSIS_NN_PATH}/Source/NNSupportFunctions/arm_nn_mat_mul_core_1x_s8.c
+    ${CMSIS_NN_ADDITIONAL_SOURCES}
 )
 
 add_custom_command(
@@ -93,5 +105,5 @@ target_sources(app PRIVATE
 target_include_directories(app
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/runtime/include ${CMAKE_CURRENT_BINARY_DIR}/codegen/host/include
-    PUBLIC ${CMSIS_PATH}/CMSIS/NN/Include/ ${CMSIS_PATH}/CMSIS/DSP/Include
+    PUBLIC ${CMSIS_NN_PATH}/Include/ ${CMSIS_PATH}/CMSIS/DSP/Include
 )
diff --git a/apps/microtvm/zephyr_cmsisnn/README.md b/apps/microtvm/zephyr_cmsisnn/README.md
index df54acfbc736..334af71d0c77 100644
--- a/apps/microtvm/zephyr_cmsisnn/README.md
+++ b/apps/microtvm/zephyr_cmsisnn/README.md
@@ -37,6 +37,11 @@ Checkout [CMSIS_5](https://github.com/ARM-software/CMSIS_5.git) (default is `/op
 git clone "https://github.com/ARM-software/CMSIS_5.git" cmsis
 ```
 
+Checkout [CMSIS NN](https://github.com/ARM-software/CMSIS-NN.git) (default is `/opt/arm/ethosu/cmsis/CMSIS-NN` to reflect `tlcpack/ci_cortexm`):
+```
+git clone "https://github.com/ARM-software/CMSIS-NN.git" cmsis/CMSIS-NN
+```
+
 And run the demo using `west`, with the path to CMSIS:
 ```
 west build -t run -- -DCMSIS_PATH=/opt/arm/ethosu/cmsis
diff --git a/docker/install/ubuntu_install_cmsis.sh b/docker/install/ubuntu_install_cmsis.sh
index a41e3df0ae55..1561cd3dfda7 100755
--- a/docker/install/ubuntu_install_cmsis.sh
+++ b/docker/install/ubuntu_install_cmsis.sh
@@ -48,4 +48,8 @@ wget ${CMSIS_URL} -O "${DOWNLOAD_PATH}"
 echo "$CMSIS_SHASUM" ${DOWNLOAD_PATH} | sha512sum -c
 tar -xf "${DOWNLOAD_PATH}" -C "${INSTALLATION_PATH}" --strip-components=1
 touch "${INSTALLATION_PATH}"/"${CMSIS_SHA}".sha
+
+CMSIS_NN_TAG="v4.0.0"
+CMSIS_NN_URL="https://github.com/ARM-software/CMSIS-NN.git"
+git clone ${CMSIS_NN_URL} --branch ${CMSIS_NN_TAG} --single-branch ${INSTALLATION_PATH}/CMSIS-NN
 echo "SUCCESS"
diff --git a/docker/install/ubuntu_install_ethosu_driver_stack.sh b/docker/install/ubuntu_install_ethosu_driver_stack.sh
index da2f955d3fb7..0fb35b13e797 100755
--- a/docker/install/ubuntu_install_ethosu_driver_stack.sh
+++ b/docker/install/ubuntu_install_ethosu_driver_stack.sh
@@ -85,6 +85,9 @@ cmake -DCMAKE_TOOLCHAIN_FILE=${ethosu_dir}/core_platform/cmake/toolchain/arm-non
 make
 
 # Build NN Library
+mkdir ${CMSIS_PATH}/CMSIS-NN/build/ && cd ${CMSIS_PATH}/CMSIS-NN/build/
+cmake .. -DCMAKE_TOOLCHAIN_FILE=${ethosu_dir}/core_platform/cmake/toolchain/arm-none-eabi-gcc.cmake -DTARGET_CPU=cortex-m55 -DBUILD_CMSIS_NN_FUNCTIONS=YES -DCMSIS_PATH=${CMSIS_PATH}
+
 mkdir ${CMSIS_PATH}/CMSIS/NN/build/ && cd ${CMSIS_PATH}/CMSIS/NN/build/
 cmake .. -DCMAKE_TOOLCHAIN_FILE=${ethosu_dir}/core_platform/cmake/toolchain/arm-none-eabi-gcc.cmake -DTARGET_CPU=cortex-m55 -DBUILD_CMSIS_NN_FUNCTIONS=YES
 make
diff --git a/python/tvm/micro/project_api/server.py b/python/tvm/micro/project_api/server.py
index 2d5db09f4bbe..5aed3a896241 100644
--- a/python/tvm/micro/project_api/server.py
+++ b/python/tvm/micro/project_api/server.py
@@ -804,7 +804,7 @@ def default_project_options(**kw) -> typing.List[ProjectOption]:
             "cmsis_path",
             optional=["generate_project"],
             type="str",
-            default=os.environ.get("CMSIS_PATH", None),
+            default=None,
             help="Path to the CMSIS directory.",
         ),
         ProjectOption(
diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py
index 6b49c043cc3d..a053c905aa34 100644
--- a/tests/micro/zephyr/test_zephyr.py
+++ b/tests/micro/zephyr/test_zephyr.py
@@ -605,6 +605,7 @@ def test_schedule_build_with_cmsis_dependency(workspace_dir, board, microtvm_deb
         "project_type": "host_driven",
         "verbose": bool(build_config.get("debug")),
         "board": board,
+        "cmsis_path": os.getenv("CMSIS_PATH"),
         "use_fvp": bool(use_fvp),
     }
 
@@ -623,7 +624,7 @@ def test_schedule_build_with_cmsis_dependency(workspace_dir, board, microtvm_deb
     assert "CMSIS/DSP/Include" in cmake_content
     assert "CMSIS/DSP/Include/dsp" in cmake_content
     assert "CMSIS/DSP/Include" in cmake_content
-    assert "CMSIS/NN/Include" in cmake_content
+    # assert "CMSIS-NN/Include" in cmake_content
 
 
 if __name__ == "__main__":
diff --git a/tests/python/relay/aot/corstone300.mk b/tests/python/relay/aot/corstone300.mk
index cb1db5ea9995..ebda50d9cfef 100644
--- a/tests/python/relay/aot/corstone300.mk
+++ b/tests/python/relay/aot/corstone300.mk
@@ -48,6 +48,11 @@ CC = arm-none-eabi-gcc
 AR = arm-none-eabi-ar
 RANLIB = arm-none-eabi-ranlib
 CC_OPTS = CC=$(CC) AR=$(AR) RANLIB=$(RANLIB)
+ifeq ($(shell [ -d ${CMSIS_PATH}/CMSIS-NN ]; echo $$?), 0)
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS-NN
+else
+	CMSIS_NN_PATH = ${CMSIS_PATH}/CMSIS/NN
+endif
 PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	${CFLAGS} \
 	-I$(build_dir)/../include \
@@ -57,7 +62,7 @@ PKG_CFLAGS = ${PKG_COMPILE_OPTS} \
 	-I${DRIVER_PATH}/include \
 	-I${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Include/ \
 	-I${CMSIS_PATH}/CMSIS/Core/Include \
-	-I${CMSIS_PATH}/CMSIS/NN/Include \
+	-I${CMSIS_NN_PATH}/Include \
 	-I${CMSIS_PATH}/CMSIS/DSP/Include \
 	-isystem$(STANDALONE_CRT_DIR)/include
 DRIVER_CMAKE_FLAGS = -DCMAKE_TOOLCHAIN_FILE=$(ETHOSU_TEST_ROOT)/arm-none-eabi-gcc.cmake \
@@ -78,7 +83,7 @@ CC_CODEGEN_SRCS = $(shell find $(abspath $(CODEGEN_ROOT)/host/src/*.cc))
 C_CODEGEN_OBJS = $(subst .c,.o,$(C_CODEGEN_SRCS))
 CC_CODEGEN_OBJS = $(subst .cc,.o,$(CC_CODEGEN_SRCS))
 CMSIS_STARTUP_SRCS = $(shell find ${CMSIS_PATH}/Device/ARM/${ARM_CPU}/Source/*.c)
-CMSIS_NN_SRCS = $(shell find ${CMSIS_PATH}/CMSIS/NN/Source/*/*.c)
+CMSIS_NN_SRCS = $(shell find ${CMSIS_NN_PATH}/Source/*/*.c)
 UART_SRCS = $(shell find ${PLATFORM_PATH}/*.c)
 
 ifdef ETHOSU_TEST_ROOT