Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/catalyst.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ jobs:
check_changes:
uses: ./.github/workflows/check_changes.yml

# Container build scripts:
# https://gitlab.kitware.com/christos.tsolakis/catalyst-amrex-docker-images
catalyst:
name: Catalyst
runs-on: ubuntu-22.04
Expand All @@ -20,7 +22,7 @@ jobs:
CC: gcc
CMAKE_PREFIX_PATH: "/opt/conduit:/opt/catalyst"
container:
image: kitware/paraview:ci-catalyst-amrex-warpx-20240701
image: kitware/paraview:ci-catalyst-amrex-warpx-20240828
steps:
- uses: actions/checkout@v5
- name: Configure
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/sensei.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ jobs:
check_changes:
uses: ./.github/workflows/check_changes.yml

# deprecated / unsupported
test_sensei:
if: 0
name: SENSEI Adaptor [test]
runs-on: ubuntu-24.04
needs: check_changes
Expand Down
13 changes: 6 additions & 7 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.18)
cmake_minimum_required(VERSION 3.25)

#
# Setting a cmake_policy to OLD is deprecated by definition and will raise a
Expand Down Expand Up @@ -27,11 +27,13 @@ endif()
#
# CMake 3.18+: CMAKE_CUDA_ARCHITECTURES
# https://cmake.org/cmake/help/latest/policy/CMP0104.html
# We have to migrate there, but maybe the new "native" option (CMake 3.24+)
# is what we want to wait for:
# We us the new "native" option (CMake 3.24+):
# https://cmake.org/cmake/help/v3.24/prop_tgt/CUDA_ARCHITECTURES.html
if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
set(CMAKE_CUDA_ARCHITECTURES native)
endif()
if(POLICY CMP0104)
cmake_policy(SET CMP0104 OLD)
cmake_policy(SET CMP0104 NEW)
endif()

#
Expand Down Expand Up @@ -106,9 +108,6 @@ include( AMReXOptions )
#
if (AMReX_CUDA)
enable_language(CUDA)
if(CMAKE_VERSION VERSION_LESS 3.20)
include(AMReX_SetupCUDA)
endif()
endif ()

#
Expand Down
2 changes: 1 addition & 1 deletion Docs/sphinx_documentation/source/BuildingAMReX_Chapter.rst
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ Fortran compiler that supports the Fortran 2003 standard, and a C
compiler that supports the C99 standard. Prerequisites for building
with GNU Make include Python (>= 2.7, including 3) and standard tools
available in any Unix-like environments (e.g., Perl and sed). For
building with CMake, the minimal requirement is version 3.18.
building with CMake, the minimal requirement is version 3.25.

Please note that we fully support AMReX for Linux systems in general and on the
DOE supercomputers (e.g. Cori, Summit) in particular. Many of our users do build
Expand Down
16 changes: 5 additions & 11 deletions Docs/sphinx_documentation/source/GPU.rst
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ check the :ref:`table <tab:cmakecudavar>` below.
+------------------------------+-------------------------------------------------+-------------+-----------------+
| Variable Name | Description | Default | Possible values |
+==============================+=================================================+=============+=================+
| AMReX_CUDA_ARCH | CUDA target architecture | Auto | User-defined |
| AMReX_CUDA_ARCH | CUDA target architecture | native | User-defined |
+------------------------------+-------------------------------------------------+-------------+-----------------+
| AMReX_CUDA_FASTMATH | Enable CUDA fastmath library | YES | YES, NO |
+------------------------------+-------------------------------------------------+-------------+-----------------+
Expand Down Expand Up @@ -297,15 +297,15 @@ For example, on Cori GPUs you can specify the architecture as follows:

::

cmake [options] -DAMReX_GPU_BACKEND=CUDA -DAMReX_CUDA_ARCH=Volta /path/to/amrex/source
cmake [options] -DAMReX_GPU_BACKEND=CUDA -DAMReX_CUDA_ARCH="80;90" /path/to/amrex/source


If no architecture is specified, CMake will default to the architecture defined in the
*environment variable* ``AMREX_CUDA_ARCH`` (note: all caps).
If the latter is not defined, CMake will try to determine which GPU architecture is supported by the system.
If more than one is found, CMake will build for all of them.
If autodetection fails, a list of "common" architectures is assumed.
`Multiple CUDA architectures <https://cmake.org/cmake/help/latest/module/FindCUDA.html#commands>`__ can also be set manually as semicolon-separated list, e.g. ``-DAMReX_CUDA_ARCH=7.0;8.0``.
`Multiple CUDA architectures <https://cmake.org/cmake/help/latest/module/FindCUDA.html#commands>`__ can also be set manually as semicolon-separated list, e.g. ``-DAMReX_CUDA_ARCH="80;90"``.
Building for multiple CUDA architectures will generally result in a larger library and longer build times.

**Note that AMReX supports NVIDIA GPU architectures with compute capability 6.0 or higher and
Expand All @@ -323,8 +323,7 @@ the following code into the appropriate CMakeLists.txt file:


If instead of using an external installation of AMReX you prefer to include AMReX as a subproject
in your CMake setup, we strongly encourage you to use the ``AMReX_SetupCUDA`` module as shown below
if the CMake version is less than 3.20:
in your CMake setup (i.e., build AMReX on the fly), do this:

.. highlight:: console

Expand All @@ -333,12 +332,7 @@ if the CMake version is less than 3.20:
# Enable CUDA in your CMake project
enable_language(CUDA)

# Include the AMReX-provided CUDA setup module -- OBSOLETE with CMake >= 3.20
if(CMAKE_VERSION VERSION_LESS 3.20)
include(AMReX_SetupCUDA)
endif()

# Include AMReX source directory ONLY AFTER the two steps above
# Include AMReX source directory ONLY AFTER enable_language
add_subdirectory(/path/to/amrex/source/dir)


Expand Down
2 changes: 1 addition & 1 deletion Tests/CMakeTestInstall/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# building and running the code
# in Tests/Amr/Advection_AmrCore/
#
cmake_minimum_required(VERSION 3.18)
cmake_minimum_required(VERSION 3.25)

project(amrex-test-install)

Expand Down
14 changes: 7 additions & 7 deletions Tests/SpackSmokeTest/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# against a currently installed version of AMReX. The resulting
# executable can then be ran to test functionality.

cmake_minimum_required(VERSION 3.18)
cmake_minimum_required(VERSION 3.25)

project(amrex-test-install)

Expand All @@ -26,9 +26,6 @@ if (AMReX_GPU_BACKEND STREQUAL "CUDA")
find_package(AMReX REQUIRED CUDA)

elseif(AMReX_GPU_BACKEND STREQUAL "HIP")
if(CMAKE_VERSION VERSION_LESS 3.20)
message(FATAL_ERROR "HIP requires CMake version 3.20 or newer")
endif()
find_package(AMReX REQUIRED HIP)
find_package(rocrand REQUIRED CONFIG)
find_package(rocprim REQUIRED CONFIG)
Expand Down Expand Up @@ -83,10 +80,13 @@ target_link_libraries(install_test PUBLIC AMReX::amrex)

# Additional CUDA configuration commands
if (AMReX_GPU_BACKEND STREQUAL "CUDA")
if (AMREX_CUDA_ARCHS)
set_target_properties(install_test PROPERTIES
CUDA_ARCHITECTURES "${AMREX_CUDA_ARCHS}")
endif()
set_target_properties(install_test PROPERTIES
LANGUAGE CUDA
CUDA_ARCHITECTURES "${AMREX_CUDA_ARCHS}"
CUDA_SEPARABLE_COMPILATION ON)
LANGUAGE CUDA
CUDA_SEPARABLE_COMPILATION ON)

get_target_property(Source_Files install_test SOURCES)
list(FILTER Source_Files INCLUDE REGEX "\\.cpp$")
Expand Down
13 changes: 7 additions & 6 deletions Tools/CMake/AMReXCUDAOptions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@ endmacro ()
#
message(STATUS "Enabled CUDA options:")

set(AMReX_CUDA_ARCH_DEFAULT "Auto")
if(DEFINED CMAKE_CUDA_ARCHITECTURES)
set(AMReX_CUDA_ARCH_DEFAULT "${CMAKE_CUDA_ARCHITECTURES}")
endif ()
if(DEFINED ENV{AMREX_CUDA_ARCH})
set(AMReX_CUDA_ARCH_DEFAULT "$ENV{AMREX_CUDA_ARCH}")
endif()
set(AMReX_CUDA_ARCH ${AMReX_CUDA_ARCH_DEFAULT} CACHE STRING "CUDA architecture (Use 'Auto' for automatic detection)")
elseif(DEFINED CMAKE_CUDA_ARCHITECTURES)
set(AMReX_CUDA_ARCH_DEFAULT "${CMAKE_CUDA_ARCHITECTURES}")
else()
set(AMReX_CUDA_ARCH_DEFAULT "native")
endif ()
set(AMReX_CUDA_ARCH ${AMReX_CUDA_ARCH_DEFAULT} CACHE STRING "CUDA architecture (Use 'native' for automatic detection)")
set(CMAKE_CUDA_ARCHITECTURES "${AMReX_CUDA_ARCH}" CACHE STRING "" FORCE)

option(AMReX_CUDA_FASTMATH "Enable CUDA fastmath" ON) # Note: inconsistent with AMReX_FASTMATH defaults
cuda_print_option( AMReX_CUDA_FASTMATH )
Expand Down
8 changes: 2 additions & 6 deletions Tools/CMake/AMReXConfig.cmake.in
Original file line number Diff line number Diff line change
Expand Up @@ -256,11 +256,7 @@ endif ()
#
# AMReX 21.06+ supports CUDA_ARCHITECTURES
if (@AMReX_CUDA@)
if (CMAKE_VERSION VERSION_LESS 3.20)
include(AMReX_SetupCUDA)
else ()
find_dependency(CUDAToolkit REQUIRED)
endif ()
find_dependency(CUDAToolkit REQUIRED)
endif ()

# CMake targets
Expand All @@ -275,7 +271,7 @@ if (NOT TARGET AMReX::amrex) # protection in case of multiple inclusions
endif()

# More Modern CUDA CMake
if (@CMAKE_VERSION@ VERSION_GREATER_EQUAL 3.20 AND @AMReX_CUDA@)
if (@AMReX_CUDA@)
foreach(D IN LISTS AMReX_SPACEDIM)
# CUDA architectures amrex was built for -- should we make
set(AMREX_CUDA_ARCHS @AMREX_CUDA_ARCHS@ CACHE INTERNAL "CUDA archs AMReX is built for")
Expand Down
51 changes: 15 additions & 36 deletions Tools/CMake/AMReXFlagsTargets.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -25,42 +25,21 @@ include_guard(GLOBAL)
# <lang> = cxx,fortran,cuda
# <id> = gnu,intel,pgi,cray,clang,appleclang,crayclang,ibmclang,intelllvm,msvc,nvidia,nvhpc,xlclang
#
if (CMAKE_VERSION VERSION_LESS 3.20)
foreach (_language CXX Fortran CUDA )
set(_comp_lang "$<COMPILE_LANGUAGE:${_language}>")
string(TOLOWER "${_language}" _lang)

foreach (_comp GNU Intel PGI Cray Clang AppleClang CrayClang IBMClang IntelLLVM MSVC NVIDIA NVHPC XLClang )
string(TOLOWER "${_comp}" _id)
# Define variables
set(_comp_id "$<${_language}_COMPILER_ID:${_comp}>")
set(_${_lang}_${_id} "$<AND:${_comp_lang},${_comp_id}>")
set(_${_lang}_${_id}_dbg "$<AND:${_comp_lang},${_comp_id},$<CONFIG:Debug>>")
set(_${_lang}_${_id}_rel "$<AND:${_comp_lang},${_comp_id},$<CONFIG:Release>>")
set(_${_lang}_${_id}_rwdbg "$<AND:${_comp_lang},${_comp_id},$<CONFIG:RelWithDebInfo>>")
unset(_comp_id)
endforeach ()

unset(_comp_lang)
unset(_lang)
endforeach ()
else ()
foreach (_language CXX Fortran CUDA )
string(TOLOWER "${_language}" _lang)

foreach (_comp GNU Intel PGI Cray Clang AppleClang CrayClang IBMClang IntelLLVM MSVC NVIDIA NVHPC XLClang )
string(TOLOWER "${_comp}" _id)
# Define variables
set(_${_lang}_${_id} "$<COMPILE_LANG_AND_ID:${_language},${_comp}>")
set(_${_lang}_${_id}_dbg "$<AND:${_${_lang}_${_id}},$<CONFIG:Debug>>")
set(_${_lang}_${_id}_rel "$<AND:${_${_lang}_${_id}},$<CONFIG:Release>>")
set(_${_lang}_${_id}_rwdbg "$<AND:${_${_lang}_${_id}},$<CONFIG:RelWithDebInfo>>")
unset(_id)
endforeach ()

unset(_lang)
endforeach ()
endif ()
foreach (_language CXX Fortran CUDA )
string(TOLOWER "${_language}" _lang)

foreach (_comp GNU Intel PGI Cray Clang AppleClang CrayClang IBMClang IntelLLVM MSVC NVIDIA NVHPC XLClang )
string(TOLOWER "${_comp}" _id)
# Define variables
set(_${_lang}_${_id} "$<COMPILE_LANG_AND_ID:${_language},${_comp}>")
set(_${_lang}_${_id}_dbg "$<AND:${_${_lang}_${_id}},$<CONFIG:Debug>>")
set(_${_lang}_${_id}_rel "$<AND:${_${_lang}_${_id}},$<CONFIG:Release>>")
set(_${_lang}_${_id}_rwdbg "$<AND:${_${_lang}_${_id}},$<CONFIG:RelWithDebInfo>>")
unset(_id)
endforeach ()

unset(_lang)
endforeach ()


#
Expand Down
38 changes: 15 additions & 23 deletions Tools/CMake/AMReXParallelBackends.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,7 @@ endif ()
#
#
#
if ( AMReX_GPU_BACKEND STREQUAL "CUDA"
AND
CMAKE_VERSION VERSION_GREATER_EQUAL 3.20 )
if (AMReX_GPU_BACKEND STREQUAL "CUDA")

find_package(CUDAToolkit REQUIRED)
foreach(D IN LISTS AMReX_SPACEDIM)
Expand All @@ -82,11 +80,9 @@ if ( AMReX_GPU_BACKEND STREQUAL "CUDA"
target_link_libraries(amrex_${D}d PUBLIC CUDA::cusparse)
endif ()

if(CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.2)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://amrex-codes.github.io/amrex/docs_html/GPU_Chapter.html

Do we need to update the docs (not that I expect someone to find cuda 11.0 or 11.1)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updated the docs 👍

# nvToolsExt: if tiny profiler or base profiler are on.
if (AMReX_TINY_PROFILE OR AMReX_BASE_PROFILE)
target_link_libraries(amrex_${D}d PUBLIC CUDA::nvToolsExt)
endif ()
# if tiny profiler or base profiler are on we add range annotations
if (AMReX_TINY_PROFILE OR AMReX_BASE_PROFILE)
target_link_libraries(amrex_${D}d PUBLIC CUDA::nvtx3)
endif ()
endforeach()

Expand All @@ -110,13 +106,14 @@ if ( AMReX_GPU_BACKEND STREQUAL "CUDA"
endforeach()

# Take care of cuda archs
set_cuda_architectures(AMReX_CUDA_ARCH)
foreach(D IN LISTS AMReX_SPACEDIM)
set_target_properties(amrex_${D}d
PROPERTIES
CUDA_ARCHITECTURES "${AMREX_CUDA_ARCHS}"
)
endforeach()
if (AMREX_CUDA_ARCHS)
foreach(D IN LISTS AMReX_SPACEDIM)
set_target_properties(amrex_${D}d
PROPERTIES
CUDA_ARCHITECTURES "${AMREX_CUDA_ARCHS}"
)
endforeach()
endif()

#
# CUDA specific warnings
Expand Down Expand Up @@ -378,14 +375,9 @@ if (AMReX_HIP)
foreach(D IN LISTS AMReX_SPACEDIM)
target_compile_options(amrex_${D}d PUBLIC
$<$<COMPILE_LANGUAGE:CXX>:-fgpu-rdc> )
if(CMAKE_VERSION VERSION_LESS 3.18)
target_link_options(amrex_${D}d PUBLIC
-fgpu-rdc)
else()
target_link_options(amrex_${D}d PUBLIC
"$<$<LINK_LANGUAGE:HIP>:-fgpu-rdc>"
"$<$<LINK_LANGUAGE:CXX>:-fgpu-rdc>")
endif()
target_link_options(amrex_${D}d PUBLIC
"$<$<LINK_LANGUAGE:HIP>:-fgpu-rdc>"
"$<$<LINK_LANGUAGE:CXX>:-fgpu-rdc>")
endforeach()
endif()
endif ()
5 changes: 2 additions & 3 deletions Tools/CMake/AMReXTargetHelpers.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -127,12 +127,11 @@ function (setup_target_for_cuda_compilation _target)
PROPERTIES
CUDA_SEPARABLE_COMPILATION ${AMReX_GPU_RDC} # This adds -dc
)
set_cpp_sources_to_cuda_language(${_target})

if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.20)
if (AMREX_CUDA_ARCHS)
set_target_properties( ${_target}
PROPERTIES
CUDA_ARCHITECTURES "${AMREX_CUDA_ARCHS}"
)
endif ()
set_cpp_sources_to_cuda_language(${_target})
endfunction ()
15 changes: 2 additions & 13 deletions Tools/CMake/AMReXThirdPartyLibraries.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,7 @@ if (AMReX_HDF5)
endif ()

foreach(D IN LISTS AMReX_SPACEDIM)
if (TARGET hdf5::hdf5) # CMake >= 3.19
target_link_libraries(amrex_${D}d PUBLIC hdf5::hdf5)
else () # CMake < 3.19 -- Remove when minimum cmake version is bumped up
target_include_directories(amrex_${D}d PUBLIC ${HDF5_INCLUDE_DIRS})
target_compile_definitions(amrex_${D}d PUBLIC ${HDF5_DEFINITIONS})
target_link_libraries(amrex_${D}d PUBLIC ${HDF5_LIBRARIES})
endif ()
target_link_libraries(amrex_${D}d PUBLIC hdf5::hdf5)
endforeach()

endif ()
Expand All @@ -81,12 +75,7 @@ if (AMReX_HDF5_ZFP)
endif ()

foreach(D IN LISTS AMReX_SPACEDIM)
if (TARGET h5z_zfp::h5z_zfp) # CMake >= 3.19
target_link_libraries(amrex_${D}d PUBLIC h5z_zfp::h5z_zfp)
else () # CMake < 3.19 -- Remove when minimum cmake version is bumped up
target_include_directories(amrex_${D}d PUBLIC ${H5Z_ZFP_INCLUDE_DIR})
target_link_libraries(amrex_${D}d PUBLIC ${H5Z_ZFP_LIBRARY})
endif ()
target_link_libraries(amrex_${D}d PUBLIC h5z_zfp::h5z_zfp)
endforeach()
endif ()

Expand Down
Loading
Loading