Merge pull request #487 from SCOREC/apw/lgmetis

Add METIS splitter/balancer

Author: cwsmith

.github/workflows/cmake.yml

@@ -17,6 +17,7 @@ jobs:
         build_type: [Debug, Release]
         no_mpi: [OFF, ON]
         cxx_standard: [11, 20]
+        metis: [OFF, ON]
 
     steps:
     - uses: actions/checkout@v4
@@ -27,12 +28,15 @@ jobs:
       run: |
            sudo apt update
            sudo apt install gcc-10 g++-10 mpich
+    - name: install metis
+      if: matrix.metis == 'ON'
+      run: sudo apt install libmetis-dev
 
     - name: Configure CMake
       env:
         MPICH_CXX: ${{matrix.compiler.CXX}}
         MPICH_CC: ${{matrix.compiler.CC}}
-      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_STANDARD=${{matrix.cxx_standard}} -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install -DSCOREC_NO_MPI=${{matrix.no_mpi}}
+      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_STANDARD=${{matrix.cxx_standard}} -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install -DSCOREC_NO_MPI=${{matrix.no_mpi}} -DENABLE_METIS=${{matrix.metis}}
 
     - name: Build
       env:

CMakeLists.txt

@@ -196,6 +196,7 @@ add_subdirectory(apf_sim)
 add_subdirectory(mds)
 add_subdirectory(parma)
 add_subdirectory(zoltan)
+add_subdirectory(metis)
 add_subdirectory(pumi)
 add_subdirectory(ma)
 add_subdirectory(crv)

Doxyfile.in

@@ -782,6 +782,8 @@ INPUT                  = \
 @CMAKE_CURRENT_SOURCE_DIR@/spr/spr.h \
 @CMAKE_CURRENT_SOURCE_DIR@/parma/parma.dox \
 @CMAKE_CURRENT_SOURCE_DIR@/parma/parma.h \
+@CMAKE_CURRENT_SOURCE_DIR@/metis/metis.dox \
+@CMAKE_CURRENT_SOURCE_DIR@/metis/apfMETIS.h \
 @CMAKE_CURRENT_SOURCE_DIR@/phasta/phasta.dox \
 @CMAKE_CURRENT_SOURCE_DIR@/phasta/chef.h \
 @CMAKE_CURRENT_SOURCE_DIR@/phasta/chef.cc \

Doxyfile_internal.in

@@ -815,6 +815,8 @@ INPUT                  = \
 @CMAKE_CURRENT_SOURCE_DIR@/spr/spr.h \
 @CMAKE_CURRENT_SOURCE_DIR@/parma/parma.dox \
 @CMAKE_CURRENT_SOURCE_DIR@/parma/parma.h \
+@CMAKE_CURRENT_SOURCE_DIR@/metis/metis.dox \
+@CMAKE_CURRENT_SOURCE_DIR@/metis/apfMETIS.h \
 @CMAKE_CURRENT_SOURCE_DIR@/phasta/phasta.dox \
 @CMAKE_CURRENT_SOURCE_DIR@/phasta/chef.h \
 @CMAKE_CURRENT_SOURCE_DIR@/phasta/chef.cc \

apf/CMakeLists.txt

@@ -20,6 +20,7 @@ set(SOURCES
   apfMesh.cc
   apfMesh2.cc
   apfMigrate.cc
+  apfOpposites.cc
   apfScalarElement.cc
   apfScalarField.cc
   apfShape.cc

apf/apfMesh.h

@@ -627,6 +627,20 @@ void unpackTagInfo(std::string& name, int& type, int& size, pcu::PCU *PCUObj);
 
 extern char const* const dimName[4];
 
+/**
+ * \brief Tag boundary faces with global ids of opposite elements
+ *
+ * This function creates a LONG tag of one value and attaches to all mesh faces
+ * (edges) classified on a partition model face (edge) (i.e., a face in 3d or
+ * edge in 2d that has a remote copy in another part) the global id of the mesh
+ * region (face) on the other side.
+ *
+ * \param gn global element numbering
+ * \param name the name of the resulting tag
+ * \return a new MeshTag called name with global IDs of opposite elements
+ */
+MeshTag* tagOpposites(GlobalNumbering* gn, const char* name);
+
 } //namespace apf
 
 #endif

apf/apfOpposites.cc

@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2025 Scientific Computation Research Center
+ *
+ * This work is open source software, licensed under the terms of the
+ * BSD license as described in the LICENSE file in the top-level directory.
+ */
+
+#include <apfMesh.h>
+#include <apfNumbering.h>
+#include <pcu_util.h>
+
+namespace apf {
+
+static bool hasOtherSide(Mesh* m, MeshEntity* s)
+{
+  if (m->isShared(s))
+    return true;
+  if (!m->hasMatching())
+    return false;
+  Matches matches;
+  m->getMatches(s, matches);
+  return matches.getSize();
+}
+
+static Copy getOtherSide(Mesh* m, MeshEntity* s)
+{
+  if (m->isShared(s))
+    return getOtherCopy(m, s);
+  Matches matches;
+  m->getMatches(s, matches);
+  PCU_ALWAYS_ASSERT(matches.getSize() == 1);
+  return matches[0];
+}
+
+static MeshEntity* getSideElement(Mesh* m, MeshEntity* s)
+{
+  return m->getUpward(s, 0);
+}
+
+static long getElementGid(GlobalNumbering* gn, MeshEntity* e)
+{
+  return getNumber(gn, Node(e, 0));
+}
+
+static void packOtherGid(GlobalNumbering* gn, MeshEntity* s)
+{
+  Mesh* m = getMesh(gn);
+  Copy other = getOtherSide(m, s);
+  long gid = getElementGid(gn, getSideElement(m, s));
+  m->getPCU()->Pack(other.peer, other.entity);
+  m->getPCU()->Pack(other.peer, gid);
+}
+
+static void unpackOtherGid(Mesh* m, MeshTag* t)
+{
+  MeshEntity* s;
+  m->getPCU()->Unpack(s);
+  long gid;
+  m->getPCU()->Unpack(gid);
+  m->setLongTag(s, t, &gid);
+}
+
+MeshTag* tagOpposites(GlobalNumbering* gn, const char* name)
+{
+  Mesh* m = getMesh(gn);
+  MeshTag* t = m->createLongTag(name, 1);
+  int sideDim = m->getDimension() - 1;
+  m->getPCU()->Begin();
+  MeshIterator* it = m->begin(sideDim);
+  MeshEntity* e;
+  while ((e = m->iterate(it)))
+    if (hasOtherSide(m, e))
+      packOtherGid(gn, e);
+  m->end(it);
+  m->getPCU()->Send();
+  while (m->getPCU()->Receive())
+    unpackOtherGid(m, t);
+  return t;
+}
+
+} // namespace apf

cmake/FindMETIS.cmake

@@ -0,0 +1,62 @@
+# Copyright (C) 2025 Scientific Computation Research Center
+#
+# This work is open source software, licensed under the terms of the
+# BSD license as described in the LICENSE file in the top-level directory.
+#
+#[====================================[
+FindMETIS.cmake
+
+Hints:
+- `METIS_PREFIX`: The prefix for the METIS installation.
+
+Once done this will define:
+- `METIS_FOUND`: True if METIS is found.
+- `METIS::METIS`: The IMPORTED library if found.
+- `METIS_LIBRARIES`: METIS libraries required for linking.
+- `METIS_INCLUDE_DIRS`: Directories containing METIS headers.
+]====================================]
+
+cmake_policy(PUSH)
+
+set(METIS_PREFIX "${Trilinos_PREFIX}" CACHE STRING "METIS install directory")
+find_path(METIS_INCLUDE_DIR metis.h HINTS "${METIS_PREFIX}/include")
+set(METIS_INCLUDE_DIRS ${METIS_INCLUDE_DIR})
+file(STRINGS "${METIS_INCLUDE_DIR}/metis.h" _METIS_VERSION_STRS
+  REGEX "#define METIS_VER_(MAJOR|(SUB)?MINOR)"
+)
+set(METIS_VERSION_STR "")
+foreach(ver_comp IN LISTS _METIS_VERSION_STRS)
+  if(METIS_VERSION_STR)
+    string(APPEND METIS_VERSION_STR .)
+  endif()
+  string(REGEX REPLACE
+    "^#define METIS_VER_(MAJOR|(SUB)?MINOR)[ \t]+" ""
+    comp_num "${ver_comp}"
+  )
+  string(APPEND METIS_VERSION_STR "${comp_num}")
+endforeach()
+find_library(METIS_LIBRARY metis HINTS "${METIS_PREFIX}/lib")
+# Add imported library.
+add_library(METIS::METIS UNKNOWN IMPORTED)
+set_target_properties(METIS::METIS PROPERTIES
+  INTERFACE_INCLUDE_DIRECTORIES "${METIS_INCLUDE_DIR}"
+  IMPORTED_LINK_INTERFACE_LANGUAGE "C;CXX"
+  IMPORTED_LOCATION "${METIS_LIBRARY}"
+)
+# Sometimes GKLib is an external dependency; usually it is in METIS itself.
+find_library(GK_LIBRARY GKlib HINTS "${METIS_PREFIX}/lib")
+if(EXISTS "${GK_LIBRARY}")
+  set_target_property(METIS::METIS PROPERTIES
+    INTERFACE_LINK_LIBRARIES "${GK_LIBRARY}"
+  )
+  set(METIS_LIBRARIES "${METIS_LIBRARY}" "${GK_LIBRARY}")
+else()
+  set(METIS_LIBRARIES "${METIS_LIBRARY}")
+endif()
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(METIS
+  REQUIRED_VARS METIS_INCLUDE_DIR METIS_LIBRARY
+  VERSION_VAR METIS_VERSION_STR
+)
+mark_as_advanced(METIS_INCLUDE_DIR METIS_LIBRARY)
+cmake_policy(POP)

ma/CMakeLists.txt

@@ -81,6 +81,7 @@ target_link_libraries(ma
      mds
      parma
      apf_zoltan
+     apf_metis
      pcu
    )
 

ma/maBalance.cc

@@ -1,7 +1,11 @@
+#include <apfMesh.h>
+#include <apfShape.h>
+#include <lionPrint.h>
 #include "maBalance.h"
 #include "maAdapt.h"
 #include <parma.h>
 #include <apfZoltan.h>
+#include <apfMETIS.h>
 
 #define MAX_ZOLTAN_GRAPH_RANKS 16*1024
 
@@ -114,6 +118,10 @@ void runParma(Adapt* a)
   runBalancer(a, Parma_MakeElmBalancer(a->mesh));
 }
 
+void runMETIS(Adapt* a) {
+  runBalancer(a, apf::makeMETISbalancer(a->mesh));
+}
+
 void printEntityImbalance(Mesh* m)
 {
   double imbalance[4];
@@ -147,6 +155,10 @@ void preBalance(Adapt* a)
     runZoltan(a,apf::RIB);
     return;
   }
+  if (in->shouldRunPreMetis) {
+    runMETIS(a);
+    return;
+  }
   if (in->shouldRunPreParma) {
     runParma(a);
     return;
@@ -171,6 +183,10 @@ void preBalance(Adapt* a)
       runZoltan(a, apf::RIB);
       return;
     }
+#elif defined(PUMI_HAS_METIS)
+    if (a->mesh->getPCU()->Peers() < APF_METIS_MAXRANKS) runMETIS(a);
+    else runParma(a);
+    return;
 #else
     runParma(a);
     return;
@@ -189,6 +205,10 @@ void midBalance(Adapt* a)
     runZoltan(a);
     return;
   }
+  if (in->shouldRunMidMetis) {
+    runMETIS(a);
+    return;
+  }
   if (in->shouldRunMidParma) {
     runParma(a);
     return;
@@ -211,6 +231,10 @@ void midBalance(Adapt* a)
       runZoltan(a, apf::RIB);
       return;
     }
+#elif defined(PUMI_HAS_METIS)
+    if (a->mesh->getPCU()->Peers() < APF_METIS_MAXRANKS) runMETIS(a);
+    else runParma(a);
+    return;
 #else
     runParma(a);
     return;
@@ -235,6 +259,11 @@ void postBalance(Adapt* a)
     printEntityImbalance(a->mesh);
     return;
   }
+  if (in->shouldRunPostMetis) {
+    runMETIS(a);
+    printEntityImbalance(a->mesh);
+    return;
+  }
   if (in->shouldRunPostParma) {
     runParma(a);
     printEntityImbalance(a->mesh);
@@ -261,6 +290,10 @@ void postBalance(Adapt* a)
       printEntityImbalance(a->mesh);
       return;
     }
+#elif defined(PUMI_HAS_METIS)
+    if (a->mesh->getPCU()->Peers() < APF_METIS_MAXRANKS) runMETIS(a);
+    else runParma(a);
+    printEntityImbalance(a->mesh);
 #else
     runParma(a);
     printEntityImbalance(a->mesh);