make metis fail when >256 procs

- metis/apfMETIS.h: add APF_METIS_MAXRANKS macro.
- metis/apfMETISbalancer.cc: fail with error message when more than 256
  peers.
- metis/apfMETISsplitter.cc: fail when more than 256 peers.
- metis/apfMETIScommon.h: add STRINGIFY macro for error messages.
- ma/maBalance.cc: fallback to Parma when PUMI_HAS_METIS but more than
  256 peers to avoid failure.

Signed-off-by: Aiden Woodruff <[email protected]>

Author: bobpaw

ma/maBalance.cc

@@ -184,7 +184,9 @@ void preBalance(Adapt* a)
       return;
     }
 #elif defined(PUMI_HAS_METIS)
-    runMETIS(a);
+    if (a->mesh->getPCU()->Peers() < APF_METIS_MAXRANKS) runMETIS(a);
+    else runParma(a);
+    return;
 #else
     runParma(a);
     return;
@@ -230,7 +232,8 @@ void midBalance(Adapt* a)
       return;
     }
 #elif defined(PUMI_HAS_METIS)
-    runMETIS(a);
+    if (a->mesh->getPCU()->Peers() < APF_METIS_MAXRANKS) runMETIS(a);
+    else runParma(a);
     return;
 #else
     runParma(a);
@@ -288,7 +291,8 @@ void postBalance(Adapt* a)
       return;
     }
 #elif defined(PUMI_HAS_METIS)
-    runMETIS(a);
+    if (a->mesh->getPCU()->Peers() < APF_METIS_MAXRANKS) runMETIS(a);
+    else runParma(a);
     printEntityImbalance(a->mesh);
 #else
     runParma(a);

metis/apfMETIS.h

@@ -7,6 +7,8 @@
 #ifndef APF_METIS_H
 #define APF_METIS_H
 
+#define APF_METIS_MAXRANKS 256
+
 /**
  * \file apfMETIS.h
  * \brief METIS partitioning for apf::Mesh objects.

metis/apfMETISbalancer.cc

@@ -16,6 +16,7 @@
 #include <apfShape.h>
 #include <lionPrint.h>
 #include <pcu_util.h>
+#include "apfMETIS.h"
 
 #include <metis.h>
 
@@ -170,6 +171,12 @@ static void remapPart(int nparts, std::vector<idx_t>& part, const std::vector<in
 
 void MetisBalancer::balance(MeshTag* weights, double tolerance) {
   PCU_ALWAYS_ASSERT(tolerance > 1.0);
+  if (mesh_->getPCU()->Peers() > APF_METIS_MAXRANKS) {
+    fail(
+      "METIS called with > " STRINGIFY(APF_METIS_MAXRANKS)
+      " procs, which is unsupported due to memory requirements\n"
+    );
+  }
   if (weights != nullptr) {
     if (mesh_->getPCU()->Self() == 0)
       lion_oprint(1, "METIS: weights are not supported\n");

metis/apfMETIScommon.h

@@ -8,6 +8,8 @@
 #ifndef APF_METIS_COMMON_H
 #define APF_METIS_COMMON_H
 
+#define STRINGIFY(arg) #arg
+
 #include <vector>
 
 namespace apf {

metis/apfMETISsplitter.cc

@@ -3,6 +3,7 @@
 #include <apfShape.h>
 #include <lionPrint.h>
 #include <pcu_util.h>
+#include "apfMETIS.h"
 
 #include <metis.h>
 
@@ -18,6 +19,12 @@ Migration* MetisSplitter::split(
 ) {
   PCU_ALWAYS_ASSERT(tolerance > 1.0);
   PCU_ALWAYS_ASSERT(multiple > 1);
+  if (mesh_->getPCU()->Peers() > APF_METIS_MAXRANKS) {
+    fail(
+      "METIS called with > " STRINGIFY(APF_METIS_MAXRANKS)
+      " procs, which is unsupported due to memory requirements\n"
+    );
+  }
   auto t0 = pcu::Time();
   if (weights != nullptr) {
     lion_oprint(1, "METIS: weights are not supported\n");