Add HYPRE_Initialized and HYPRE_Finalized (#889)

* Add hypre_State type to track initialization state of hypre
* Add HYPRE_Initialized to determine whether hypre has been initialized
* Add HYPRE_Finalized to determine whether hypre has been finalized
* Add private implementations for hypre_initialized/finalized
* Add HYPRE_Initialize
* Update Fortran interface for HYPRE_Initialize
* Use HYPRE_Initialize in test drivers and examples
* Clean-up mentions of HYPRE_Init
* Add HYPRE_DEPRECATED macro to autotools and CMake builds
* Add regression test for library initialization/finalization

.gitignore

@@ -34,6 +34,7 @@ src/test/ams_driver
 src/test/ij
 src/test/ij_assembly
 src/test/ij_mv
+src/test/ij_mm
 src/test/maxwell_unscaled
 src/test/sstruct
 src/test/sstruct_fac

src/config/HYPRE_config.h.cmake.in

@@ -170,5 +170,8 @@
 /* As F77_FUNC, but for C identifiers containing underscores. */
 #define HYPRE_F77_FUNC_(name,NAME) name ## __
 
+/* Deprecated function macro */
+#define HYPRE_DEPRECATED(reason) _Pragma(reason)
+
 /* Define to 1 if using host memory only */
 #cmakedefine HYPRE_USING_HOST_MEMORY 1

src/config/HYPRE_config.h.in

@@ -65,6 +65,9 @@
 /* Define to 1 if in debug mode */
 #undef HYPRE_DEBUG
 
+/* Deprecated function macro */
+#undef HYPRE_DEPRECATED
+
 /* Main development branch? */
 #undef HYPRE_DEVELOP_BRANCH
 

src/config/configure.in

@@ -89,6 +89,8 @@ HYPRE_TIME="M4_HYPRE_TIME"
 HYPRE_BUGS="M4_HYPRE_BUGS"
 HYPRE_SRCDIR="M4_HYPRE_SRCDIR"
 
+AC_DEFINE(HYPRE_DEPRECATED(reason), _Pragma(reason), [Deprecated function macro])
+
 AC_DEFINE_UNQUOTED(HYPRE_RELEASE_NAME,    ["$HYPRE_NAME"],    [Release name])
 AC_DEFINE_UNQUOTED(HYPRE_RELEASE_VERSION, ["$HYPRE_VERSION"], [Release version])
 AC_DEFINE_UNQUOTED(HYPRE_RELEASE_NUMBER,  [$HYPRE_NUMBER],    [Release number])

src/configure

@@ -3212,6 +3212,10 @@ HYPRE_BUGS="https://github.com/hypre-space/hypre/issues"
 HYPRE_SRCDIR="`pwd`"
 
 
+printf "%s\n" "#define HYPRE_DEPRECATED(reason) _Pragma(reason)" >>confdefs.h
+
+
+
 printf "%s\n" "#define HYPRE_RELEASE_NAME \"$HYPRE_NAME\"" >>confdefs.h
 
 
@@ -12487,3 +12491,4 @@ fi
 mv HYPRE_config.h HYPRE_config.h.tmp
 sed 's/FC_FUNC/HYPRE_FC_FUNC/g' < HYPRE_config.h.tmp > HYPRE_config.h
 rm -f HYPRE_config.h.tmp
+

src/docs/usr-manual/solvers-boomeramg.rst

@@ -170,7 +170,7 @@ GPU-supported Options
 
 In general, CUDA unified memory is required for running BoomerAMG solvers on GPUs.
 However, hypre can also be built without ``--enable-unified-memory`` if
-all the selected parameters have GPU-support. 
+all the selected parameters have GPU-support.
 The currently available  GPU-supported BoomerAMG options include:
 
 * Coarsening: PMIS (8)
@@ -187,7 +187,7 @@ on GPUs is shown below.
 
  cudaSetDevice(device_id); /* GPU binding */
  ...
- HYPRE_Init(); /* must be the first HYPRE function call */
+ HYPRE_Initialize(); /* must be the first HYPRE function call */
  ...
  /* AMG in GPU memory (default) */
  HYPRE_SetMemoryLocation(HYPRE_MEMORY_DEVICE);
@@ -238,7 +238,7 @@ on GPUs is shown below.
  ...
  HYPRE_Finalize(); /* must be the last HYPRE function call */
 
-``HYPRE_Init()`` must be called and precede all the other ``HYPRE_`` functions, and
+``HYPRE_Initialize()`` must be called and precede all the other ``HYPRE_`` functions, and
 ``HYPRE_Finalize()`` must be called before exiting.
 
 Miscellaneous
@@ -253,4 +253,3 @@ algorithms.  For 3-dimensional problems a better choice appears to be 0.5, when
 using the default coarsening algorithm. However, the choice of the strength
 threshold is problem dependent and therefore there could be better choices than
 the two suggested ones.
-

src/examples/ex1.c

@@ -65,7 +65,7 @@ int main (int argc, char *argv[])
    }
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex10.cxx

@@ -57,7 +57,7 @@ int main(int argc, char *argv[])
    MPI_Comm_rank(MPI_COMM_WORLD, &mypid);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex11.c

@@ -74,7 +74,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex12.c

@@ -68,7 +68,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex12cuf.cuf

@@ -86,7 +86,7 @@
       call MPI_Comm_rank(MPI_COMM_WORLD, myid, ierr)
       call MPI_Comm_size(MPI_COMM_WORLD, num_procs, ierr)
 
-      call HYPRE_Init(ierr)
+      call HYPRE_Initialize(ierr)
 
       if (num_procs .ne. 2) then
          if (myid .eq. 0) then
@@ -490,4 +490,3 @@
 
       deallocate(values)
       end
-

src/examples/ex12f.f

@@ -84,7 +84,7 @@
       call MPI_Comm_rank(MPI_COMM_WORLD, myid, ierr)
       call MPI_Comm_size(MPI_COMM_WORLD, num_procs, ierr)
 
-      call HYPRE_Init(ierr)
+      call HYPRE_Initialize(ierr)
 
       if (num_procs .ne. 2) then
          if (myid .eq. 0) then
@@ -510,4 +510,3 @@
 
       stop
       end
-

src/examples/ex12f_cptr.f

@@ -95,7 +95,7 @@
       call MPI_Comm_rank(MPI_COMM_WORLD, myid, ierr)
       call MPI_Comm_size(MPI_COMM_WORLD, num_procs, ierr)
 
-      call HYPRE_Init(ierr)
+      call HYPRE_Initialize(ierr)
 
       if (num_procs .ne. 2) then
          if (myid .eq. 0) then
@@ -501,4 +501,3 @@
 
       stop
       end
-

src/examples/ex13.c

@@ -175,7 +175,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex14.c

@@ -172,7 +172,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex15.c

@@ -247,7 +247,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex15big.c

@@ -237,7 +237,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex16.c

@@ -255,7 +255,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex17.c

@@ -54,7 +54,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex18.c

@@ -58,7 +58,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex18comp.c

@@ -58,7 +58,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex2.c

@@ -67,7 +67,7 @@ int main (int argc, char *argv[])
    }
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex3.c

@@ -153,7 +153,7 @@ int main (int argc, char *argv[])
    }
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex4.c

@@ -245,7 +245,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex5.c

@@ -70,7 +70,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex5big.c

@@ -73,7 +73,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Default problem parameters */
    n = 33;

src/examples/ex5cuf.cuf

@@ -86,7 +86,7 @@
       call MPI_COMM_SIZE(MPI_COMM_WORLD, num_procs, ierr)
       mpi_comm = MPI_COMM_WORLD
 
-      call HYPRE_Init(ierr)
+      call HYPRE_Initialize(ierr)
 
       call HYPRE_SetMemoryLocation(HYPRE_MEMORY_DEVICE, ierr)
       call HYPRE_SetExecutionPolicy(HYPRE_EXEC_DEVICE, ierr)

src/examples/ex5f.f

@@ -79,14 +79,14 @@
       call MPI_COMM_SIZE(MPI_COMM_WORLD, num_procs, ierr)
       mpi_comm = MPI_COMM_WORLD
 
-      call HYPRE_Init(ierr)
+      call HYPRE_Initialize(ierr)
 
       call HYPRE_SetMemoryLocation(HYPRE_MEMORY_DEVICE, ierr)
       call HYPRE_SetExecutionPolicy(HYPRE_EXEC_DEVICE, ierr)
 
       call HYPRE_SetSpGemmUseVendor(0, ierr)
 
-!   Call omp target after HYPRE_Init()
+!   Call omp target after HYPRE_Initialize()
 
       !$omp target enter data map(alloc:rhs_values)
       !$omp target enter data map(alloc:x_values)

src/examples/ex5f_cptr.f

@@ -108,7 +108,7 @@
       call MPI_COMM_SIZE(MPI_COMM_WORLD, num_procs, ierr)
       mpi_comm = MPI_COMM_WORLD
 
-      call HYPRE_Init(ierr)
+      call HYPRE_Initialize(ierr)
 
       call HYPRE_SetMemoryLocation(HYPRE_MEMORY_DEVICE, ierr)
       call HYPRE_SetExecutionPolicy(HYPRE_EXEC_DEVICE, ierr)

src/examples/ex6.c

@@ -63,7 +63,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex7.c

@@ -244,7 +244,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex8.c

@@ -61,7 +61,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/examples/ex9.c

@@ -88,7 +88,7 @@ int main (int argc, char *argv[])
    MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
 
    /* Initialize HYPRE */
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* Print GPU info */
    /* HYPRE_PrintDeviceInfo(); */

src/test/TEST_ij/state.jobs

@@ -0,0 +1,12 @@
+#!/bin/sh
+# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
+# HYPRE Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+#=============================================================================
+# Test library initialization (sequential and parallel)
+#=============================================================================
+
+mpirun -np 1 ./ij -test_init > state.out.1
+mpirun -np 4 ./ij -test_init > state.out.100

src/test/TEST_ij/state.saved

@@ -0,0 +1,13 @@
+# Output file: state.out.1
+hypre library has not been initialized or finalized yet
+hypre library has been initialized
+hypre library has been finalized
+hypre library has been re-initialized
+hypre library has been finalized
+
+# Output file: state.out.100
+hypre library has not been initialized or finalized yet
+hypre library has been initialized
+hypre library has been finalized
+hypre library has been re-initialized
+hypre library has been finalized

src/test/TEST_ij/state.saved.florentia

@@ -0,0 +1,13 @@
+# Output file: state.out.1
+hypre library has not been initialized or finalized yet
+hypre library has been initialized
+hypre library has been finalized
+hypre library has been re-initialized
+hypre library has been finalized
+
+# Output file: state.out.100
+hypre library has not been initialized or finalized yet
+hypre library has been initialized
+hypre library has been finalized
+hypre library has been re-initialized
+hypre library has been finalized

src/test/TEST_ij/state.saved.lassen

@@ -0,0 +1,13 @@
+# Output file: state.out.1
+hypre library has not been initialized or finalized yet
+hypre library has been initialized
+hypre library has been finalized
+hypre library has been re-initialized
+hypre library has been finalized
+
+# Output file: state.out.100
+hypre library has not been initialized or finalized yet
+hypre library has been initialized
+hypre library has been finalized
+hypre library has been re-initialized
+hypre library has been finalized

src/test/TEST_ij/state.saved.lassen_cpu

@@ -0,0 +1,13 @@
+# Output file: state.out.1
+hypre library has not been initialized or finalized yet
+hypre library has been initialized
+hypre library has been finalized
+hypre library has been re-initialized
+hypre library has been finalized
+
+# Output file: state.out.100
+hypre library has not been initialized or finalized yet
+hypre library has been initialized
+hypre library has been finalized
+hypre library has been re-initialized
+hypre library has been finalized

src/test/TEST_ij/state.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
+# HYPRE Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+TNAME=`basename $0 .sh`
+
+#=============================================================================
+# Compare with baseline case
+#=============================================================================
+
+FILES="\
+ ${TNAME}.out.1\
+ ${TNAME}.out.100\
+"
+
+for i in $FILES
+do
+  echo "# Output file: $i"
+  tail -3 $i
+done > ${TNAME}.out

src/test/ams_driver.c

@@ -135,14 +135,14 @@ hypre_int main (hypre_int argc, char *argv[])
 
    /*-----------------------------------------------------------------
     * GPU Device binding
-    * Must be done before HYPRE_Init() and should not be changed after
+    * Must be done before HYPRE_Initialize() and should not be changed after
     *-----------------------------------------------------------------*/
    hypre_bind_device(myid, num_procs, hypre_MPI_COMM_WORLD);
 
    /*-----------------------------------------------------------
     * Initialize : must be the first HYPRE function to call
     *-----------------------------------------------------------*/
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /* default memory location */
    HYPRE_SetMemoryLocation(memory_location);

src/test/ij.c

@@ -209,6 +209,7 @@ main( hypre_int argc,
    HYPRE_Real          Q_trunc = 0;
 
    /* Specific tests */
+   HYPRE_Int           test_init = 0;
    HYPRE_Int           test_ij = 0;
    HYPRE_Int           test_multivec = 0;
    HYPRE_Int           test_scaling = 0;
@@ -481,19 +482,75 @@ main( hypre_int argc,
    hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
    hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
 
+   /* Should we test library initialization? */
+   for (arg_index = 1; arg_index < argc; arg_index ++)
+   {
+      if (strcmp(argv[arg_index], "-test_init") == 0)
+      {
+         test_init = 1;
+         break;
+      }
+   }
+
    /*-----------------------------------------------------------------
     * GPU Device binding
-    * Must be done before HYPRE_Init() and should not be changed after
+    * Must be done before HYPRE_Initialize() and should not be changed after
     *-----------------------------------------------------------------*/
    hypre_bind_device(myid, num_procs, hypre_MPI_COMM_WORLD);
 
-   time_index = hypre_InitializeTiming("Hypre init");
-   hypre_BeginTiming(time_index);
-
    /*-----------------------------------------------------------
     * Initialize : must be the first HYPRE function to call
     *-----------------------------------------------------------*/
-   HYPRE_Init();
+
+   if (test_init)
+   {
+      /* The library should not be initialized or finalized */
+      if (!HYPRE_Initialized() && !HYPRE_Finalized())
+      {
+         hypre_ParPrintf(hypre_MPI_COMM_WORLD, "hypre library has not been initialized or finalized yet\n");
+      }
+
+      HYPRE_Initialize();
+
+      /* Check if the library is in initialized state */
+      if (HYPRE_Initialized() && !HYPRE_Finalized())
+      {
+         hypre_ParPrintf(hypre_MPI_COMM_WORLD, "hypre library has been initialized\n");
+      }
+
+      HYPRE_Finalize();
+
+      /* Check if the library is in finalized state */
+      if (!HYPRE_Initialized() && HYPRE_Finalized())
+      {
+         hypre_ParPrintf(hypre_MPI_COMM_WORLD, "hypre library has been finalized\n");
+      }
+
+      HYPRE_Initialize();
+
+      /* Check if the library is in initialized state */
+      if (HYPRE_Initialized() && !HYPRE_Finalized())
+      {
+         hypre_ParPrintf(hypre_MPI_COMM_WORLD, "hypre library has been re-initialized\n");
+      }
+
+      HYPRE_Finalize();
+
+      /* Check if the library is in finalized state */
+      if (!HYPRE_Initialized() && HYPRE_Finalized())
+      {
+         hypre_ParPrintf(hypre_MPI_COMM_WORLD, "hypre library has been finalized\n");
+      }
+
+      hypre_MPI_Finalize();
+
+      return 0;
+   }
+
+   time_index = hypre_InitializeTiming("Hypre init");
+   hypre_BeginTiming(time_index);
+
+   HYPRE_Initialize();
 
    hypre_EndTiming(time_index);
    hypre_PrintTiming("Hypre init times", hypre_MPI_COMM_WORLD);

src/test/ij_assembly.c

@@ -115,7 +115,7 @@ main( hypre_int  argc,
 
    /*-----------------------------------------------------------------
     * GPU Device binding
-    * Must be done before HYPRE_Init() and should not be changed after
+    * Must be done before HYPRE_Initialize() and should not be changed after
     *-----------------------------------------------------------------*/
    hypre_bind_device(myid, num_procs, hypre_MPI_COMM_WORLD);
 
@@ -123,7 +123,7 @@ main( hypre_int  argc,
    /* Initialize Hypre: must be the first Hypre function to call */
    time_index = hypre_InitializeTiming("Hypre init");
    hypre_BeginTiming(time_index);
-   HYPRE_Init();
+   HYPRE_Initialize();
    hypre_EndTiming(time_index);
    hypre_PrintTiming("Hypre init times", hypre_MPI_COMM_WORLD);
    hypre_FinalizeTiming(time_index);

src/test/ij_device.c

@@ -116,14 +116,14 @@ main( hypre_int argc,
 
    /*-----------------------------------------------------------------
     * GPU Device binding
-    * Must be done before HYPRE_Init() and should not be changed after
+    * Must be done before HYPRE_Initialize() and should not be changed after
     *-----------------------------------------------------------------*/
    hypre_bind_device(myid, num_procs, hypre_MPI_COMM_WORLD);
 
    /*-----------------------------------------------------------
     * Initialize : must be the first HYPRE function to call
     *-----------------------------------------------------------*/
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    hypre_SetNumThreads(5);
    hypre_printf("CPU #OMP THREADS %d\n", hypre_NumThreads());

src/test/ij_mm.c

@@ -787,14 +787,14 @@ main( hypre_int argc,
 
    /*-----------------------------------------------------------------
     * GPU Device binding
-    * Must be done before HYPRE_Init() and should not be changed after
+    * Must be done before HYPRE_Initialize() and should not be changed after
     *-----------------------------------------------------------------*/
    hypre_bind_device(myid, num_procs, hypre_MPI_COMM_WORLD);
 
    /*-----------------------------------------------------------
     * Initialize : must be the first HYPRE function to call
     *-----------------------------------------------------------*/
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    if (myid == 0)
    {

src/test/maxwell_unscaled.c

@@ -1377,14 +1377,14 @@ main( hypre_int argc,
 
    /*-----------------------------------------------------------------
     * GPU Device binding
-    * Must be done before HYPRE_Init() and should not be changed after
+    * Must be done before HYPRE_Initialize() and should not be changed after
     *-----------------------------------------------------------------*/
    hypre_bind_device(myid, num_procs, hypre_MPI_COMM_WORLD);
 
    /*-----------------------------------------------------------
     * Initialize : must be the first HYPRE function to call
     *-----------------------------------------------------------*/
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /*-----------------------------------------------------------
     * Read input file
@@ -2028,4 +2028,3 @@ main( hypre_int argc,
 
    return (0);
 }
-

src/test/sstruct.c

@@ -2469,14 +2469,14 @@ main( hypre_int argc,
 
    /*-----------------------------------------------------------------
     * GPU Device binding
-    * Must be done before HYPRE_Init() and should not be changed after
+    * Must be done before HYPRE_Initialize() and should not be changed after
     *-----------------------------------------------------------------*/
    hypre_bind_device(myid, num_procs, comm);
 
    /*-----------------------------------------------------------
     * Initialize : must be the first HYPRE function to call
     *-----------------------------------------------------------*/
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /*-----------------------------------------------------------
     * Set defaults

src/test/sstruct_fac.c

@@ -1299,14 +1299,14 @@ main( hypre_int argc,
 
    /*-----------------------------------------------------------------
     * GPU Device binding
-    * Must be done before HYPRE_Init() and should not be changed after
+    * Must be done before HYPRE_Initialize() and should not be changed after
     *-----------------------------------------------------------------*/
    hypre_bind_device(myid, num_procs, hypre_MPI_COMM_WORLD);
 
    /*-----------------------------------------------------------
     * Initialize : must be the first HYPRE function to call
     *-----------------------------------------------------------*/
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    /*-----------------------------------------------------------
     * Read input file

src/test/struct.c

@@ -199,14 +199,14 @@ main( hypre_int argc,
 
    /*-----------------------------------------------------------------
     * GPU Device binding
-    * Must be done before HYPRE_Init() and should not be changed after
+    * Must be done before HYPRE_Initialize() and should not be changed after
     *-----------------------------------------------------------------*/
    hypre_bind_device(myid, num_procs, hypre_MPI_COMM_WORLD);
 
    /*-----------------------------------------------------------
     * Initialize : must be the first HYPRE function to call
     *-----------------------------------------------------------*/
-   HYPRE_Init();
+   HYPRE_Initialize();
 
 #if defined(HYPRE_USING_KOKKOS)
    Kokkos::initialize (argc, argv);

src/test/struct_migrate.c

@@ -65,14 +65,14 @@ main( hypre_int argc,
 
    /*-----------------------------------------------------------------
     * GPU Device binding
-    * Must be done before HYPRE_Init() and should not be changed after
+    * Must be done before HYPRE_Initialize() and should not be changed after
     *-----------------------------------------------------------------*/
    hypre_bind_device(myid, num_procs, hypre_MPI_COMM_WORLD);
 
    /*-----------------------------------------------------------
     * Initialize : must be the first HYPRE function to call
     *-----------------------------------------------------------*/
-   HYPRE_Init();
+   HYPRE_Initialize();
 
 #if defined(HYPRE_USING_KOKKOS)
    Kokkos::initialize (argc, argv);
@@ -477,4 +477,3 @@ AddValuesVector( hypre_StructGrid   *grid,
 
    return ierr;
 }
-

src/test/test_mgr.c

@@ -275,14 +275,14 @@ main( hypre_int argc,
    /*-----------------------------------------------------------
    * Initialize : must be the first HYPRE function to call
    *-----------------------------------------------------------*/
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    hypre_EndTiming(time_index);
    hypre_PrintTiming("Hypre init times", hypre_MPI_COMM_WORLD);
    hypre_FinalizeTiming(time_index);
    hypre_ClearTiming();
 
-   /* To be effective, hypre_SetCubMemPoolSize must immediately follow HYPRE_Init */
+   /* To be effective, hypre_SetCubMemPoolSize must immediately follow HYPRE_Initialize */
    hypre_SetCubMemPoolSize( mempool_bin_growth, mempool_min_bin,
                             mempool_max_bin, mempool_max_cached_bytes );
 
@@ -1168,5 +1168,3 @@ BuildParFromFile( HYPRE_Int                  argc,
 
    return (0);
 }
-
-

src/test/test_mgr_gpu.c

@@ -287,7 +287,7 @@ main( hypre_int argc,
    /*-----------------------------------------------------------
    * Initialize : must be the first HYPRE function to call
    *-----------------------------------------------------------*/
-   HYPRE_Init();
+   HYPRE_Initialize();
 
    hypre_EndTiming(time_index);
    hypre_PrintTiming("Hypre init times", hypre_MPI_COMM_WORLD);

src/test/zboxloop.c

@@ -50,7 +50,7 @@ main( hypre_int argc,
    hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
    hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
 
-   HYPRE_Init();
+   HYPRE_Initialize();
 
 #if defined(HYPRE_USING_KOKKOS)
    Kokkos::initialize (argc, argv);
@@ -584,4 +584,3 @@ main( hypre_int argc,
 
    return (0);
 }
-

src/utilities/CMakeLists.txt

@@ -45,6 +45,7 @@ set(SRCS
   mpistubs.c
   qsplit.c
   random.c
+  state.c
   threading.c
   timer.c
   timing.c

src/utilities/F90_HYPRE_general.c

@@ -16,7 +16,14 @@ void
 hypre_F90_IFACE(hypre_init, HYPRE_INIT)
 (hypre_F90_Int *ierr)
 {
-   *ierr = (hypre_F90_Int) HYPRE_Init();
+   *ierr = (hypre_F90_Int) HYPRE_Initialize();
+}
+
+void
+hypre_F90_IFACE(hypre_initialize, HYPRE_INITIALIZE)
+(hypre_F90_Int *ierr)
+{
+   *ierr = (hypre_F90_Int) HYPRE_Initialize();
 }
 
 void

src/utilities/HYPRE_utilities.h

@@ -122,9 +122,39 @@ typedef HYPRE_Int MPI_Comm;
  * HYPRE init/finalize
  *--------------------------------------------------------------------------*/
 
-HYPRE_Int HYPRE_Init(void);
+/**
+ * Initializes the hypre library.
+ **/
+
+HYPRE_Int HYPRE_Initialize(void);
+
+/**
+ * (Required) Initializes the hypre library. This function is provided for backward compatibility.
+ * Please, use HYPRE_Initialize instead.
+ **/
+
+#define HYPRE_Init() \
+   HYPRE_DEPRECATED("GCC warning \"HYPRE_Init is deprecated. Use HYPRE_Initialize() instead.\"")  \
+   HYPRE_Initialize()
+
+/**
+ * (Required) Finalizes the hypre library.
+ **/
+
 HYPRE_Int HYPRE_Finalize(void);
 
+/**
+ * (Optional) Returns true if the hypre library has been initialized but not finalized yet.
+ **/
+
+HYPRE_Int HYPRE_Initialized(void);
+
+/**
+ * (Optional) Returns true if the hypre library has been finalized but not re-initialized yet.
+ **/
+
+HYPRE_Int HYPRE_Finalized(void);
+
 /*--------------------------------------------------------------------------
  * HYPRE error user functions
  *--------------------------------------------------------------------------*/

src/utilities/Makefile

@@ -61,6 +61,7 @@ FILES =\
  mpistubs.c\
  qsplit.c\
  random.c\
+ state.c\
  threading.c\
  timer.c\
  timing.c

src/utilities/_hypre_utilities.h

@@ -14,6 +14,30 @@
 extern "C" {
 #endif
 
+/******************************************************************************
+ * Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
+ * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
+ *
+ * SPDX-License-Identifier: (Apache-2.0 OR MIT)
+ ******************************************************************************/
+
+#ifndef hypre_STATE_HEADER
+#define hypre_STATE_HEADER
+
+/*--------------------------------------------------------------------------
+ * hypre library state
+ *--------------------------------------------------------------------------*/
+
+typedef enum hypre_State_enum
+{
+   HYPRE_STATE_NONE        = 0,
+   HYPRE_STATE_INITIALIZED = 1,
+   HYPRE_STATE_FINALIZED   = 2
+} hypre_State;
+
+extern hypre_State hypre__global_state;
+
+#endif /* hypre_STATE_HEADER */
 /******************************************************************************
  * Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
  * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
@@ -32,6 +56,10 @@ extern "C" {
 
 #include <math.h>
 
+/*--------------------------------------------------------------------------
+ * typedefs
+ *--------------------------------------------------------------------------*/
+
 /* This allows us to consistently avoid 'int' throughout hypre */
 typedef int                    hypre_int;
 typedef long int               hypre_longint;
@@ -166,7 +194,6 @@ typedef double                 hypre_double;
 #endif
 
 #endif /* hypre_GENERAL_HEADER */
-
 /******************************************************************************
  * Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
  * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
@@ -1719,6 +1746,12 @@ HYPRE_Complex hypre_csqrt( HYPRE_Complex value );
 #define hypre_csqrt(value) hypre_sqrt(value)
 #endif
 
+/* state.c */
+HYPRE_Int hypre_Initialized( void );
+HYPRE_Int hypre_Finalized( void );
+HYPRE_Int hypre_SetInitialized( void );
+HYPRE_Int hypre_SetFinalized( void );
+
 /* general.c */
 hypre_Handle* hypre_handle(void);
 hypre_Handle* hypre_HandleCreate(void);

src/utilities/device_utils.c

@@ -2190,7 +2190,7 @@ hypreGPUKernel_CompileFlagSafetyCheck( hypre_DeviceItem &item,
 /*--------------------------------------------------------------------
  * hypre_CudaCompileFlagCheck
  *
- * Assume this function is called inside HYPRE_Init(), at a place
+ * Assume this function is called inside HYPRE_Initialize(), at a place
  * where we do not want to activate memory pooling, so we do not use
  * hypre's memory model to Alloc and Free.
  *

src/utilities/general.c

@@ -20,8 +20,8 @@ hypre_handle(void)
    if (!_hypre_handle)
    {
       hypre_error_w_msg(HYPRE_ERROR_GENERIC,
-                        "ERROR - _hypre_handle is not initialized. Calling HYPRE_Init(). All HYPRE_* or hypre_* function calls should occur between HYPRE_Init() and HYPRE_Finalize().\n");
-      HYPRE_Init();
+                        "ERROR - _hypre_handle is not initialized. Calling HYPRE_Initialize(). All HYPRE_* or hypre_* function calls should occur between HYPRE_Initialize() and HYPRE_Finalize().\n");
+      HYPRE_Initialize();
    }
 
    return _hypre_handle;
@@ -235,8 +235,17 @@ hypre_GetDeviceLastError(void)
  *****************************************************************************/
 
 HYPRE_Int
-HYPRE_Init(void)
+HYPRE_Initialize(void)
 {
+   /* Return if the hypre library is in initialized state */
+   if (hypre_Initialized())
+   {
+      return hypre_error_flag;
+   }
+
+   /* Update library state */
+   hypre_SetInitialized();
+
 #ifdef HYPRE_USING_MEMORY_TRACKER
    if (!_hypre_memory_tracker)
    {
@@ -250,75 +259,81 @@ HYPRE_Init(void)
    }
 
 #if defined(HYPRE_USING_GPU) || defined(HYPRE_USING_DEVICE_OPENMP)
+   /* If the library has not been initialized or finalized yet,
+      meaning that it is the first time HYPRE_Init is being called,
+      then perform the initialization of device structures below */
+   if (!HYPRE_Initialized() && !HYPRE_Finalized())
+   {
 #if !defined(HYPRE_USING_SYCL)
-   /* With sycl, cannot call hypre_GetDeviceLastError() until after device and queue setup */
-   hypre_GetDeviceLastError();
+      /* With sycl, cannot call hypre_GetDeviceLastError() until after device and queue setup */
+      hypre_GetDeviceLastError();
 #endif
 
-   /* Notice: the cudaStream created is specific to the device
-    * that was in effect when you created the stream.
-    * So, we should first set the device and create the streams
-    */
-   hypre_int device_id;
-   hypre_GetDevice(&device_id);
-   hypre_SetDevice(device_id, _hypre_handle);
-   hypre_GetDeviceMaxShmemSize(device_id, _hypre_handle);
+      /* Notice: the cudaStream created is specific to the device
+       * that was in effect when you created the stream.
+       * So, we should first set the device and create the streams
+       */
+      hypre_int device_id;
+      hypre_GetDevice(&device_id);
+      hypre_SetDevice(device_id, _hypre_handle);
+      hypre_GetDeviceMaxShmemSize(device_id, _hypre_handle);
 
 #if defined(HYPRE_USING_DEVICE_MALLOC_ASYNC)
-   cudaMemPool_t mempool;
-   cudaDeviceGetDefaultMemPool(&mempool, device_id);
-   uint64_t threshold = UINT64_MAX;
-   cudaMemPoolSetAttribute(mempool, cudaMemPoolAttrReleaseThreshold, &threshold);
+      cudaMemPool_t mempool;
+      cudaDeviceGetDefaultMemPool(&mempool, device_id);
+      uint64_t threshold = UINT64_MAX;
+      cudaMemPoolSetAttribute(mempool, cudaMemPoolAttrReleaseThreshold, &threshold);
 #endif
 
-   /* To include the cost of creating streams/cudahandles in HYPRE_Init */
-   /* If not here, will be done at the first use */
+      /* To include the cost of creating streams/cudahandles in HYPRE_Init */
+      /* If not here, will be done at the first use */
 #if defined(HYPRE_USING_CUDA_STREAMS)
-   hypre_HandleComputeStream(_hypre_handle);
+      hypre_HandleComputeStream(_hypre_handle);
 #endif
 
-   /* A separate stream for prefetching */
-   //hypre_HandleCudaPrefetchStream(_hypre_handle);
-#endif // #if defined(HYPRE_USING_GPU) || defined(HYPRE_USING_DEVICE_OPENMP)
+      /* A separate stream for prefetching */
+      //hypre_HandleCudaPrefetchStream(_hypre_handle);
 
 #if defined(HYPRE_USING_CUBLAS)
-   hypre_HandleCublasHandle(_hypre_handle);
+      hypre_HandleCublasHandle(_hypre_handle);
 #endif
 
 #if defined(HYPRE_USING_CUSPARSE) || defined(HYPRE_USING_ROCSPARSE)
-   hypre_HandleCusparseHandle(_hypre_handle);
+      hypre_HandleCusparseHandle(_hypre_handle);
 #endif
 
 #if defined(HYPRE_USING_CURAND) || defined(HYPRE_USING_ROCRAND)
-   hypre_HandleCurandGenerator(_hypre_handle);
+      hypre_HandleCurandGenerator(_hypre_handle);
 #endif
 
 #if defined(HYPRE_USING_CUSOLVER) || defined(HYPRE_USING_ROCSOLVER)
-   hypre_HandleVendorSolverHandle(_hypre_handle);
+      hypre_HandleVendorSolverHandle(_hypre_handle);
 #endif
 
    /* Check if cuda arch flags in compiling match the device */
 #if defined(HYPRE_USING_CUDA) && defined(HYPRE_DEBUG)
-   hypre_CudaCompileFlagCheck();
-#endif
-
-#if defined(HYPRE_USING_DEVICE_OPENMP)
-   HYPRE_OMPOffloadOn();
+      hypre_CudaCompileFlagCheck();
 #endif
 
 #if defined(HYPRE_USING_DEVICE_POOL)
-   /* Keep this check here at the end of HYPRE_Init()
-    * Make sure that device pool allocator has not been setup in HYPRE_Init,
-    * otherwise users are not able to set all the parameters
-    */
-   if ( hypre_HandleCubDevAllocator(_hypre_handle) ||
-        hypre_HandleCubUvmAllocator(_hypre_handle) )
-   {
-      char msg[256];
-      hypre_sprintf(msg, "%s %s", "ERROR: device pool allocators have been created in", __func__);
-      hypre_fprintf(stderr, "%s\n", msg);
-      hypre_error_w_msg(-1, msg);
+      /* Keep this check here at the end of HYPRE_Initialize()
+       * Make sure that device pool allocator has not been setup in HYPRE_Initialize,
+       * otherwise users are not able to set all the parameters
+       */
+      if ( hypre_HandleCubDevAllocator(_hypre_handle) ||
+           hypre_HandleCubUvmAllocator(_hypre_handle) )
+      {
+         char msg[256];
+         hypre_sprintf(msg, "%s %s", "ERROR: device pool allocators have been created in", __func__);
+         hypre_fprintf(stderr, "%s\n", msg);
+         hypre_error_w_msg(-1, msg);
+      }
+#endif
    }
+#endif /* if defined(HYPRE_USING_GPU) || defined(HYPRE_USING_DEVICE_OPENMP) */
+
+#if defined(HYPRE_USING_DEVICE_OPENMP)
+   HYPRE_OMPOffloadOn();
 #endif
 
 #if defined(HYPRE_USING_UMPIRE)
@@ -337,6 +352,15 @@ HYPRE_Init(void)
 HYPRE_Int
 HYPRE_Finalize(void)
 {
+   /* Return if the hypre library has already been finalized */
+   if (hypre_Finalized())
+   {
+      return hypre_error_flag;
+   }
+
+   /* Update library state */
+   hypre_SetFinalized();
+
 #if defined(HYPRE_USING_UMPIRE)
    hypre_UmpireFinalize(_hypre_handle);
 #endif

src/utilities/general.h

@@ -16,6 +16,10 @@
 
 #include <math.h>
 
+/*--------------------------------------------------------------------------
+ * typedefs
+ *--------------------------------------------------------------------------*/
+
 /* This allows us to consistently avoid 'int' throughout hypre */
 typedef int                    hypre_int;
 typedef long int               hypre_longint;
@@ -150,4 +154,3 @@ typedef double                 hypre_double;
 #endif
 
 #endif /* hypre_GENERAL_HEADER */
-

src/utilities/headers

@@ -33,6 +33,7 @@ extern "C" {
 # Structures and prototypes
 #===========================================================================
 
+cat state.h                    >> $INTERNAL_HEADER
 cat general.h                  >> $INTERNAL_HEADER
 cat printf.h                   >> $INTERNAL_HEADER
 cat error.h                    >> $INTERNAL_HEADER
@@ -110,4 +111,3 @@ cat >> $INTERNAL_HEADER <<@
 #endif
 
 @
-

src/utilities/protos.h

@@ -39,6 +39,12 @@ HYPRE_Complex hypre_csqrt( HYPRE_Complex value );
 #define hypre_csqrt(value) hypre_sqrt(value)
 #endif
 
+/* state.c */
+HYPRE_Int hypre_Initialized( void );
+HYPRE_Int hypre_Finalized( void );
+HYPRE_Int hypre_SetInitialized( void );
+HYPRE_Int hypre_SetFinalized( void );
+
 /* general.c */
 hypre_Handle* hypre_handle(void);
 hypre_Handle* hypre_HandleCreate(void);

src/utilities/state.c

@@ -0,0 +1,88 @@
+/******************************************************************************
+ * Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
+ * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
+ *
+ * SPDX-License-Identifier: (Apache-2.0 OR MIT)
+ ******************************************************************************/
+
+#include "_hypre_utilities.h"
+
+/* Global variable: library state (initialized, finalized, or none) */
+hypre_State hypre__global_state = HYPRE_STATE_NONE;
+
+/*--------------------------------------------------------------------------
+ * HYPRE_Initialized
+ *
+ * Public function for hypre_Initialized
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+HYPRE_Initialized( void )
+{
+   return hypre_Initialized();
+}
+
+/*--------------------------------------------------------------------------
+ * HYPRE_Finalized
+ *
+ * Public function for hypre_Finalized
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+HYPRE_Finalized( void )
+{
+   return hypre_Finalized();
+}
+
+/*--------------------------------------------------------------------------
+ * hypre_Initialized
+ *
+ * This function returns True when the library has been initialized, but not
+ * finalized yet.
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+hypre_Initialized( void )
+{
+   return (hypre__global_state == HYPRE_STATE_INITIALIZED);
+}
+
+/*--------------------------------------------------------------------------
+ * hypre_Finalized
+ *
+ * This function returns True when the library is in finalized state;
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+hypre_Finalized( void )
+{
+   return (hypre__global_state == HYPRE_STATE_FINALIZED);
+}
+
+/*--------------------------------------------------------------------------
+ * hypre_SetInitialized
+ *
+ * This function sets the library state to initialized
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+hypre_SetInitialized( void )
+{
+   hypre__global_state = HYPRE_STATE_INITIALIZED;
+
+   return hypre_error_flag;
+}
+
+/*--------------------------------------------------------------------------
+ * hypre_SetFinalized
+ *
+ * This function sets the library state to finalized
+ *--------------------------------------------------------------------------*/
+
+HYPRE_Int
+hypre_SetFinalized( void )
+{
+   hypre__global_state = HYPRE_STATE_FINALIZED;
+
+   return hypre_error_flag;
+}

src/utilities/state.h

@@ -0,0 +1,24 @@
+/******************************************************************************
+ * Copyright (c) 1998 Lawrence Livermore National Security, LLC and other
+ * HYPRE Project Developers. See the top-level COPYRIGHT file for details.
+ *
+ * SPDX-License-Identifier: (Apache-2.0 OR MIT)
+ ******************************************************************************/
+
+#ifndef hypre_STATE_HEADER
+#define hypre_STATE_HEADER
+
+/*--------------------------------------------------------------------------
+ * hypre library state
+ *--------------------------------------------------------------------------*/
+
+typedef enum hypre_State_enum
+{
+   HYPRE_STATE_NONE        = 0,
+   HYPRE_STATE_INITIALIZED = 1,
+   HYPRE_STATE_FINALIZED   = 2
+} hypre_State;
+
+extern hypre_State hypre__global_state;
+
+#endif /* hypre_STATE_HEADER */