diff --git a/src/parcsr_ls/HYPRE_parcsr_ls.h b/src/parcsr_ls/HYPRE_parcsr_ls.h index 0cdd8f95b..41d06c1cb 100644 --- a/src/parcsr_ls/HYPRE_parcsr_ls.h +++ b/src/parcsr_ls/HYPRE_parcsr_ls.h @@ -4275,12 +4275,29 @@ HYPRE_Int HYPRE_MGRSetCoarseSolver(HYPRE_Solver solver, HYPRE_Solver coarse_grid_solver ); /** - * (Optional) Set the print level to print setup and solve information. + * @brief (Optional) Set the verbosity level for MGR. * - * - 0 : no printout (default) - * - 1 : print setup information - * - 2 : print solve information - * - 3 : print both setup and solve information + * @details You can control what information gets printed by specifying the + * output levels using this function. Each option corresponds to a specific type + * of information, and you can activate several of them at the same time by summing + * their respective numeric codes, which are given below: + * + * - 1: Print MGR's setup information. + * - 2: Print MGR's solve information. + * - 4: Print MGR's parameters information. + * - 8: Set print mode for matrices and vectors to ASCII (binary mode is used by default) + * - 16: Print the finest level matrix to NP files where NP is the number of ranks. + * - 32: Print the finest level right-hand-side to NP files. + * + * @param solver [IN] The solver to configure. + * @param print_level [IN] The desired output level. + * + * @example To print setup information (1); matrix (16) and rhs (32) to binary files, + * set \c print_level to 49 (1 + 16 + 32). In the previous example, to use ASCII + * files for matrices and vectors, set \c print_level to 57 (1 + 8 + 16 + 32). + * + * @note The default print level is zero, which means no information will be + * printed by default. **/ HYPRE_Int HYPRE_MGRSetPrintLevel( HYPRE_Solver solver, diff --git a/src/parcsr_ls/_hypre_parcsr_ls.h b/src/parcsr_ls/_hypre_parcsr_ls.h index 2073e5d14..7edb29676 100644 --- a/src/parcsr_ls/_hypre_parcsr_ls.h +++ b/src/parcsr_ls/_hypre_parcsr_ls.h @@ -3644,7 +3644,6 @@ HYPRE_Int hypre_MGRComputeNonGalerkinCoarseGrid( hypre_ParCSRMatrix *A, hypre_Pa HYPRE_Int ordering, HYPRE_Int method, HYPRE_Int Pmax, HYPRE_Int *CF_marker, hypre_ParCSRMatrix **A_H_ptr ); -HYPRE_Int hypre_MGRWriteSolverParams( void *mgr_vdata ); HYPRE_Int hypre_MGRSetAffSolverType( void *systg_vdata, HYPRE_Int *aff_solver_type ); HYPRE_Int hypre_MGRSetCoarseSolverType( void *systg_vdata, HYPRE_Int coarse_solver_type ); HYPRE_Int hypre_MGRSetCoarseSolverIter( void *systg_vdata, HYPRE_Int coarse_solver_iter ); @@ -3679,6 +3678,7 @@ HYPRE_Int hypre_MGRSetMaxIter( void *mgr_vdata, HYPRE_Int max_iter ); HYPRE_Int hypre_MGRSetPMaxElmts( void *mgr_vdata, HYPRE_Int P_max_elmts ); HYPRE_Int hypre_MGRSetLevelPMaxElmts( void *mgr_vdata, HYPRE_Int *P_max_elmts ); HYPRE_Int hypre_MGRSetTol( void *mgr_vdata, HYPRE_Real tol ); +HYPRE_Int hypre_MGRDataPrint(void *mgr_vdata); #ifdef HYPRE_USING_DSUPERLU void *hypre_MGRDirectSolverCreate( void ); HYPRE_Int hypre_MGRDirectSolverSetup( void *solver, hypre_ParCSRMatrix *A, diff --git a/src/parcsr_ls/par_cheby_device.c b/src/parcsr_ls/par_cheby_device.c index fa65e1f68..550798424 100644 --- a/src/parcsr_ls/par_cheby_device.c +++ b/src/parcsr_ls/par_cheby_device.c @@ -169,7 +169,7 @@ hypre_ParCSRRelax_Cheby_SolveDevice(hypre_ParCSRMatrix *A, /* matrix to relax wi std::for_each, oneapi::dpl::make_zip_iterator(orig_u, u_data, r_data), oneapi::dpl::make_zip_iterator(orig_u + num_rows, u_data + num_rows, - r_data + num_rows), + r_data + num_rows), save_and_scale(coefs[cheby_order])); #else HYPRE_THRUST_CALL( @@ -241,7 +241,8 @@ hypre_ParCSRRelax_Cheby_SolveDevice(hypre_ParCSRMatrix *A, /* matrix to relax wi /* v = D^(-1/2)AD^(-1/2)u */ /* tmp = ds .* u */ #if defined(HYPRE_USING_SYCL) - HYPRE_ONEDPL_CALL( std::transform, ds_data, ds_data + num_rows, u_data, tmp_data, std::multiplies() ); + HYPRE_ONEDPL_CALL( std::transform, ds_data, ds_data + num_rows, u_data, tmp_data, + std::multiplies() ); #else HYPRE_THRUST_CALL( transform, ds_data, ds_data + num_rows, u_data, tmp_data, _1 * _2 ); #endif diff --git a/src/parcsr_ls/par_mgr.c b/src/parcsr_ls/par_mgr.c index 0f1529453..fe76c1581 100644 --- a/src/parcsr_ls/par_mgr.c +++ b/src/parcsr_ls/par_mgr.c @@ -122,6 +122,7 @@ hypre_MGRCreate(void) (mgr_data -> print_level) = 0; (mgr_data -> frelax_print_level) = 0; (mgr_data -> cg_print_level) = 0; + (mgr_data -> info_path) = NULL; (mgr_data -> l1_norms) = NULL; @@ -471,6 +472,9 @@ hypre_MGRDestroy( void *data ) hypre_TFree(mgr_data -> GSElimData, HYPRE_MEMORY_HOST); } + /* Print info path */ + hypre_TFree(mgr_data -> info_path, HYPRE_MEMORY_HOST); + /* mgr data */ hypre_TFree(mgr_data, HYPRE_MEMORY_HOST); @@ -5690,7 +5694,11 @@ HYPRE_Int hypre_MGRSetPrintLevel( void *mgr_vdata, HYPRE_Int print_level ) { hypre_ParMGRData *mgr_data = (hypre_ParMGRData*) mgr_vdata; - (mgr_data -> print_level) = print_level; + + /* Unset reserved bits if any are active */ + (mgr_data -> print_level) = print_level & ~(HYPRE_MGR_PRINT_RESERVED_A | + HYPRE_MGR_PRINT_RESERVED_B | + HYPRE_MGR_PRINT_RESERVED_C); return hypre_error_flag; } @@ -6590,62 +6598,165 @@ hypre_MGRPrintCoarseSystem( void *mgr_vdata, HYPRE_Int print_flag) return hypre_error_flag; } -/* Print solver params */ +/*-------------------------------------------------------------------------- + * hypre_MGRDataPrint + *--------------------------------------------------------------------------*/ + HYPRE_Int -hypre_MGRWriteSolverParams(void *mgr_vdata) +hypre_MGRDataPrint(void *mgr_vdata) { - hypre_ParMGRData *mgr_data = (hypre_ParMGRData*) mgr_vdata; - HYPRE_Int i, j; - HYPRE_Int max_num_coarse_levels = (mgr_data -> max_num_coarse_levels); - hypre_printf("MGR Setup parameters: \n"); - hypre_printf("Block size: %d\n", (mgr_data -> block_size)); - hypre_printf("Max number of coarse levels: %d\n", (mgr_data -> max_num_coarse_levels)); - // hypre_printf("Relax type: %d\n", (mgr_data -> relax_type)); - hypre_printf("Set non-Cpoints to F-points: %d\n", (mgr_data -> set_non_Cpoints_to_F)); - hypre_printf("Set Cpoints method: %d\n", (mgr_data -> set_c_points_method)); - for (i = 0; i < max_num_coarse_levels; i++) + hypre_ParMGRData *mgr_data = (hypre_ParMGRData*) mgr_vdata; + HYPRE_Int print_level = (mgr_data -> print_level); + hypre_ParCSRMatrix *par_A = (mgr_data -> A_array)[0]; + hypre_ParVector *par_b = (mgr_data -> F_array)[0]; + HYPRE_Int *point_marker_array = (mgr_data -> point_marker_array); + HYPRE_Int block_size = (mgr_data -> block_size); + char *info_path = (mgr_data -> info_path); + + char topdir[] = "./hypre-data"; + char *filename = NULL; + hypre_IntArray *dofmap = NULL; + MPI_Comm comm; + HYPRE_Int myid; + HYPRE_Int info_path_length; + + /* Sanity check */ + if (!par_A) { - hypre_printf("Lev = %d, Interpolation type: %d\n", i, (mgr_data -> interp_type)[i]); - hypre_printf("Lev = %d, Restriction type: %d\n", i, (mgr_data -> restrict_type)[i]); - hypre_printf("Lev = %d, F-relaxation type: %d\n", i, (mgr_data -> Frelax_type)[i]); - hypre_printf("lev = %d, Number of relax sweeps: %d\n", i, (mgr_data -> num_relax_sweeps)[i]); - hypre_printf("Lev = %d, Use non-Galerkin coarse grid: %d\n", i, - (mgr_data -> mgr_coarse_grid_method)[i]); - HYPRE_Int lvl_num_coarse_points = (mgr_data -> block_num_coarse_indexes)[i]; - hypre_printf("Lev = %d, Number of Cpoints: %d\n", i, lvl_num_coarse_points); - hypre_printf("Cpoints indices: "); - for (j = 0; j < lvl_num_coarse_points; j++) + return hypre_error_flag; + } + + /* Get rank ID */ + comm = hypre_ParCSRMatrixComm(par_A); + hypre_MPI_Comm_rank(comm, &myid); + + /* Create new "ls_" folder (info_path) */ + if (((print_level & HYPRE_MGR_PRINT_INFO_PARAMS) || + (print_level & HYPRE_MGR_PRINT_FINE_MATRIX) || + (print_level & HYPRE_MGR_PRINT_FINE_RHS)) && + (info_path == NULL)) + { + if (!myid) { - if ((mgr_data -> block_cf_marker)[i][j] == 1) + if (!hypre_CheckDirExists(topdir)) { - hypre_printf("%d ", j); + hypre_CreateDir(topdir); + } + + hypre_CreateNextDirOfSequence(topdir, "ls_", &info_path); + info_path_length = strlen(info_path) + 1; + } + hypre_MPI_Bcast(&info_path_length, 1, HYPRE_MPI_INT, 0, comm); + + if (info_path_length > 0) + { + if (myid) + { + info_path = hypre_TAlloc(char, info_path_length, HYPRE_MEMORY_HOST); } } - hypre_printf("\n"); - } - hypre_printf("Number of Reserved Cpoints: %d\n", (mgr_data -> reserved_coarse_size)); - hypre_printf("Keep reserved Cpoints to level: %d\n", (mgr_data -> lvl_to_keep_cpoints)); - - hypre_printf("\n MGR Solver Parameters: \n"); - hypre_printf("Number of interpolation sweeps: %d\n", (mgr_data -> num_interp_sweeps)); - hypre_printf("Number of restriction sweeps: %d\n", (mgr_data -> num_restrict_sweeps)); - if (mgr_data -> level_smooth_type != NULL) - { - hypre_printf("Global smoother type: %d\n", (mgr_data -> level_smooth_type)[0]); - hypre_printf("Number of global smoother sweeps: %d\n", (mgr_data -> level_smooth_iters)[0]); - } - hypre_printf("Max number of iterations: %d\n", (mgr_data -> max_iter)); - hypre_printf("Stopping tolerance: %e\n", (mgr_data -> tol)); - hypre_printf("Use default coarse grid solver: %d\n", (mgr_data -> use_default_cgrid_solver)); - /* - if ((mgr_data -> fsolver_mode) >= 0) + else { - hypre_printf("Use AMG solver for full AMG F-relaxation: %d\n", (mgr_data -> fsolver_mode)); + hypre_error_w_msg(HYPRE_ERROR_GENERIC, "Unable to create info path!"); + return hypre_error_flag; } - */ + hypre_MPI_Bcast(info_path, info_path_length, hypre_MPI_CHAR, 0, comm); + + /* Save info_path */ + (mgr_data -> info_path) = info_path; + } + else + { + if (info_path) + { + info_path_length = strlen(info_path); + } + } + + /* Print MGR parameters to file */ + if (print_level & HYPRE_MGR_PRINT_INFO_PARAMS) + { + /* TODO (VPM): print internal MGR parameters to file */ + + /* Signal that the MGR parameters have already been printed */ + (mgr_data -> print_level) &= ~HYPRE_MGR_PRINT_INFO_PARAMS; + (mgr_data -> print_level) |= HYPRE_MGR_PRINT_RESERVED_A; + } + + /* Print linear system matrix at the finest level and dofmap */ + if ((print_level & HYPRE_MGR_PRINT_FINE_MATRIX) && par_A) + { + /* Build dofmap array */ + dofmap = hypre_IntArrayCreate(hypre_ParCSRMatrixNumRows(par_A)); + hypre_IntArrayInitialize_v2(dofmap, HYPRE_MEMORY_HOST); + if (point_marker_array) + { + hypre_TMemcpy(hypre_IntArrayData(dofmap), point_marker_array, + HYPRE_Int, hypre_ParCSRMatrixNumRows(par_A), + HYPRE_MEMORY_HOST, HYPRE_MEMORY_HOST); + } + else + { + hypre_IntArraySetInterleavedValues(dofmap, block_size); + } + + /* Print Matrix */ + hypre_ParPrintf(comm, "Writing matrix to path: %s\n", info_path); + filename = hypre_TAlloc(char, strlen(info_path) + 16, HYPRE_MEMORY_HOST); + hypre_sprintf(filename, "%s/IJ.out.A", info_path); + if (print_level & HYPRE_MGR_PRINT_MODE_ASCII) + { + hypre_ParCSRMatrixPrintIJ(par_A, 0, 0, filename); + } + else + { + hypre_ParCSRMatrixPrintBinaryIJ(par_A, 0, 0, filename); + } + + /* Print dofmap */ + hypre_ParPrintf(comm, "Writing dofmap to path: %s\n", info_path); + hypre_sprintf(filename, "%s/dofmap.out", info_path); + hypre_IntArrayPrint(comm, dofmap, filename); + + /* Free memory */ + hypre_TFree(filename, HYPRE_MEMORY_HOST); + hypre_IntArrayDestroy(dofmap); + + /* Signal that the matrix has already been printed */ + (mgr_data -> print_level) &= ~HYPRE_MGR_PRINT_FINE_MATRIX; + (mgr_data -> print_level) |= HYPRE_MGR_PRINT_RESERVED_B; + } + + /* Print linear system RHS at the finest level */ + if ((print_level & HYPRE_MGR_PRINT_FINE_RHS) && par_b) + { + /* Print RHS */ + hypre_ParPrintf(comm, "Writing RHS to path: %s\n", info_path); + filename = hypre_TAlloc(char, strlen(info_path) + 16, HYPRE_MEMORY_HOST); + hypre_sprintf(filename, "%s/IJ.out.b", info_path); + if (print_level & HYPRE_MGR_PRINT_MODE_ASCII) + { + hypre_ParVectorPrintIJ(par_b, 0, filename); + } + else + { + hypre_ParVectorPrintBinaryIJ(par_b, filename); + } + + /* Free memory */ + hypre_TFree(filename, HYPRE_MEMORY_HOST); + + /* Signal that the vector has already been printed */ + (mgr_data -> print_level) &= ~HYPRE_MGR_PRINT_FINE_RHS; + (mgr_data -> print_level) |= HYPRE_MGR_PRINT_RESERVED_C; + } + return hypre_error_flag; } +/*************************************************************************** + ***************************************************************************/ + #ifdef HYPRE_USING_DSUPERLU void * hypre_MGRDirectSolverCreate() diff --git a/src/parcsr_ls/par_mgr.h b/src/parcsr_ls/par_mgr.h index 94b05080b..43f7ca44a 100644 --- a/src/parcsr_ls/par_mgr.h +++ b/src/parcsr_ls/par_mgr.h @@ -8,6 +8,24 @@ #ifndef hypre_ParMGR_DATA_HEADER #define hypre_ParMGR_DATA_HEADER +/*-------------------------------------------------------------------------- + * MGR print level codes + *--------------------------------------------------------------------------*/ + +#define HYPRE_MGR_PRINT_INFO_SETUP 0x01 /* 1 (1st bit) */ +#define HYPRE_MGR_PRINT_INFO_SOLVE 0x02 /* 2 (2nd bit) */ +#define HYPRE_MGR_PRINT_INFO_PARAMS 0x04 /* 4 (3rd bit) */ +#define HYPRE_MGR_PRINT_MODE_ASCII 0x08 /* 8 (4th bit) */ +#define HYPRE_MGR_PRINT_FINE_MATRIX 0x10 /* 16 (5th bit) */ +#define HYPRE_MGR_PRINT_FINE_RHS 0x20 /* 32 (6th bit) */ +#define HYPRE_MGR_PRINT_CRSE_MATRIX 0x40 /* 64 (7th bit) */ +#define HYPRE_MGR_PRINT_LVLS_MATRIX 0x80 /* 128 (8th bit) */ +/* ... */ +/* Reserved codes */ +#define HYPRE_MGR_PRINT_RESERVED_C 0x10000000 /* 268435456 (29th bit) */ +#define HYPRE_MGR_PRINT_RESERVED_B 0x20000000 /* 536870912 (30th bit) */ +#define HYPRE_MGR_PRINT_RESERVED_A 0x40000000 /* 1073741824 (31th bit) */ + /*-------------------------------------------------------------------------- * hypre_ParMGRData *--------------------------------------------------------------------------*/ @@ -73,6 +91,7 @@ typedef struct HYPRE_Int max_iter; HYPRE_Int relax_order; HYPRE_Int *num_relax_sweeps; + char *info_path; HYPRE_Solver coarse_grid_solver; HYPRE_Int (*coarse_grid_solver_setup)(void*, void*, void*, void*); @@ -212,7 +231,7 @@ typedef struct #define hypre_ParMGRDataRelaxType(data) ((data) -> relax_type) #define hypre_ParMGRDataFRelaxType(data) ((data) -> Frelax_type) #define hypre_ParMGRDataFRelaxTypeI(data, i) ((data) -> Frelax_type[i]) -#define hypre_ParMGRDataAFFsolver(data) ((data) -> aff_solver) /* TODO (VPM): does aff_solver need to be a double pointer? */ +#define hypre_ParMGRDataAFFsolver(data) ((data) -> aff_solver) #define hypre_ParMGRDataAFFsolverI(data) ((data) -> aff_solver[i]) #define hypre_ParMGRDataCoarseGridMethod(data) ((data) -> mgr_coarse_grid_method) diff --git a/src/parcsr_ls/par_mgr_setup.c b/src/parcsr_ls/par_mgr_setup.c index e3bd613d8..aaba3e033 100644 --- a/src/parcsr_ls/par_mgr_setup.c +++ b/src/parcsr_ls/par_mgr_setup.c @@ -177,6 +177,16 @@ hypre_MGRSetup( void *mgr_vdata, hypre_MPI_Comm_size(comm, &num_procs); hypre_MPI_Comm_rank(comm, &my_id); + /* Reset print_level codes. This is useful for printing + information when solving a sequence of linear systems */ + print_level |= ((print_level & HYPRE_MGR_PRINT_RESERVED_A) == HYPRE_MGR_PRINT_RESERVED_A) ? + HYPRE_MGR_PRINT_INFO_PARAMS : 0; + print_level |= ((print_level & HYPRE_MGR_PRINT_RESERVED_B) == HYPRE_MGR_PRINT_RESERVED_B) ? + HYPRE_MGR_PRINT_FINE_MATRIX : 0; + print_level |= ((print_level & HYPRE_MGR_PRINT_RESERVED_C) == HYPRE_MGR_PRINT_RESERVED_C) ? + HYPRE_MGR_PRINT_FINE_RHS : 0; + (mgr_data -> print_level) = print_level; + /* Trivial case: simply solve the coarse level problem */ if (block_size < 2 || (mgr_data -> max_num_coarse_levels) < 1) { @@ -1924,6 +1934,9 @@ hypre_MGRSetup( void *mgr_vdata, /* Print statistics */ hypre_MGRSetupStats(mgr_vdata); + /* Print MGR and linear system info according to print level */ + hypre_MGRDataPrint(mgr_vdata); + HYPRE_ANNOTATE_FUNC_END; hypre_GpuProfilingPopRange(); diff --git a/src/parcsr_ls/par_mgr_solve.c b/src/parcsr_ls/par_mgr_solve.c index 450274927..0269699b6 100644 --- a/src/parcsr_ls/par_mgr_solve.c +++ b/src/parcsr_ls/par_mgr_solve.c @@ -52,14 +52,7 @@ hypre_MGRSolve( void *mgr_vdata, HYPRE_Real ieee_check = 0.; HYPRE_Int iter, num_procs, my_id; - HYPRE_Int Solve_err_flag; - /* - HYPRE_Real total_coeffs; - HYPRE_Real total_variables; - HYPRE_Real operat_cmplxty; - HYPRE_Real grid_cmplxty; - */ HYPRE_Solver cg_solver = (mgr_data -> coarse_grid_solver); HYPRE_Int (*coarse_grid_solver_solve)(void*, void*, void*, void*) = (mgr_data -> coarse_grid_solver_solve); @@ -94,36 +87,24 @@ hypre_MGRSolve( void *mgr_vdata, /*----------------------------------------------------------------------- * Write the solver parameters *-----------------------------------------------------------------------*/ - if (my_id == 0 && print_level > 1) - { - hypre_MGRWriteSolverParams(mgr_data); - } - /*----------------------------------------------------------------------- - * Initialize the solver error flag and assorted bookkeeping variables - *-----------------------------------------------------------------------*/ + /* Print MGR and linear system info according to print level */ + hypre_MGRDataPrint(mgr_vdata); - Solve_err_flag = 0; - /* - total_coeffs = 0; - total_variables = 0; - operat_cmplxty = 0; - grid_cmplxty = 0; - */ /*----------------------------------------------------------------------- * write some initial info *-----------------------------------------------------------------------*/ - if (my_id == 0 && print_level > 1 && tol > 0.) + if (my_id == 0 && (print_level & HYPRE_MGR_PRINT_INFO_SOLVE) && tol > 0.) { - hypre_printf("\n\nTWO-GRID SOLVER SOLUTION INFO:\n"); + hypre_printf("\n\nMGR SOLVER SOLUTION INFO:\n"); } - /*----------------------------------------------------------------------- * Compute initial fine-grid residual and print *-----------------------------------------------------------------------*/ - if (print_level > 1 || logging > 1 || tol > 0.) + + if ((print_level & HYPRE_MGR_PRINT_INFO_SOLVE) || logging > 1 || tol > 0.) { if (logging > 1) { @@ -196,7 +177,7 @@ hypre_MGRSolve( void *mgr_vdata, rel_resnorm = 1.; } - if (my_id == 0 && print_level > 1) + if (my_id == 0 && (print_level & HYPRE_MGR_PRINT_INFO_SOLVE)) { hypre_printf(" relative\n"); hypre_printf(" residual factor residual\n"); @@ -216,7 +197,7 @@ hypre_MGRSolve( void *mgr_vdata, * Compute fine-grid residual and residual norm *----------------------------------------------------------------*/ - if (print_level > 1 || logging > 1 || tol > 0.) + if ((print_level & HYPRE_MGR_PRINT_INFO_SOLVE) || logging > 1 || tol > 0.) { old_resnorm = resnorm; @@ -242,7 +223,7 @@ hypre_MGRSolve( void *mgr_vdata, (mgr_data -> num_iterations) = iter; (mgr_data -> final_rel_residual_norm) = rel_resnorm; - if (my_id == 0 && print_level > 1) + if (my_id == 0 && (print_level & HYPRE_MGR_PRINT_INFO_SOLVE)) { hypre_printf(" MGRCycle %2d %e %f %e \n", iter, resnorm, conv_factor, rel_resnorm); @@ -252,43 +233,32 @@ hypre_MGRSolve( void *mgr_vdata, /* check convergence within max_iter */ if (iter == max_iter && tol > 0.) { - Solve_err_flag = 1; hypre_error(HYPRE_ERROR_CONV); - } - /*----------------------------------------------------------------------- - * Print closing statistics - * Add operator and grid complexity stats - *-----------------------------------------------------------------------*/ - - if (iter > 0 && init_resnorm) - { - conv_factor = hypre_pow((resnorm / init_resnorm), (fp_one / (HYPRE_Real) iter)); - } - else - { - conv_factor = fp_one; - } - - if (print_level > 1) - { - /*** compute operator and grid complexities here ?? ***/ - if (my_id == 0) + if (!my_id && (print_level & HYPRE_MGR_PRINT_INFO_SOLVE)) { - if (Solve_err_flag == 1) - { - hypre_printf("\n\n=============================================="); - hypre_printf("\n NOTE: Convergence tolerance was not achieved\n"); - hypre_printf(" within the allowed %d iterations\n", max_iter); - hypre_printf("=============================================="); - } - hypre_printf("\n\n Average Convergence Factor = %f \n", conv_factor); - hypre_printf(" Number of coarse levels = %d \n", (mgr_data -> num_coarse_levels)); - // hypre_printf("\n\n Complexity: grid = %f\n",grid_cmplxty); - // hypre_printf(" operator = %f\n",operat_cmplxty); - // hypre_printf(" cycle = %f\n\n\n\n",cycle_cmplxty); + hypre_printf("\n\n=============================================="); + hypre_printf("\n NOTE: Convergence tolerance was not achieved\n"); + hypre_printf(" within the allowed %d iterations\n", max_iter); + hypre_printf("=============================================="); } } + + if ((my_id == 0) && (print_level & HYPRE_MGR_PRINT_INFO_SOLVE)) + { + if (iter > 0 && init_resnorm) + { + conv_factor = hypre_pow((resnorm / init_resnorm), + (fp_one / (HYPRE_Real) iter)); + } + else + { + conv_factor = fp_one; + } + + hypre_printf("\n\n Average Convergence Factor = %f \n", conv_factor); + } + HYPRE_ANNOTATE_FUNC_END; return hypre_error_flag; diff --git a/src/parcsr_ls/par_mgr_stats.c b/src/parcsr_ls/par_mgr_stats.c index 02fd12fba..7026fdbc5 100644 --- a/src/parcsr_ls/par_mgr_stats.c +++ b/src/parcsr_ls/par_mgr_stats.c @@ -13,7 +13,7 @@ * hypre_MGRGetGlobalRelaxName *--------------------------------------------------------------------*/ -char* +const char* hypre_MGRGetGlobalRelaxName(hypre_ParMGRData *mgr_data, HYPRE_Int level ) { @@ -119,7 +119,7 @@ hypre_MGRGetGlobalRelaxName(hypre_ParMGRData *mgr_data, * hypre_MGRGetFRelaxName *--------------------------------------------------------------------*/ -char* +const char* hypre_MGRGetFRelaxName(hypre_ParMGRData *mgr_data, HYPRE_Int level ) { @@ -193,7 +193,7 @@ hypre_MGRGetFRelaxName(hypre_ParMGRData *mgr_data, * hypre_MGRGetProlongationName *--------------------------------------------------------------------*/ -char* +const char* hypre_MGRGetProlongationName(hypre_ParMGRData *mgr_data, HYPRE_Int level ) { @@ -232,7 +232,7 @@ hypre_MGRGetProlongationName(hypre_ParMGRData *mgr_data, * hypre_MGRGetRestrictionName *--------------------------------------------------------------------*/ -char* +const char* hypre_MGRGetRestrictionName(hypre_ParMGRData *mgr_data, HYPRE_Int level ) { @@ -265,7 +265,7 @@ hypre_MGRGetRestrictionName(hypre_ParMGRData *mgr_data, * hypre_MGRGetCoarseGridName *--------------------------------------------------------------------*/ -char* +const char* hypre_MGRGetCoarseGridName(hypre_ParMGRData *mgr_data, HYPRE_Int level ) { @@ -339,7 +339,7 @@ hypre_MGRSetupStats(void *mgr_vdata) HYPRE_Int divisors[1]; /* Print statistics only if first print_level bit is set */ - if (!(print_level & 0x1)) + if (!(print_level & HYPRE_MGR_PRINT_INFO_SETUP)) { return hypre_error_flag; } @@ -356,14 +356,15 @@ hypre_MGRSetupStats(void *mgr_vdata) num_sublevels_amg = hypre_CTAlloc(HYPRE_Int, num_levels_mgr + 1, HYPRE_MEMORY_HOST); /* Check MGR's coarse level solver */ - if ((void*) hypre_ParMGRDataCoarseGridSolverSetup(mgr_data) == (void*) HYPRE_BoomerAMGSetup) + if ((HYPRE_PtrToParSolverFcn) hypre_ParMGRDataCoarseGridSolverSetup(mgr_data) == + HYPRE_BoomerAMGSetup) { coarse_amg_solver = (hypre_ParAMGData *) coarse_solver; num_sublevels_amg[coarsest_mgr_level] = hypre_ParAMGDataNumLevels(coarse_amg_solver); } #ifdef HYPRE_USING_DSUPERLU - else if ((void*) hypre_ParMGRDataCoarseGridSolverSetup(mgr_data) == - (void*) hypre_MGRDirectSolverSetup) + else if ((HYPRE_PtrToParSolverFcn) hypre_ParMGRDataCoarseGridSolverSetup(mgr_data) == + hypre_MGRDirectSolverSetup) { /* TODO (VPM): Set SuperLU solver specifics */ num_sublevels_amg[coarsest_mgr_level] = 0; @@ -371,6 +372,7 @@ hypre_MGRSetupStats(void *mgr_vdata) #endif else { + hypre_TFree(num_sublevels_amg, HYPRE_MEMORY_HOST); hypre_error_w_msg(HYPRE_ERROR_GENERIC, "Unknown coarsest level solver for MGR!\n"); return hypre_error_flag; } @@ -450,11 +452,11 @@ hypre_MGRSetupStats(void *mgr_vdata) if (!myid) { - char *msg[] = { "Full Operator Matrix Hierarchy Information:\n\n", - "MGR's coarsest level", - "\t( MGR )", - "\t( AMG )" - }; + const char *msg[] = { "Full Operator Matrix Hierarchy Information:\n\n", + "MGR's coarsest level", + "\t( MGR )", + "\t( AMG )" + }; num_levels[0] = num_levels_mgr - 1; num_levels[1] = num_sublevels_amg[coarsest_mgr_level] + 1; @@ -483,11 +485,11 @@ hypre_MGRSetupStats(void *mgr_vdata) if (!myid) { - char *msg[] = { "Full Prolongation Matrix Hierarchy Information:\n\n", - "MGR's coarsest level", - "\t( MGR )", - "\t( AMG )" - }; + const char *msg[] = { "Full Prolongation Matrix Hierarchy Information:\n\n", + "MGR's coarsest level", + "\t( MGR )", + "\t( AMG )" + }; num_levels[0] = num_levels_mgr; num_levels[1] = num_sublevels_amg[coarsest_mgr_level] - 1; @@ -526,7 +528,7 @@ hypre_MGRSetupStats(void *mgr_vdata) /* Print A matrices info */ if (!myid) { - char *msg[] = {"Operator Matrix Hierarchy Information:\n\n"}; + const char *msg[] = {"Operator Matrix Hierarchy Information:\n\n"}; num_levels[0] = num_sublevels_amg[i]; hypre_MatrixStatsArrayPrint(1, num_levels, 1, 3, msg, stats_array); } @@ -543,7 +545,7 @@ hypre_MGRSetupStats(void *mgr_vdata) /* Print P matrices info */ if (!myid) { - char *msg[] = {"Prolongation Matrix Hierarchy Information:\n\n"}; + const char *msg[] = {"Prolongation Matrix Hierarchy Information:\n\n"}; num_levels[0] = num_sublevels_amg[i] - 1; hypre_MatrixStatsArrayPrint(1, num_levels, 1, 3, msg, stats_array); } diff --git a/src/parcsr_ls/par_relax_more_device.c b/src/parcsr_ls/par_relax_more_device.c index 08f9f54cd..5c21011ae 100644 --- a/src/parcsr_ls/par_relax_more_device.c +++ b/src/parcsr_ls/par_relax_more_device.c @@ -307,7 +307,7 @@ hypre_ParCSRMaxEigEstimateCGDevice(hypre_ParCSRMatrix *A, /* matrix to relax #if defined(HYPRE_USING_SYCL) HYPRE_ONEDPL_CALL(std::transform, r_data, r_data + local_size, r_data, - [] (auto x) { return 2.0 * x - 1.0; } ); + [] (auto x) { return 2.0 * x - 1.0; } ); #else HYPRE_THRUST_CALL(transform, r_data, r_data + local_size, r_data, @@ -377,7 +377,7 @@ hypre_ParCSRMaxEigEstimateCGDevice(hypre_ParCSRMatrix *A, /* matrix to relax /* u = ds .* p */ #if defined(HYPRE_USING_SYCL) HYPRE_ONEDPL_CALL( std::transform, ds_data, ds_data + local_size, p_data, u_data, - [] (auto x, auto y) { return x*y; } ); + [] (auto x, auto y) { return x * y; } ); #else HYPRE_THRUST_CALL( transform, ds_data, ds_data + local_size, p_data, u_data, _1 * _2 ); #endif @@ -387,7 +387,7 @@ hypre_ParCSRMaxEigEstimateCGDevice(hypre_ParCSRMatrix *A, /* matrix to relax /* s = ds .* s */ #if defined(HYPRE_USING_SYCL) HYPRE_ONEDPL_CALL( std::transform, ds_data, ds_data + local_size, s_data, s_data, - [] (auto x, auto y) { return x*y; } ); + [] (auto x, auto y) { return x * y; } ); #else HYPRE_THRUST_CALL( transform, ds_data, ds_data + local_size, s_data, s_data, _1 * _2 ); #endif diff --git a/src/parcsr_ls/par_stats.c b/src/parcsr_ls/par_stats.c index f78dad224..ef0d69adf 100644 --- a/src/parcsr_ls/par_stats.c +++ b/src/parcsr_ls/par_stats.c @@ -1709,7 +1709,7 @@ hypre_BoomerAMGGetCoarseningName(hypre_ParAMGData *amg_data) const char* hypre_BoomerAMGGetCycleName(hypre_ParAMGData *amg_data) { - char *name = hypre_CTAlloc(char, 10, HYPRE_MEMORY_HOST); + static char name[10]; switch (hypre_ParAMGDataCycleType(amg_data)) { diff --git a/src/parcsr_ls/protos.h b/src/parcsr_ls/protos.h index b7f30a875..6ee8d3b24 100644 --- a/src/parcsr_ls/protos.h +++ b/src/parcsr_ls/protos.h @@ -2227,7 +2227,6 @@ HYPRE_Int hypre_MGRComputeNonGalerkinCoarseGrid( hypre_ParCSRMatrix *A, hypre_Pa HYPRE_Int ordering, HYPRE_Int method, HYPRE_Int Pmax, HYPRE_Int *CF_marker, hypre_ParCSRMatrix **A_H_ptr ); -HYPRE_Int hypre_MGRWriteSolverParams( void *mgr_vdata ); HYPRE_Int hypre_MGRSetAffSolverType( void *systg_vdata, HYPRE_Int *aff_solver_type ); HYPRE_Int hypre_MGRSetCoarseSolverType( void *systg_vdata, HYPRE_Int coarse_solver_type ); HYPRE_Int hypre_MGRSetCoarseSolverIter( void *systg_vdata, HYPRE_Int coarse_solver_iter ); @@ -2262,6 +2261,7 @@ HYPRE_Int hypre_MGRSetMaxIter( void *mgr_vdata, HYPRE_Int max_iter ); HYPRE_Int hypre_MGRSetPMaxElmts( void *mgr_vdata, HYPRE_Int P_max_elmts ); HYPRE_Int hypre_MGRSetLevelPMaxElmts( void *mgr_vdata, HYPRE_Int *P_max_elmts ); HYPRE_Int hypre_MGRSetTol( void *mgr_vdata, HYPRE_Real tol ); +HYPRE_Int hypre_MGRDataPrint(void *mgr_vdata); #ifdef HYPRE_USING_DSUPERLU void *hypre_MGRDirectSolverCreate( void ); HYPRE_Int hypre_MGRDirectSolverSetup( void *solver, hypre_ParCSRMatrix *A, diff --git a/src/parcsr_mv/CMakeLists.txt b/src/parcsr_mv/CMakeLists.txt index 1ae35c094..00ac6a5bf 100644 --- a/src/parcsr_mv/CMakeLists.txt +++ b/src/parcsr_mv/CMakeLists.txt @@ -28,7 +28,6 @@ set(SRCS par_csr_matrix.c par_csr_matrix_stats.c par_csr_matop_marked.c - par_csr_matstats.c par_csr_matvec.c par_csr_matvec_device.c par_vector.c @@ -48,7 +47,6 @@ target_sources(${PROJECT_NAME} if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) set(GPU_SRCS - par_csr_matstats.c par_csr_matvec_device.c par_csr_fffc_device.c par_csr_matop_device.c diff --git a/src/parcsr_mv/par_csr_matrix_stats.c b/src/parcsr_mv/par_csr_matrix_stats.c index 11c872bfa..7925d1a20 100644 --- a/src/parcsr_mv/par_csr_matrix_stats.c +++ b/src/parcsr_mv/par_csr_matrix_stats.c @@ -254,6 +254,8 @@ hypre_ParCSRMatrixStatsComputePassTwoLocalHost(hypre_ParCSRMatrix *A, /* Free memory */ hypre_TFree(nnzrow_sqsum, HYPRE_MEMORY_HOST); hypre_TFree(rowsum_sqsum, HYPRE_MEMORY_HOST); + hypre_TFree(nnzrow_avg, HYPRE_MEMORY_HOST); + hypre_TFree(rowsum_avg, HYPRE_MEMORY_HOST); return hypre_error_flag; } diff --git a/src/seq_mv/csr_matop_device.c b/src/seq_mv/csr_matop_device.c index 3178ee8d1..a70caf362 100644 --- a/src/seq_mv/csr_matop_device.c +++ b/src/seq_mv/csr_matop_device.c @@ -1639,10 +1639,10 @@ hypre_CSRMatrixCheckDiagFirstDevice( hypre_CSRMatrix *A ) __global__ void hypreGPUKernel_CSRMatrixCheckForMissingDiagonal( hypre_DeviceItem &item, - HYPRE_Int nrows, - HYPRE_Int *ia, - HYPRE_Int *ja, - HYPRE_Int *result ) + HYPRE_Int nrows, + HYPRE_Int *ia, + HYPRE_Int *ja, + HYPRE_Int *result ) { const HYPRE_Int row = hypre_gpu_get_grid_warp_id<1, 1>(item); @@ -3206,11 +3206,11 @@ hypre_SortCSRRocsparse( HYPRE_Int n, HYPRE_Int hypre_CSRMatrixTriLowerUpperSolveOnemklsparse(char uplo, - HYPRE_Int unit_diag, - hypre_CSRMatrix *A, - HYPRE_Real *l1_norms, - HYPRE_Complex *f_data, - HYPRE_Complex *u_data ) + HYPRE_Int unit_diag, + hypre_CSRMatrix *A, + HYPRE_Real *l1_norms, + HYPRE_Complex *f_data, + HYPRE_Complex *u_data ) { HYPRE_Int *A_j = hypre_CSRMatrixJ(A); HYPRE_Complex *A_a = hypre_CSRMatrixData(A); @@ -3252,22 +3252,22 @@ hypre_CSRMatrixTriLowerUpperSolveOnemklsparse(char uplo, (!hypre_CsrsvDataAnalyzedU(csrsv_data) && uplo == 'U') ) { HYPRE_ONEMKL_CALL( oneapi::mkl::sparse::optimize_trsv( *hypre_HandleComputeStream(hypre_handle()), - (uplo == 'L') ? oneapi::mkl::uplo::L : oneapi::mkl::uplo::U, - oneapi::mkl::transpose::N, - unit_diag ? oneapi::mkl::diag::U : oneapi::mkl::diag::N, - handle_A, - {} ).wait() ); + (uplo == 'L') ? oneapi::mkl::uplo::L : oneapi::mkl::uplo::U, + oneapi::mkl::transpose::N, + unit_diag ? oneapi::mkl::diag::U : oneapi::mkl::diag::N, + handle_A, + {} ).wait() ); } /* Do the triangular solve */ HYPRE_ONEMKL_CALL( oneapi::mkl::sparse::trsv( *hypre_HandleComputeStream(hypre_handle()), - (uplo == 'L') ? oneapi::mkl::uplo::L : oneapi::mkl::uplo::U, - oneapi::mkl::transpose::N, - unit_diag ? oneapi::mkl::diag::U : oneapi::mkl::diag::N, - handle_A, - f_data, - u_data, - {} ).wait() ); + (uplo == 'L') ? oneapi::mkl::uplo::L : oneapi::mkl::uplo::U, + oneapi::mkl::transpose::N, + unit_diag ? oneapi::mkl::diag::U : oneapi::mkl::diag::N, + handle_A, + f_data, + u_data, + {} ).wait() ); /* Restore the original matrix data */ hypre_CSRMatrixJ(A) = A_j; diff --git a/src/utilities/_hypre_utilities.h b/src/utilities/_hypre_utilities.h index df77915d4..11f064f5a 100644 --- a/src/utilities/_hypre_utilities.h +++ b/src/utilities/_hypre_utilities.h @@ -2005,7 +2005,8 @@ HYPRE_Int hypre_MatrixStatsDestroy( hypre_MatrixStats *stats ); hypre_MatrixStatsArray* hypre_MatrixStatsArrayCreate( HYPRE_Int capacity ); HYPRE_Int hypre_MatrixStatsArrayDestroy( hypre_MatrixStatsArray *stats_array ); HYPRE_Int hypre_MatrixStatsArrayPrint( HYPRE_Int num_hierarchies, HYPRE_Int *num_levels, - HYPRE_Int use_divisors, HYPRE_Int shift, char **messages, + HYPRE_Int use_divisors, HYPRE_Int shift, + const char **messages, hypre_MatrixStatsArray *stats_array ); /* qsort.c */ @@ -2306,6 +2307,10 @@ void hypre_GpuProfilingPopRange(void); HYPRE_Int hypre_multmod(HYPRE_Int a, HYPRE_Int b, HYPRE_Int mod); void hypre_partition1D(HYPRE_Int n, HYPRE_Int p, HYPRE_Int j, HYPRE_Int *s, HYPRE_Int *e); char *hypre_strcpy(char *destination, const char *source); +HYPRE_Int hypre_CheckDirExists(const char *path); +HYPRE_Int hypre_CreateDir(const char *path); +HYPRE_Int hypre_CreateNextDirOfSequence(const char *basepath, const char *prefix, + char **fullpath_ptr); HYPRE_Int hypre_SetSyncCudaCompute(HYPRE_Int action); HYPRE_Int hypre_RestoreSyncCudaCompute(void); diff --git a/src/utilities/matrix_stats.c b/src/utilities/matrix_stats.c index fce13ff22..a309ffcb1 100644 --- a/src/utilities/matrix_stats.c +++ b/src/utilities/matrix_stats.c @@ -110,7 +110,7 @@ hypre_MatrixStatsArrayDestroy(hypre_MatrixStatsArray *stats_array) { hypre_MatrixStatsDestroy(hypre_MatrixStatsArrayEntry(stats_array, i)); } - + hypre_TFree(hypre_MatrixStatsArrayEntries(stats_array), HYPRE_MEMORY_HOST); hypre_TFree(stats_array, HYPRE_MEMORY_HOST); } @@ -126,7 +126,7 @@ hypre_MatrixStatsArrayPrint(HYPRE_Int num_hierarchies, HYPRE_Int *num_levels, HYPRE_Int use_divisors, HYPRE_Int shift, - char **messages, + const char **messages, hypre_MatrixStatsArray *stats_array) { HYPRE_Int capacity = hypre_MatrixStatsArrayCapacity(stats_array); @@ -230,15 +230,15 @@ hypre_MatrixStatsArrayPrint(HYPRE_Int num_hierarchies, ndigits[1] = hypre_max(ndigits[1], 1 + hypre_ndigits(fine_num_rows)); ndigits[3] = hypre_max(ndigits[3], - 1 + hypre_ndigits(hypre_MatrixStatsSparsity(stats))); + 4 + hypre_ndigits((HYPRE_Int) hypre_MatrixStatsSparsity(stats))); ndigits[4] = hypre_max(ndigits[4], 1 + hypre_ndigits(hypre_MatrixStatsNnzrowMin(stats))); ndigits[5] = hypre_max(ndigits[5], 1 + hypre_ndigits(hypre_MatrixStatsNnzrowMax(stats))); ndigits[6] = hypre_max(ndigits[6], - 4 + hypre_ndigits(hypre_MatrixStatsNnzrowAvg(stats))); + 4 + hypre_ndigits((HYPRE_Int) hypre_MatrixStatsNnzrowAvg(stats))); ndigits[7] = hypre_max(ndigits[7], - 4 + hypre_ndigits(hypre_MatrixStatsNnzrowStDev(stats))); + 4 + hypre_ndigits((HYPRE_Int) hypre_MatrixStatsNnzrowStDev(stats))); } /* Column offsets calculation */ diff --git a/src/utilities/protos.h b/src/utilities/protos.h index 780dbce71..14c959894 100644 --- a/src/utilities/protos.h +++ b/src/utilities/protos.h @@ -62,7 +62,8 @@ HYPRE_Int hypre_MatrixStatsDestroy( hypre_MatrixStats *stats ); hypre_MatrixStatsArray* hypre_MatrixStatsArrayCreate( HYPRE_Int capacity ); HYPRE_Int hypre_MatrixStatsArrayDestroy( hypre_MatrixStatsArray *stats_array ); HYPRE_Int hypre_MatrixStatsArrayPrint( HYPRE_Int num_hierarchies, HYPRE_Int *num_levels, - HYPRE_Int use_divisors, HYPRE_Int shift, char **messages, + HYPRE_Int use_divisors, HYPRE_Int shift, + const char **messages, hypre_MatrixStatsArray *stats_array ); /* qsort.c */ @@ -363,6 +364,10 @@ void hypre_GpuProfilingPopRange(void); HYPRE_Int hypre_multmod(HYPRE_Int a, HYPRE_Int b, HYPRE_Int mod); void hypre_partition1D(HYPRE_Int n, HYPRE_Int p, HYPRE_Int j, HYPRE_Int *s, HYPRE_Int *e); char *hypre_strcpy(char *destination, const char *source); +HYPRE_Int hypre_CheckDirExists(const char *path); +HYPRE_Int hypre_CreateDir(const char *path); +HYPRE_Int hypre_CreateNextDirOfSequence(const char *basepath, const char *prefix, + char **fullpath_ptr); HYPRE_Int hypre_SetSyncCudaCompute(HYPRE_Int action); HYPRE_Int hypre_RestoreSyncCudaCompute(void); diff --git a/src/utilities/utilities.c b/src/utilities/utilities.c index d233daec1..878db97a1 100644 --- a/src/utilities/utilities.c +++ b/src/utilities/utilities.c @@ -7,6 +7,16 @@ #include "_hypre_utilities.h" +#include +#include +#ifdef _WIN32 +#include +#define mkdir(path, mode) _mkdir(path) +#else +#include +#include +#endif + /*-------------------------------------------------------------------------- * hypre_multmod *--------------------------------------------------------------------------*/ @@ -88,3 +98,85 @@ hypre_strcpy(char *destination, const char *source) return ((char *) memmove(destination, source, len + 1)); } } + +/*-------------------------------------------------------------------------- + * hypre_CheckDirExists + *--------------------------------------------------------------------------*/ + +HYPRE_Int +hypre_CheckDirExists(const char *path) +{ + DIR *dir = opendir(path); + + if (dir) + { + closedir(dir); + return 1; + } + return 0; +} + +/*-------------------------------------------------------------------------- + * hypre_CreateDir + *--------------------------------------------------------------------------*/ + +HYPRE_Int +hypre_CreateDir(const char *path) +{ + char msg[HYPRE_MAX_MSG_LEN]; + + if (mkdir(path, 0777)) + { + hypre_sprintf(msg, "Could not create directory: %s", path); + hypre_error_w_msg(HYPRE_ERROR_GENERIC, msg); + } + + return hypre_error_flag; +} + +/*-------------------------------------------------------------------------- + * hypre_CreateNextDirOfSequence + *--------------------------------------------------------------------------*/ + +HYPRE_Int +hypre_CreateNextDirOfSequence(const char *basepath, const char *prefix, char **fullpath_ptr) +{ + DIR *dir; + struct dirent *entry; + HYPRE_Int max_suffix, suffix; + char msg[HYPRE_MAX_MSG_LEN]; + char *fullpath; + + if ((dir = opendir(basepath)) == NULL) + { + hypre_sprintf(msg, "Could not open directory: %s", basepath); + hypre_error_w_msg(HYPRE_ERROR_GENERIC, msg); + return hypre_error_flag; + } + + max_suffix = -1; + while ((entry = readdir(dir)) != NULL) + { + if (strncmp(entry->d_name, prefix, strlen(prefix)) == 0) + { + if (hypre_sscanf(entry->d_name + strlen(prefix), "%d", &suffix) == 1) + { + if (suffix > max_suffix) + { + max_suffix = suffix; + } + } + } + } + closedir(dir); + + /* Create directory */ + fullpath = hypre_TAlloc(char, strlen(basepath) + 10, HYPRE_MEMORY_HOST); + hypre_sprintf(fullpath, "%s/%s%05d", basepath, prefix, max_suffix + 1); + hypre_CreateDir(fullpath); + + /* Set output pointer */ + *fullpath_ptr = fullpath; + + return hypre_error_flag; +}