From 26f334002f2dadbad1de9bcfd30800ba267f5824 Mon Sep 17 00:00:00 2001 From: "Victor A. Paludetto Magri" <50467563+victorapm@users.noreply.github.com> Date: Thu, 28 Jul 2022 15:13:34 -0700 Subject: [PATCH] [Multivec 1/5]: Fix code compilation (#692) This PR fixes a few compilation errors when building hypre with CUDA and without cusparse support --- src/parcsr_ls/par_ilu.c | 18 ++++++++--------- src/parcsr_ls/par_ilu_setup.c | 37 +++++++++++++++++------------------ src/parcsr_ls/par_ilu_solve.c | 30 +++++++++++++--------------- src/utilities/device_utils.c | 2 ++ 4 files changed, 43 insertions(+), 44 deletions(-) diff --git a/src/parcsr_ls/par_ilu.c b/src/parcsr_ls/par_ilu.c index 2b68bc71f..2e9c929c0 100644 --- a/src/parcsr_ls/par_ilu.c +++ b/src/parcsr_ls/par_ilu.c @@ -23,7 +23,7 @@ hypre_ILUCreate() ilu_data = hypre_CTAlloc(hypre_ParILUData, 1, HYPRE_MEMORY_HOST); -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) hypre_ParILUDataMatLMatrixDescription(ilu_data) = NULL; hypre_ParILUDataMatUMatrixDescription(ilu_data) = NULL; hypre_ParILUDataMatBLILUSolveInfo(ilu_data) = NULL; @@ -148,7 +148,7 @@ hypre_ILUDestroy( void *data ) { hypre_ParILUData * ilu_data = (hypre_ParILUData*) data; -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) if (hypre_ParILUDataILUSolveBuffer(ilu_data)) { hypre_TFree(hypre_ParILUDataILUSolveBuffer(ilu_data), HYPRE_MEMORY_DEVICE); @@ -373,12 +373,12 @@ hypre_ILUDestroy( void *data ) switch (hypre_ParILUDataIluType(ilu_data)) { case 10: case 11: case 40: case 41: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) if (hypre_ParILUDataIluType(ilu_data) != 10 && hypre_ParILUDataIluType(ilu_data) != 11) { #endif HYPRE_ILUDestroy(hypre_ParILUDataSchurPrecond(ilu_data)); //ILU as precond for Schur -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) } #endif break; @@ -497,12 +497,12 @@ hypre_ILUSetType( void *ilu_vdata, HYPRE_Int ilu_type ) switch (hypre_ParILUDataIluType(ilu_data)) { case 10: case 11: case 40: case 41: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) if (hypre_ParILUDataIluType(ilu_data) != 10 && hypre_ParILUDataIluType(ilu_data) != 11) { #endif HYPRE_ILUDestroy(hypre_ParILUDataSchurPrecond(ilu_data)); //ILU as precond for Schur -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) } #endif break; @@ -1740,7 +1740,7 @@ hypre_ILUBuildRASExternalMatrix(hypre_ParCSRMatrix *A, HYPRE_Int *rperm, HYPRE_I /* data objects for communication */ MPI_Comm comm = hypre_ParCSRMatrixComm(A); hypre_ParCSRCommPkg *comm_pkg; - hypre_ParCSRCommPkg *comm_pkg_tmp; + hypre_ParCSRCommPkg *comm_pkg_tmp = NULL; hypre_ParCSRCommHandle *comm_handle_count; hypre_ParCSRCommHandle *comm_handle_marker; hypre_ParCSRCommHandle *comm_handle_j; @@ -2763,7 +2763,7 @@ hypre_ILULocalRCMReverse(HYPRE_Int *perm, HYPRE_Int start, HYPRE_Int end) return hypre_error_flag; } -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /*-------------------------------------------------------------------------- * hypre_ParILUCusparseSchurGMRESDummySetup @@ -3661,7 +3661,7 @@ hypre_ParILURAPSchurGMRESMatvecDestroyH( void *matvec_data ) return 0; } -#endif +#endif /* if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) */ /* NSH create and solve and help functions */ diff --git a/src/parcsr_ls/par_ilu_setup.c b/src/parcsr_ls/par_ilu_setup.c index 06b8fda08..728d693b3 100644 --- a/src/parcsr_ls/par_ilu_setup.c +++ b/src/parcsr_ls/par_ilu_setup.c @@ -39,7 +39,7 @@ hypre_ILUSetup( void *ilu_vdata, HYPRE_Int *qperm = hypre_ParILUDataQPerm(ilu_data); HYPRE_Real tol_ddPQ = hypre_ParILUDataTolDDPQ(ilu_data); -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* pointers to cusparse data, note that they are not NULL only when needed */ cusparseMatDescr_t matL_des = hypre_ParILUDataMatLMatrixDescription(ilu_data); cusparseMatDescr_t matU_des = hypre_ParILUDataMatUMatrixDescription(ilu_data); @@ -62,6 +62,7 @@ hypre_ILUSetup( void *ilu_vdata, HYPRE_Int *A_diag_fake = hypre_ParILUDataMatAFakeDiagonal(ilu_data); hypre_Vector *Ftemp_upper = NULL; hypre_Vector *Utemp_lower = NULL; + HYPRE_Int test_opt; #endif hypre_ParCSRMatrix *matA = hypre_ParILUDataMatA(ilu_data); @@ -108,9 +109,7 @@ hypre_ILUSetup( void *ilu_vdata, HYPRE_Int buffer_size; HYPRE_Int send_size; HYPRE_Int recv_size; -#ifdef HYPRE_USING_CUDA - HYPRE_Int test_opt; -#endif + /* ----- begin -----*/ HYPRE_ANNOTATE_FUNC_BEGIN; @@ -119,7 +118,7 @@ hypre_ILUSetup( void *ilu_vdata, hypre_MPI_Comm_size(comm, &num_procs); hypre_MPI_Comm_rank(comm, &my_id); -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* create cuda and cusparse information when needed */ /* Use most of them from global information */ /* set matrix L descripter, L is a lower triangular matrix with unit diagonal entries */ @@ -362,13 +361,13 @@ hypre_ILUSetup( void *ilu_vdata, switch (ilu_type) { case 10: case 11: case 40: case 41: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) if (hypre_ParILUDataIluType(ilu_data) != 10 && hypre_ParILUDataIluType(ilu_data) != 11) { #endif HYPRE_ILUDestroy(hypre_ParILUDataSchurPrecond(ilu_data)); //ILU as precond for Schur -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) } #endif break; @@ -418,7 +417,7 @@ hypre_ILUSetup( void *ilu_vdata, switch (ilu_type) { case 0: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* only apply the setup of ILU0 with cusparse */ if (fill_level == 0) { @@ -440,7 +439,7 @@ hypre_ILUSetup( void *ilu_vdata, #endif break; case 1: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) hypre_ILUSetupILUTDevice(matA, max_row_elmts, droptol, perm, perm, n, n, matL_des, matU_des, ilu_solve_policy, &ilu_solve_buffer, &matBL_info, &matBU_info, &matSL_info, &matSU_info, &matBLU_d, &matS, @@ -451,7 +450,7 @@ hypre_ILUSetup( void *ilu_vdata, #endif break; case 10: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) if (fill_level == 0) { /* Only support ILU0 */ @@ -473,7 +472,7 @@ hypre_ILUSetup( void *ilu_vdata, #endif break; case 11: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) hypre_ILUSetupILUTDevice(matA, max_row_elmts, droptol, perm, perm, n, nLU, matL_des, matU_des, ilu_solve_policy, &ilu_solve_buffer, &matBL_info, &matBU_info, &matSL_info, &matSU_info, &matBLU_d, &matS, @@ -502,7 +501,7 @@ hypre_ILUSetup( void *ilu_vdata, &matS, &u_end); //ddPQ + GMRES + hypre_ilut() break; case 50: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) test_opt = hypre_ParILUDataTestOption(ilu_data); hypre_ILUSetupRAPILU0Device(matA, perm, n, nLU, matL_des, matU_des, ilu_solve_policy, &ilu_solve_buffer, &matAL_info, &matAU_info, &matBL_info, &matBU_info, &matSL_info, &matSU_info, @@ -514,7 +513,7 @@ hypre_ILUSetup( void *ilu_vdata, #endif break; default: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) hypre_ILUSetupILU0Device(matA, perm, perm, n, n, matL_des, matU_des, ilu_solve_policy, &ilu_solve_buffer, &matBL_info, &matBU_info, &matSL_info, &matSU_info, &matBLU_d, &matS, @@ -530,7 +529,7 @@ hypre_ILUSetup( void *ilu_vdata, case 10: case 11: if (matS) { -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* create working vectors */ Xtemp = hypre_ParVectorCreate(hypre_ParCSRMatrixComm(matS), @@ -805,7 +804,7 @@ hypre_ILUSetup( void *ilu_vdata, break; case 50: { -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) if (matS) { /* create working vectors */ @@ -1027,7 +1026,7 @@ hypre_ILUSetup( void *ilu_vdata, break; } /* set pointers to ilu data */ -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* set cusparse pointers */ //hypre_ParILUDataILUSolveBuffer(ilu_data) = ilu_solve_buffer; hypre_ParILUDataMatAILUDevice(ilu_data) = matALU_d; @@ -1077,7 +1076,7 @@ hypre_ILUSetup( void *ilu_vdata, size_C = hypre_ParCSRMatrixGlobalNumRows(matA); /* switch to compute complexity */ -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) HYPRE_Int nnzBEF = 0; HYPRE_Int nnzG;/* Global nnz */ if (ilu_type == 0 && fill_level == 0) @@ -1170,7 +1169,7 @@ hypre_ILUSetup( void *ilu_vdata, hypre_ParCSRMatrixDNumNonzeros(matL) + hypre_ParCSRMatrixDNumNonzeros(matU)) / hypre_ParCSRMatrixDNumNonzeros(matA); -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) } #endif if ((my_id == 0) && (print_level > 0)) @@ -1199,7 +1198,7 @@ hypre_ILUSetup( void *ilu_vdata, return hypre_error_flag; } -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* Extract submatrix from diagonal part of A into a new CSRMatrix without sort rows * WARNING: We don't put diagonal to the first entry of each row since this function is now for cuSparse only diff --git a/src/parcsr_ls/par_ilu_solve.c b/src/parcsr_ls/par_ilu_solve.c index b1d5a3a57..074a7b555 100644 --- a/src/parcsr_ls/par_ilu_solve.c +++ b/src/parcsr_ls/par_ilu_solve.c @@ -29,7 +29,7 @@ hypre_ILUSolve( void *ilu_vdata, hypre_ParILUData *ilu_data = (hypre_ParILUData*) ilu_vdata; -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* pointers to cusparse data, note that they are not NULL only when needed */ cusparseMatDescr_t matL_des = hypre_ParILUDataMatLMatrixDescription(ilu_data); cusparseMatDescr_t matU_des = hypre_ParILUDataMatUMatrixDescription(ilu_data); @@ -49,6 +49,10 @@ hypre_ILUSolve( void *ilu_vdata, hypre_ParCSRMatrix *Aperm = hypre_ParILUDataAperm(ilu_data); //hypre_ParCSRMatrix *R = hypre_ParILUDataR(ilu_data); //hypre_ParCSRMatrix *P = hypre_ParILUDataP(ilu_data); +#else + hypre_ParCSRMatrix *matmL = hypre_ParILUDataMatLModified(ilu_data); + HYPRE_Real *matmD = hypre_ParILUDataMatDModified(ilu_data); + hypre_ParCSRMatrix *matmU = hypre_ParILUDataMatUModified(ilu_data); #endif /* get matrices */ @@ -59,11 +63,6 @@ hypre_ILUSolve( void *ilu_vdata, hypre_ParCSRMatrix *matL = hypre_ParILUDataMatL(ilu_data); HYPRE_Real *matD = hypre_ParILUDataMatD(ilu_data); hypre_ParCSRMatrix *matU = hypre_ParILUDataMatU(ilu_data); -#ifndef HYPRE_USING_CUDA - hypre_ParCSRMatrix *matmL = hypre_ParILUDataMatLModified(ilu_data); - HYPRE_Real *matmD = hypre_ParILUDataMatDModified(ilu_data); - hypre_ParCSRMatrix *matmU = hypre_ParILUDataMatUModified(ilu_data); -#endif hypre_ParCSRMatrix *matS = hypre_ParILUDataMatS(ilu_data); HYPRE_Int iter, num_procs, my_id; @@ -97,7 +96,7 @@ hypre_ILUSolve( void *ilu_vdata, HYPRE_Real operat_cmplxty = hypre_ParILUDataOperatorComplexity(ilu_data); HYPRE_Int Solve_err_flag; -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) HYPRE_Int test_opt; #endif @@ -147,7 +146,6 @@ hypre_ILUSolve( void *ilu_vdata, hypre_printf("\n\n ILU SOLVER SOLUTION INFO:\n"); } - /*----------------------------------------------------------------------- * Compute initial residual and print *-----------------------------------------------------------------------*/ @@ -155,10 +153,10 @@ hypre_ILUSolve( void *ilu_vdata, { if ( logging > 1 ) { - hypre_ParVectorCopy(f, residual ); + hypre_ParVectorCopy(f, residual); if (tol > 0.0) { - hypre_ParCSRMatrixMatvec(alpha, A, u, beta, residual ); + hypre_ParCSRMatrixMatvec(alpha, A, u, beta, residual); } resnorm = sqrt(hypre_ParVectorInnerProd( residual, residual )); } @@ -247,7 +245,7 @@ hypre_ILUSolve( void *ilu_vdata, switch (ilu_type) { case 0: case 1: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* Apply GPU-accelerated LU solve */ hypre_ILUSolveCusparseLU(matA, matL_des, matU_des, matBL_info, matBU_info, matBLU_d, ilu_solve_policy, @@ -257,7 +255,7 @@ hypre_ILUSolve( void *ilu_vdata, #endif break; case 10: case 11: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* Apply GPU-accelerated LU solve */ hypre_ILUSolveCusparseSchurGMRES(matA, F_array, U_array, perm, nLU, matS, Utemp, Ftemp, schur_solver, schur_precond, rhs, x, u_end, @@ -280,7 +278,7 @@ hypre_ILUSolve( void *ilu_vdata, Utemp, Ftemp, schur_solver, schur_precond, rhs, x, u_end); //GMRES break; case 50: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) test_opt = hypre_ParILUDataTestOption(ilu_data); hypre_ILUSolveRAPGMRES(matA, F_array, U_array, perm, nLU, matS, Utemp, Ftemp, Xtemp, Ytemp, schur_solver, schur_precond, rhs, x, u_end, @@ -293,7 +291,7 @@ hypre_ILUSolve( void *ilu_vdata, #endif break; default: -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* Apply GPU-accelerated LU solve */ hypre_ILUSolveCusparseLU(matA, matL_des, matU_des, matBL_info, matBU_info, matBLU_d, ilu_solve_policy, @@ -988,7 +986,7 @@ hypre_ILUSolveLURAS(hypre_ParCSRMatrix *A, hypre_ParVector *f, return hypre_error_flag; } -#ifdef HYPRE_USING_CUDA +#if defined(HYPRE_USING_CUDA) && defined(HYPRE_USING_CUSPARSE) /* Permutation function (for GPU version, can just call thrust) * option 00: perm integer array @@ -2084,7 +2082,7 @@ hypre_NSHSolve( void *nsh_vdata, { old_resnorm = resnorm; - if ( logging > 1 ) + if (logging > 1) { hypre_ParVectorCopy(F_array, residual); hypre_ParCSRMatrixMatvec(alpha, matA, U_array, beta, residual ); diff --git a/src/utilities/device_utils.c b/src/utilities/device_utils.c index 63e7976fd..f9668ba34 100644 --- a/src/utilities/device_utils.c +++ b/src/utilities/device_utils.c @@ -39,6 +39,8 @@ hypre_DeviceDataCreate() hypre_DeviceDataSpTransUseVendor(data) = 1; #else hypre_DeviceDataSpgemmUseVendor(data) = 0; + hypre_DeviceDataSpMVUseVendor(data) = 0; + hypre_DeviceDataSpTransUseVendor(data) = 0; #endif /* for CUDA, it seems cusparse is slow due to memory allocation inside the transposition */ #if defined(HYPRE_USING_CUDA)