CFD/hypre
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

generalize communicator in assumed partition and moved Gussian elimination

Browse Source 
matrix iteration into setup phase
This commit is contained in:
ulrikey 2012-05-24 22:46:26 +00:00
parent 3a1748b414
commit ea4b7462f9
16 changed files with 383 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
IJ_mv/IJMatrix_parcsr.c
View File

@ -2377,7 +2377,7 @@ hypre_IJMatrixAssembleOffProcValsParCSR( hypre_IJMatrix *matrix,
row_list[i] = row;
row_list_num_elements[i] = off_proc_i[i*2+1];
hypre_GetAssumedPartitionProcFromRow (row, global_num_cols, &proc_id);
hypre_GetAssumedPartitionProcFromRow (comm, row, global_num_cols, &proc_id);
a_proc_id[i] = proc_id;
orig_order[i] = i;
}
@ -2429,7 +2429,7 @@ hypre_IJMatrixAssembleOffProcValsParCSR( hypre_IJMatrix *matrix,
ex_contact_buf[counter*2] = row_list[i];
counter++;
hypre_GetAssumedPartitionRowRange(proc_id, global_num_cols,
hypre_GetAssumedPartitionRowRange(comm, proc_id, global_num_cols,
&range_start, &range_end);

4
IJ_mv/IJVector_parcsr.c
View File

@ -1174,7 +1174,7 @@ hypre_IJVectorAssembleOffProcValsPar( hypre_IJVector *vector,
{
row = off_proc_i[i];
row_list[i] = row;
hypre_GetAssumedPartitionProcFromRow (row, global_num_rows, &proc_id);
hypre_GetAssumedPartitionProcFromRow (comm, row, global_num_rows, &proc_id);
a_proc_id[i] = proc_id;
orig_order[i] = i;
}
@ -1224,7 +1224,7 @@ hypre_IJVectorAssembleOffProcValsPar( hypre_IJVector *vector,
ex_contact_buf[counter*2] = row_list[i];
counter++;
hypre_GetAssumedPartitionRowRange(proc_id, global_num_rows,
hypre_GetAssumedPartitionRowRange(comm, proc_id, global_num_rows,
&range_start, &range_end);

4
parcsr_block_mv/par_csr_block_comm.c
View File

@ -186,7 +186,7 @@ hypre_ParCSRBlockMatrixCreateAssumedPartition( hypre_ParCSRBlockMatrix *matrix)
/* get my assumed partitioning - we want partitioning of the vector that the
matrix multiplies - so we use the col start and end */
hypre_GetAssumedPartitionRowRange( myid, global_num_cols, &(apart->row_start),
hypre_GetAssumedPartitionRowRange( comm, myid, global_num_cols, &(apart->row_start),
&(apart->row_end));
/*allocate some space for the partition of the assumed partition */
@ -199,7 +199,7 @@ hypre_ParCSRBlockMatrixCreateAssumedPartition( hypre_ParCSRBlockMatrix *matrix)
/* now we want to reconcile our actual partition with the assumed partition */
hypre_LocateAssummedPartition(col_start, col_end, global_num_cols, apart, myid);
hypre_LocateAssummedPartition(comm, col_start, col_end, global_num_cols, apart, myid);
/* this partition will be saved in the matrix data structure until the matrix is destroyed */
hypre_ParCSRBlockMatrixAssumedPartition(matrix) = apart;

13
parcsr_ls/_hypre_parcsr_ls.h
View File

@ -223,6 +223,11 @@ typedef struct
hypre_ParVector *u_coarse;
MPI_Comm new_comm;
/* store matrix, vector and communication info for Gaussian elimination */
double *A_mat;
double *b_vec;
HYPRE_Int *comm_info;
} hypre_ParAMGData;
/*--------------------------------------------------------------------------
@ -397,6 +402,10 @@ typedef struct
#define hypre_ParAMGDataUCoarse(amg_data) ((amg_data)->u_coarse)
#define hypre_ParAMGDataNewComm(amg_data) ((amg_data)->new_comm)
#define hypre_ParAMGDataAMat(amg_data) ((amg_data)->A_mat)
#define hypre_ParAMGDataBVec(amg_data) ((amg_data)->b_vec)
#define hypre_ParAMGDataCommInfo(amg_data) ((amg_data)->comm_info)
#endif
@ -578,6 +587,8 @@ HYPRE_Int BuildParLaplacian27pt ( HYPRE_Int argc , char *argv [], HYPRE_Int arg_
/* gen_redcs_mat.c */
HYPRE_Int hypre_seqAMGSetup ( hypre_ParAMGData *amg_data , HYPRE_Int p_level , HYPRE_Int coarse_threshold );
HYPRE_Int hypre_seqAMGCycle ( hypre_ParAMGData *amg_data , HYPRE_Int p_level , hypre_ParVector **Par_F_array , hypre_ParVector **Par_U_array );
HYPRE_Int hypre_GenerateSubComm ( MPI_Comm comm , HYPRE_Int participate , MPI_Comm *new_comm_ptr );
void merge_lists ( HYPRE_Int *list1 , HYPRE_Int *list2 , hypre_int *np1 , hypre_MPI_Datatype *dptr );
/* HYPRE_ads.c */
HYPRE_Int HYPRE_ADSCreate ( HYPRE_Solver *solver );
@ -1302,6 +1313,8 @@ HYPRE_Int hypre_GenerateSendMapAndCommPkg ( MPI_Comm comm , HYPRE_Int num_sends
/* par_relax.c */
HYPRE_Int hypre_BoomerAMGRelax ( hypre_ParCSRMatrix *A , hypre_ParVector *f , HYPRE_Int *cf_marker , HYPRE_Int relax_type , HYPRE_Int relax_points , double relax_weight , double omega , double *l1_norms , hypre_ParVector *u , hypre_ParVector *Vtemp , hypre_ParVector *Ztemp );
HYPRE_Int hypre_GaussElimSetup ( hypre_ParAMGData *amg_data , HYPRE_Int level , HYPRE_Int relax_type );
HYPRE_Int hypre_GaussElimSolve ( void *amg_vdata , HYPRE_Int level , HYPRE_Int relax_type );
HYPRE_Int gselim ( double *A , double *x , HYPRE_Int n );
/* par_relax_interface.c */

122
parcsr_ls/gen_redcs_mat.c
View File

@ -75,8 +75,12 @@ HYPRE_Int hypre_seqAMGSetup( hypre_ParAMGData *amg_data,
double *A_offd_data = hypre_CSRMatrixData(A_offd);
HYPRE_Int num_rows = hypre_CSRMatrixNumRows(A_diag);
HYPRE_Int first_row_index = hypre_ParCSRMatrixFirstRowIndex(A);
HYPRE_Int new_num_procs, *row_starts;
hypre_MPI_Group orig_group, new_group;
hypre_GenerateSubComm(comm, num_rows, &new_comm);
/*hypre_MPI_Group orig_group, new_group;
HYPRE_Int *ranks, new_num_procs, *row_starts;
info = hypre_CTAlloc(HYPRE_Int, num_procs);
@ -93,14 +97,19 @@ HYPRE_Int hypre_seqAMGSetup( hypre_ParAMGData *amg_data,
info[new_num_procs++] = info[i];
}
MPI_Comm_group(comm, &orig_group);
hypre_MPI_Comm_group(comm, &orig_group);
hypre_MPI_Group_incl(orig_group, new_num_procs, ranks, &new_group);
MPI_Comm_create(comm, new_group, &new_comm);
hypre_MPI_Comm_create(comm, new_group, &new_comm);
hypre_MPI_Group_free(&new_group);
hypre_MPI_Group_free(&orig_group);
hypre_MPI_Group_free(&orig_group); */
if (num_rows)
{
hypre_MPI_Comm_size(new_comm, &new_num_procs);
info = hypre_CTAlloc(HYPRE_Int, new_num_procs);
hypre_MPI_Allgather(&num_rows, 1, HYPRE_MPI_INT, info, 1, HYPRE_MPI_INT, new_comm);
/* alloc space in seq data structure only for participating procs*/
HYPRE_BoomerAMGCreate(&coarse_solver);
HYPRE_BoomerAMGSetMaxRowSum(coarse_solver,
@ -202,6 +211,7 @@ HYPRE_Int hypre_seqAMGSetup( hypre_ParAMGData *amg_data,
A_seq_data, info, displs2,
hypre_MPI_DOUBLE, new_comm );
hypre_TFree(info);
hypre_TFree(displs);
hypre_TFree(displs2);
hypre_TFree(A_tmp_i);
@ -242,8 +252,6 @@ HYPRE_Int hypre_seqAMGSetup( hypre_ParAMGData *amg_data,
hypre_ParAMGDataUCoarse(amg_data) = U_seq;
hypre_ParAMGDataNewComm(amg_data) = new_comm;
}
hypre_TFree(info);
hypre_TFree(ranks);
}
return 0;
@ -362,3 +370,105 @@ hypre_seqAMGCycle( hypre_ParAMGData *amg_data,
return(Solve_err_flag);
}
/* generate sub communicator, which contains only idle processors */
HYPRE_Int hypre_GenerateSubComm(MPI_Comm comm, HYPRE_Int participate, MPI_Comm *new_comm_ptr)
{
MPI_Comm new_comm;
hypre_MPI_Group orig_group, new_group;
hypre_MPI_Op hypre_MPI_MERGE;
HYPRE_Int *ranks, *info, new_num_procs, my_info, my_id, num_procs;
hypre_int *list_len;
hypre_MPI_Comm_rank(comm,&my_id);
if (participate)
my_info = 1;
else
my_info = 0;
hypre_MPI_Allreduce(&my_info, &new_num_procs, 1, HYPRE_MPI_INT, hypre_MPI_SUM, comm);
ranks = hypre_CTAlloc(HYPRE_Int, new_num_procs+2);
if (new_num_procs == 1)
{
if (participate) ranks[2] = my_id;
hypre_MPI_Bcast ( ranks, 1, HYPRE_MPI_INT, my_id, comm);
}
else
{
info = hypre_CTAlloc(HYPRE_Int, new_num_procs+2);
list_len = hypre_CTAlloc(HYPRE_Int, 1);
if (participate)
{
info[0] = 1;
info[1] = 1;
info[2] = my_id;
}
else
info[0] = 0;
list_len[0] = (hypre_int)new_num_procs + 2;
hypre_MPI_Op_create((hypre_MPI_User_function *)merge_lists, 0, &hypre_MPI_MERGE);
hypre_MPI_Allreduce(info, ranks, list_len[0], HYPRE_MPI_INT, hypre_MPI_MERGE, comm);
hypre_MPI_Op_free (&hypre_MPI_MERGE);
hypre_TFree(list_len);
hypre_TFree(info);
}
hypre_MPI_Comm_size(comm,&num_procs);
hypre_MPI_Comm_group(comm, &orig_group);
hypre_MPI_Group_incl(orig_group, new_num_procs, &ranks[2], &new_group);
hypre_MPI_Comm_create(comm, new_group, &new_comm);
hypre_MPI_Group_free(&new_group);
hypre_MPI_Group_free(&orig_group);
hypre_TFree(ranks);
*new_comm_ptr = new_comm;
return 0;
}
void merge_lists (HYPRE_Int *list1, HYPRE_Int* list2, hypre_int *np1, hypre_MPI_Datatype *dptr)
{
HYPRE_Int i, len1, len2, indx1, indx2;
if (list1[0] == 0 || (list2[0] == 0 && list1[0] == 0))
{
return;
}
else
{
list2[0] = 1;
len1 = list1[1];
len2 = list2[1];
list2[1] = len1+len2;
if ((hypre_int)(list2[1]) > *np1+2) printf("segfault in MPI User function merge_list\n");
indx1 = len1+1;
indx2 = len2+1;
for (i=len1+len2+1; i > 1; i--)
{
if (indx2 > 1 && indx1 > 1 && list1[indx1] > list2[indx2])
{
list2[i] = list1[indx1];
indx1--;
}
else if (indx2 > 1)
{
list2[i] = list2[indx2];
indx2--;
}
else if (indx1 > 1)
{
list2[i] = list1[indx1];
indx1--;
}
}
}
}

9
parcsr_ls/par_amg.c
View File

@ -342,6 +342,11 @@ hypre_BoomerAMGCreate()
hypre_ParAMGDataUCoarse(amg_data) = NULL;
hypre_ParAMGDataNewComm(amg_data) = hypre_MPI_COMM_NULL;
/* for Gaussian elimination coarse grid solve */
hypre_ParAMGDataAMat(amg_data) = NULL;
hypre_ParAMGDataBVec(amg_data) = NULL;
hypre_ParAMGDataCommInfo(amg_data) = NULL;
return (void *) amg_data;
}
@ -560,6 +565,10 @@ hypre_BoomerAMGDestroy( void *data )
if (hypre_ParAMGDataFCoarse(amg_data))
hypre_ParVectorDestroy(hypre_ParAMGDataFCoarse(amg_data));
if (hypre_ParAMGDataAMat(amg_data)) hypre_TFree(hypre_ParAMGDataAMat(amg_data));
if (hypre_ParAMGDataBVec(amg_data)) hypre_TFree(hypre_ParAMGDataBVec(amg_data));
if (hypre_ParAMGDataCommInfo(amg_data)) hypre_TFree(hypre_ParAMGDataCommInfo(amg_data));
if (new_comm != hypre_MPI_COMM_NULL)
{
MPI_Comm_free (&new_comm);

9
parcsr_ls/par_amg.h
View File

@ -192,6 +192,11 @@ typedef struct
hypre_ParVector *u_coarse;
MPI_Comm new_comm;
/* store matrix, vector and communication info for Gaussian elimination */
double *A_mat;
double *b_vec;
HYPRE_Int *comm_info;
} hypre_ParAMGData;
/*--------------------------------------------------------------------------
@ -366,6 +371,10 @@ typedef struct
#define hypre_ParAMGDataUCoarse(amg_data) ((amg_data)->u_coarse)
#define hypre_ParAMGDataNewComm(amg_data) ((amg_data)->new_comm)
#define hypre_ParAMGDataAMat(amg_data) ((amg_data)->A_mat)
#define hypre_ParAMGDataBVec(amg_data) ((amg_data)->b_vec)
#define hypre_ParAMGDataCommInfo(amg_data) ((amg_data)->comm_info)
#endif

5
parcsr_ls/par_amg_setup.c
View File

@ -1743,7 +1743,10 @@ hypre_BoomerAMGSetup( void *amg_vdata,
{
hypre_seqAMGSetup( amg_data, level, coarse_threshold);
}
else if (grid_relax_type[3] == 9 || grid_relax_type[3] == 99) /*use of Gaussian elimination on coarsest level */
{
hypre_GaussElimSetup(amg_data, level, grid_relax_type[3]);
}
if (level > 0)
{

5
parcsr_ls/par_cycle.c
View File

@ -377,6 +377,11 @@ hypre_BoomerAMGCycle( void *amg_vdata,
(HYPRE_ParVector) Aux_F,
(HYPRE_ParVector) Aux_U);
}
/*else if (relax_type == 99)*/
else if (relax_type == 9 || relax_type == 99)
{ /* Gaussian elimination */
hypre_GaussElimSolve(amg_data, level, relax_type);
}
else if (relax_type == 18)
{ /* L1 - Jacobi*/
if (relax_order == 1 && cycle_type < 3)

167
parcsr_ls/par_relax.c
View File

@ -3091,8 +3091,6 @@ HYPRE_Int hypre_BoomerAMGRelax( hypre_ParCSRMatrix *A,
}
relax_error = gselim(A_mat,b_vec,n_global);
/* use version with pivoting */
/* relax_error = gselim_piv(A_mat,b_vec,n_global);*/
for (i = 0; i < n; i++)
{
@ -3228,6 +3226,171 @@ HYPRE_Int hypre_BoomerAMGRelax( hypre_ParCSRMatrix *A,
*
*------------------------------------------------------------------------ */
HYPRE_Int hypre_GaussElimSetup (hypre_ParAMGData *amg_data, HYPRE_Int level, HYPRE_Int relax_type)
{
/* Par Data Structure variables */
hypre_ParCSRMatrix *A = hypre_ParAMGDataAArray(amg_data)[level];
hypre_CSRMatrix *A_diag = hypre_ParCSRMatrixDiag(A);
HYPRE_Int num_rows = hypre_CSRMatrixNumRows(A_diag);
HYPRE_Int global_num_rows = hypre_ParCSRMatrixGlobalNumRows(A);
MPI_Comm comm = hypre_ParCSRMatrixComm(A);
MPI_Comm new_comm;
/* Generate sub communicator */
hypre_GenerateSubComm(comm, num_rows, &new_comm);
hypre_ParAMGDataNewComm(amg_data) = new_comm;
if (num_rows)
{
hypre_CSRMatrix *A_offd = hypre_ParCSRMatrixOffd(A);
HYPRE_Int *col_map_offd = hypre_ParCSRMatrixColMapOffd(A);
HYPRE_Int *A_diag_i = hypre_CSRMatrixI(A_diag);
HYPRE_Int *A_offd_i = hypre_CSRMatrixI(A_offd);
HYPRE_Int *A_diag_j = hypre_CSRMatrixJ(A_diag);
HYPRE_Int *A_offd_j = hypre_CSRMatrixJ(A_offd);
double *A_diag_data = hypre_CSRMatrixData(A_diag);
double *A_offd_data = hypre_CSRMatrixData(A_offd);
double *A_mat, *A_mat_local;
HYPRE_Int *comm_info, *info, *displs;
HYPRE_Int *mat_info, *mat_displs;
HYPRE_Int new_num_procs, A_mat_local_size, i, jj, column;
HYPRE_Int first_row_index = hypre_ParCSRMatrixFirstRowIndex(A);
hypre_MPI_Comm_size(new_comm, &new_num_procs);
comm_info = hypre_CTAlloc(HYPRE_Int, 2*new_num_procs+1);
mat_info = hypre_CTAlloc(HYPRE_Int, new_num_procs);
mat_displs = hypre_CTAlloc(HYPRE_Int, new_num_procs+1);
info = &comm_info[0];
displs = &comm_info[new_num_procs];
hypre_MPI_Allgather(&num_rows, 1, HYPRE_MPI_INT, info, 1, HYPRE_MPI_INT, new_comm);
displs[0] = 0;
mat_displs[0] = 0;
for (i=0; i < new_num_procs; i++)
{
displs[i+1] = displs[i]+info[i];
mat_displs[i+1] = global_num_rows*displs[i+1];
mat_info[i] = global_num_rows*info[i];
}
hypre_ParAMGDataBVec(amg_data) = hypre_CTAlloc(double, global_num_rows);
A_mat_local_size = global_num_rows*num_rows;
A_mat_local = hypre_CTAlloc(double, A_mat_local_size);
A_mat = hypre_CTAlloc(double, global_num_rows*global_num_rows);
/* load local matrix into A_mat_local */
for (i = 0; i < num_rows; i++)
{
for (jj = A_diag_i[i]; jj < A_diag_i[i+1]; jj++)
{
/* need col major */
column = A_diag_j[jj]+first_row_index;
A_mat_local[i*global_num_rows + column] = A_diag_data[jj];
}
for (jj = A_offd_i[i]; jj < A_offd_i[i+1]; jj++)
{
/* need col major */
column = col_map_offd[A_offd_j[jj]];
A_mat_local[i*global_num_rows + column] = A_offd_data[jj];
}
}
hypre_MPI_Allgatherv( A_mat_local, A_mat_local_size, hypre_MPI_DOUBLE, A_mat,
mat_info, mat_displs, hypre_MPI_DOUBLE, new_comm);
if (relax_type == 99)
{
double *AT_mat;
AT_mat = hypre_CTAlloc(double, global_num_rows*global_num_rows);
for (i=0; i < global_num_rows; i++)
for (jj=0; jj < global_num_rows; jj++)
AT_mat[i*global_num_rows + jj] = A_mat[i+ jj*global_num_rows];
hypre_ParAMGDataAMat(amg_data) = AT_mat;
hypre_TFree (A_mat);
}
else
hypre_ParAMGDataAMat(amg_data) = A_mat;
hypre_ParAMGDataCommInfo(amg_data) = comm_info;
hypre_TFree(mat_info);
hypre_TFree(mat_displs);
}
return hypre_error_flag;
}
HYPRE_Int hypre_GaussElimSolve (void *amg_vdata, HYPRE_Int level, HYPRE_Int relax_type)
{
hypre_ParAMGData *amg_data = amg_vdata;
hypre_ParCSRMatrix *A = hypre_ParAMGDataAArray(amg_data)[level];
HYPRE_Int n = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A));
HYPRE_Int error_flag = 0;
if (n)
{
MPI_Comm new_comm = hypre_ParAMGDataNewComm(amg_data);
hypre_ParVector *f = hypre_ParAMGDataFArray(amg_data)[level];
hypre_ParVector *u = hypre_ParAMGDataUArray(amg_data)[level];
double *A_mat = hypre_ParAMGDataAMat(amg_data);
double *b_vec = hypre_ParAMGDataBVec(amg_data);
double *f_data = hypre_VectorData(hypre_ParVectorLocalVector(f));
double *u_data = hypre_VectorData(hypre_ParVectorLocalVector(u));
double *A_tmp;
HYPRE_Int *comm_info = hypre_ParAMGDataCommInfo(amg_data);
HYPRE_Int *displs, *info;
HYPRE_Int n_global = hypre_ParCSRMatrixGlobalNumRows(A);
HYPRE_Int new_num_procs, i, my_info;
HYPRE_Int first_index = hypre_ParCSRMatrixFirstRowIndex(A);
HYPRE_Int one_i = 1;
hypre_MPI_Comm_size(new_comm, &new_num_procs);
info = &comm_info[0];
displs = &comm_info[new_num_procs];
hypre_MPI_Allgatherv ( f_data, n, hypre_MPI_DOUBLE,
b_vec, info, displs,
hypre_MPI_DOUBLE, new_comm );
A_tmp = hypre_CTAlloc (double, n_global*n_global);
for (i=0; i < n_global*n_global; i++)
A_tmp[i] = A_mat[i];
if (relax_type == 9)
{
error_flag = gselim(A_tmp,b_vec,n_global);
}
else if (relax_type == 99) /* use pivoting */
{
HYPRE_Int *piv;
piv = hypre_CTAlloc(HYPRE_Int, n_global);
/* write over A with LU */
#ifdef HYPRE_USING_ESSL
dgetrf(n_global, n_global, A_tmp, n_global, piv, &my_info);
#else
hypre_F90_NAME_LAPACK(dgetrf, DGETRF)(&n_global, &n_global,
A_tmp, &n_global, piv, &my_info);
#endif
/*now b_vec = inv(A)*b_vec */
#ifdef HYPRE_USING_ESSL
dgetrs("N", n_global, &one_i, A_tmp,
n_global, piv, b_vec,
n_global, &my_info);
#else
hypre_F90_NAME_LAPACK(dgetrs, DGETRS)("N", &n_global, &one_i, A_tmp,
&n_global, piv, b_vec,
&n_global, &my_info);
#endif
hypre_TFree(piv);
}
for (i = 0; i < n; i++)
{
u_data[i] = b_vec[first_index+i];
}
hypre_TFree(A_tmp);
}
if (error_flag) hypre_error(HYPRE_ERROR_GENERIC);
return hypre_error_flag;
}
HYPRE_Int gselim(A,x,n)
double *A;
double *x;

6
parcsr_mv/_hypre_parcsr_mv.h
View File

@ -622,11 +622,11 @@ hypre_ParCSRMatrix *hypre_ParCSRAAt ( hypre_ParCSRMatrix *A );
hypre_CSRMatrix *hypre_ParCSRMatrixExtractAExt ( hypre_ParCSRMatrix *A , HYPRE_Int data , HYPRE_Int **pA_ext_row_map );
/* par_csr_assumed_part.c */
HYPRE_Int hypre_LocateAssummedPartition ( HYPRE_Int row_start , HYPRE_Int row_end , HYPRE_Int global_num_rows , hypre_IJAssumedPart *part , HYPRE_Int myid );
HYPRE_Int hypre_LocateAssummedPartition ( MPI_Comm comm , HYPRE_Int row_start , HYPRE_Int row_end , HYPRE_Int global_num_rows , hypre_IJAssumedPart *part , HYPRE_Int myid );
HYPRE_Int hypre_ParCSRMatrixCreateAssumedPartition ( hypre_ParCSRMatrix *matrix );
HYPRE_Int hypre_ParCSRMatrixDestroyAssumedPartition ( hypre_ParCSRMatrix *matrix );
HYPRE_Int hypre_GetAssumedPartitionProcFromRow ( HYPRE_Int row , HYPRE_Int global_num_rows , HYPRE_Int *proc_id );
HYPRE_Int hypre_GetAssumedPartitionRowRange ( HYPRE_Int proc_id , HYPRE_Int global_num_rows , HYPRE_Int *row_start , HYPRE_Int *row_end );
HYPRE_Int hypre_GetAssumedPartitionProcFromRow ( MPI_Comm comm , HYPRE_Int row , HYPRE_Int global_num_rows , HYPRE_Int *proc_id );
HYPRE_Int hypre_GetAssumedPartitionRowRange ( MPI_Comm comm , HYPRE_Int proc_id , HYPRE_Int global_num_rows , HYPRE_Int *row_start , HYPRE_Int *row_end );
HYPRE_Int hypre_ParVectorCreateAssumedPartition ( hypre_ParVector *vector );
HYPRE_Int hypre_ParVectorDestroyAssumedPartition ( hypre_ParVector *vector );

4
parcsr_mv/new_commpkg.c
View File

@ -219,7 +219,7 @@ hypre_NewCommPkgCreate_core(
{
hypre_GetAssumedPartitionProcFromRow(col_map_off_d[i],
hypre_GetAssumedPartitionProcFromRow(comm, col_map_off_d[i],
global_num_cols, &tmp_id);
if (ex_num_contacts == size) /*need more space? */
@ -241,7 +241,7 @@ hypre_NewCommPkgCreate_core(
ex_num_contacts++;
hypre_GetAssumedPartitionRowRange(tmp_id, global_num_cols,
hypre_GetAssumedPartitionRowRange(comm, tmp_id, global_num_cols,
&range_start, &range_end);
}

34
parcsr_mv/par_csr_assumed_part.c
View File

@ -32,7 +32,8 @@
HYPRE_Int
hypre_LocateAssummedPartition(HYPRE_Int row_start, HYPRE_Int row_end, HYPRE_Int global_num_rows,
hypre_LocateAssummedPartition(MPI_Comm comm, HYPRE_Int row_start, HYPRE_Int row_end,
HYPRE_Int global_num_rows,
hypre_IJAssumedPart *part, HYPRE_Int myid)
{
@ -122,9 +123,9 @@ hypre_LocateAssummedPartition(HYPRE_Int row_start, HYPRE_Int row_end, HYPRE_Int
{
/*get start and end row owners */
hypre_GetAssumedPartitionProcFromRow(contact_row_start[i], global_num_rows,
hypre_GetAssumedPartitionProcFromRow(comm, contact_row_start[i], global_num_rows,
&owner_start);
hypre_GetAssumedPartitionProcFromRow(contact_row_end[i], global_num_rows,
hypre_GetAssumedPartitionProcFromRow(comm, contact_row_end[i], global_num_rows,
&owner_end);
if (owner_start == owner_end) /* same processor owns the whole range */
@ -146,7 +147,7 @@ hypre_LocateAssummedPartition(HYPRE_Int row_start, HYPRE_Int row_end, HYPRE_Int
complete = 0;
while (!complete)
{
hypre_GetAssumedPartitionRowRange(owner_start, global_num_rows,
hypre_GetAssumedPartitionRowRange(comm, owner_start, global_num_rows,
&tmp_row_start, &tmp_row_end);
if (tmp_row_end >= contact_row_end[i])
@ -184,7 +185,8 @@ hypre_LocateAssummedPartition(HYPRE_Int row_start, HYPRE_Int row_end, HYPRE_Int
for (i=0; i< contact_list_length; i++)
{
hypre_MPI_Isend(&CONTACT(i,1) ,2, HYPRE_MPI_INT, CONTACT(i,0), flag1 ,
hypre_MPI_COMM_WORLD, &requests[i]);
comm, &requests[i]);
/*hypre_MPI_COMM_WORLD, &requests[i]);*/
}
/*-----------------------------------------------------------
@ -264,7 +266,8 @@ hypre_LocateAssummedPartition(HYPRE_Int row_start, HYPRE_Int row_end, HYPRE_Int
while (rows_found != locate_row_count) {
hypre_MPI_Recv( tmp_range, 2 , HYPRE_MPI_INT, hypre_MPI_ANY_SOURCE,
flag1 , hypre_MPI_COMM_WORLD, &status0);
flag1 , comm, &status0);
/*flag1 , hypre_MPI_COMM_WORLD, &status0);*/
if (part->length==part->storage_length)
{
@ -350,7 +353,7 @@ hypre_ParCSRMatrixCreateAssumedPartition( hypre_ParCSRMatrix *matrix)
/* get my assumed partitioning - we want partitioning of the vector that the
matrix multiplies - so we use the col start and end */
hypre_GetAssumedPartitionRowRange( myid, global_num_cols, &(apart->row_start),
hypre_GetAssumedPartitionRowRange( comm, myid, global_num_cols, &(apart->row_start),
&(apart->row_end));
/*allocate some space for the partition of the assumed partition */
@ -363,7 +366,7 @@ hypre_ParCSRMatrixCreateAssumedPartition( hypre_ParCSRMatrix *matrix)
/* now we want to reconcile our actual partition with the assumed partition */
hypre_LocateAssummedPartition(col_start, col_end, global_num_cols, apart, myid);
hypre_LocateAssummedPartition(comm, col_start, col_end, global_num_cols, apart, myid);
/* this partition will be saved in the matrix data structure until the matrix is destroyed */
hypre_ParCSRMatrixAssumedPartition(matrix) = apart;
@ -407,14 +410,16 @@ hypre_ParCSRMatrixDestroyAssumedPartition(hypre_ParCSRMatrix *matrix )
HYPRE_Int
hypre_GetAssumedPartitionProcFromRow( HYPRE_Int row, HYPRE_Int global_num_rows, HYPRE_Int *proc_id)
hypre_GetAssumedPartitionProcFromRow( MPI_Comm comm,
HYPRE_Int row, HYPRE_Int global_num_rows, HYPRE_Int *proc_id)
{
HYPRE_Int num_procs;
HYPRE_Int size, switch_row, extra;
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
hypre_MPI_Comm_size(comm, &num_procs );
/*hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );*/
/* j = floor[(row*p/N] - this overflows*/
/* *proc_id = (row*num_procs)/global_num_rows;*/
@ -449,7 +454,7 @@ hypre_GetAssumedPartitionProcFromRow( HYPRE_Int row, HYPRE_Int global_num_rows,
HYPRE_Int
hypre_GetAssumedPartitionRowRange( HYPRE_Int proc_id, HYPRE_Int global_num_rows,
hypre_GetAssumedPartitionRowRange( MPI_Comm comm, HYPRE_Int proc_id, HYPRE_Int global_num_rows,
HYPRE_Int *row_start, HYPRE_Int* row_end)
{
@ -457,7 +462,8 @@ hypre_GetAssumedPartitionRowRange( HYPRE_Int proc_id, HYPRE_Int global_num_rows,
HYPRE_Int size, extra;
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
hypre_MPI_Comm_size(comm, &num_procs );
/*hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );*/
/* this may look non-intuitive, but we have to be very careful that
@ -520,7 +526,7 @@ hypre_ParVectorCreateAssumedPartition( hypre_ParVector *vector)
/* get my assumed partitioning - we want partitioning of the vector that the
matrix multiplies - so we use the col start and end */
hypre_GetAssumedPartitionRowRange( myid, global_num, &(apart->row_start),
hypre_GetAssumedPartitionRowRange( comm, myid, global_num, &(apart->row_start),
&(apart->row_end));
/*allocate some space for the partition of the assumed partition */
@ -533,7 +539,7 @@ hypre_ParVectorCreateAssumedPartition( hypre_ParVector *vector)
/* now we want to reconcile our actual partition with the assumed partition */
hypre_LocateAssummedPartition(start, end, global_num, apart, myid);
hypre_LocateAssummedPartition(comm, start, end, global_num, apart, myid);
/* this partition will be saved in the vector data structure until the vector is destroyed */
hypre_ParVectorAssumedPartition(vector) = apart;

7
utilities/_hypre_utilities.h
View File

@ -190,6 +190,9 @@ extern "C" {
#define MPI_Type_struct hypre_MPI_Type_struct
#define MPI_Type_commit hypre_MPI_Type_commit
#define MPI_Type_free hypre_MPI_Type_free
#define MPI_Op_free hypre_MPI_Op_free
#define MPI_Op_create hypre_MPI_Op_create
#define MPI_User_function hypre_MPI_User_function
/*--------------------------------------------------------------------------
* Types, etc.
@ -200,6 +203,7 @@ typedef HYPRE_Int hypre_MPI_Comm;
typedef HYPRE_Int hypre_MPI_Group;
typedef HYPRE_Int hypre_MPI_Request;
typedef HYPRE_Int hypre_MPI_Datatype;
typedef HYPRE_Int hypre_MPI_User_function;
typedef struct
{
@ -244,6 +248,7 @@ typedef MPI_Datatype hypre_MPI_Datatype;
typedef MPI_Status hypre_MPI_Status;
typedef MPI_Op hypre_MPI_Op;
typedef MPI_Aint hypre_MPI_Aint;
typedef MPI_User_function hypre_MPI_User_function;
#define hypre_MPI_COMM_WORLD MPI_COMM_WORLD
#define hypre_MPI_COMM_NULL MPI_COMM_NULL
@ -330,6 +335,8 @@ HYPRE_Int hypre_MPI_Type_hvector( HYPRE_Int count , HYPRE_Int blocklength , hypr
HYPRE_Int hypre_MPI_Type_struct( HYPRE_Int count , HYPRE_Int *array_of_blocklengths , hypre_MPI_Aint *array_of_displacements , hypre_MPI_Datatype *array_of_types , hypre_MPI_Datatype *newtype );
HYPRE_Int hypre_MPI_Type_commit( hypre_MPI_Datatype *datatype );
HYPRE_Int hypre_MPI_Type_free( hypre_MPI_Datatype *datatype );
HYPRE_Int hypre_MPI_Op_free( hypre_MPI_Op *op );
HYPRE_Int hypre_MPI_Op_create( hypre_MPI_User_function *function , hypre_int commute , hypre_MPI_Op *op );
#ifdef __cplusplus
}

1
utilities/exchange_data.c
View File

@ -374,6 +374,7 @@ HYPRE_Int hypre_DataExchangeList(HYPRE_Int num_contacts,
hypre_MPI_Isend(post_array[post_array_size], size,
hypre_MPI_BYTE, proc, post_tag,
hypre_MPI_COMM_WORLD,
/*comm,*/
&post_send_requests[post_array_size]);
post_array_size++;

23
utilities/mpistubs.c
View File

@ -565,6 +565,17 @@ hypre_MPI_Type_free( hypre_MPI_Datatype *datatype )
return(0);
}
HYPRE_Int
hypre_MPI_Op_create( hypre_MPI_User_function *function, hypre_int commute, hypre_MPI_Op *op )
{
return(0);
}
HYPRE_Int
hypre_MPI_Op_free( hypre_MPI_Op *op )
{
return(0);
}
/******************************************************************************
* MPI stubs to do casting of HYPRE_Int and hypre_int correctly
*****************************************************************************/
@ -1100,4 +1111,16 @@ hypre_MPI_Type_free( hypre_MPI_Datatype *datatype )
return (HYPRE_Int) MPI_Type_free(datatype);
}
HYPRE_Int
hypre_MPI_Op_free( hypre_MPI_Op *op )
{
return (HYPRE_Int) MPI_Op_free(op);
}
HYPRE_Int
hypre_MPI_Op_create( hypre_MPI_User_function *function, hypre_int commute, hypre_MPI_Op *op )
{
return (HYPRE_Int) MPI_Op_create(function, commute, op);
}
#endif